Merge nitotv apt fixes into 1.4.81.4.8+nitotv

55 files changed, 1971 insertions, 1509 deletions

diff --git a/CMake/Documentation.cmake b/CMake/Documentation.cmake
index f3bbfdc6b..e5735873b 100644
--- a/CMake/Documentation.cmake
+++ b/CMake/Documentation.cmake
@@ -34,6 +34,8 @@ find_path(DOCBOOK_XSL manpages/docbook.xsl
           /usr/share/xml/docbook/xsl-stylesheets
          # Fedora
          /usr/share/sgml/docbook/xsl-stylesheets
+	 #Brew
+	 /usr/local/Cellar/docbook-xsl/1.79.1/docbook-xsl/
          # Fink
          ${CMAKE_INSTALL_PREFIX}/share/xml/xsl/docbook-xsl
          # FreeBSD
@@ -88,7 +90,7 @@ endfunction()
 
 # Process one document
 function(po4a_one stamp_out out full_document language deps)
-    path_join(full_path "${CMAKE_CURRENT_SOURCE_DIR}" "${full_document}")
+	path_join(full_path "${CMAKE_CURRENT_SOURCE_DIR}" "${full_document}")
     po4a_components(document _ section ext "${full_document}")
 
     # Calculate target file name
@@ -266,7 +268,7 @@ function(add_docbook target)
 
     foreach(document ${DOC_DOCUMENTS})
         foreach(lang ${DOC_LINGUAS})
-            po4a_one(po4a_stamp po4a_out ${document} "${lang}" "${DOC_DEPENDS}")
+	    po4a_one(po4a_stamp po4a_out ${document} "${lang}" "${DOC_DEPENDS}")
             xsltproc_one(STAMP_OUT xslt_stamp
                          STAMP ${po4a_stamp}
                          FULL_DOCUMENT ${po4a_out}
diff --git a/CMake/Misc.cmake b/CMake/Misc.cmake
index 6ad0b9479..72bd61f14 100644
--- a/CMake/Misc.cmake
+++ b/CMake/Misc.cmake
@@ -66,16 +66,16 @@ endfunction()
 
 # Generates a simple version script versioning everything with current SOVERSION
 function(add_version_script target)
-    get_target_property(soversion ${target} SOVERSION)
-    set(script "${CMAKE_CURRENT_BINARY_DIR}/${target}.versionscript")
-    string(REPLACE "-" "" name "${target}_${soversion}")
-    string(TOUPPER "${name}" name)
-    add_custom_command(OUTPUT "${script}"
-                       COMMAND echo "${name} {global: *; };" > "${script}"
-                       VERBATIM )
-    add_custom_target(${target}-versionscript DEPENDS "${script}")
-    target_link_libraries(${target} PRIVATE -Wl,-version-script="${script}")
-    add_dependencies(${target} ${target}-versionscript)
+	#get_target_property(soversion ${target} SOVERSION)
+    	#set(script "${CMAKE_CURRENT_BINARY_DIR}/${target}.versionscript")
+    	#string(REPLACE "-" "" name "${target}_${soversion}")
+    	#string(TOUPPER "${name}" name)
+    	#add_custom_command(OUTPUT "${script}"
+    	#                   COMMAND echo "${name} {global: *; };" > "${script}"
+    	#                   VERBATIM )
+    	#add_custom_target(${target}-versionscript DEPENDS "${script}")
+    	#target_link_libraries(${target} PRIVATE -Wl,-version-script="${script}")
+    	#add_dependencies(${target} ${target}-versionscript)
 endfunction()
 
 function(path_join out path1 path2)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 7ad363676..3d257ceb1 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -13,12 +13,15 @@ enable_testing()
 option(WITH_DOC "Build documentation." ON)
 option(USE_NLS "Localisation support." ON)
 
+INCLUDE_DIRECTORIES(/usr/src/skel/usr/include)
+LINK_DIRECTORIES(/usr/src/skel/usr/lib)
+
 set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/CMake")
 
 # Add coverage target
 set(CMAKE_CXX_FLAGS_COVERAGE "-g -fprofile-arcs -ftest-coverage")
-set(CMAKE_EXE_LINKER_FLAGS_COVERAGE "-lgcov")
-set(CMAKE_SHARED_LINKER_FLAGS_COVERAGE "-lgcov")
+set(CMAKE_EXE_LINKER_FLAGS_COVERAGE "-lgcov -lSystem")
+set(CMAKE_SHARED_LINKER_FLAGS_COVERAGE "-lgcov -lSystem")
 
 # Work around bug in GNUInstallDirs
 if (EXISTS "/etc/debian_version")
@@ -211,7 +214,9 @@ add_subdirectory(apt-private)
 add_subdirectory(apt-inst)
 add_subdirectory(cmdline)
 add_subdirectory(completions)
+if (WITH_DOC)
 add_subdirectory(doc)
+endif()
 add_subdirectory(dselect)
 add_subdirectory(ftparchive)
 add_subdirectory(methods)
diff --git a/apt-pkg/CMakeLists.txt b/apt-pkg/CMakeLists.txt
index e3e078b81..8d7fcd9e6 100644
--- a/apt-pkg/CMakeLists.txt
+++ b/apt-pkg/CMakeLists.txt
@@ -37,7 +37,7 @@ file(GLOB_RECURSE headers "*.h")
 
 # Create a library using the C++ files
 add_library(apt-pkg SHARED ${library})
-add_dependencies(apt-pkg apt-pkg-versionscript)
+#add_dependencies(apt-pkg apt-pkg-versionscript)
 # Link the library and set the SONAME
 target_include_directories(apt-pkg
                            PRIVATE ${ZLIB_INCLUDE_DIRS}
diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index 3c85f8adf..231c5eed9 100644
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -266,7 +266,7 @@ static bool APT_NONNULL(3, 4, 5) AllowInsecureRepositories(InsecureType const ms
 
    if (TargetIsAllowedToBe(TransactionManager->Target, msg) == true)
    {
-      MessageInsecureRepository(false, msgstr, repo);
+      //MessageInsecureRepository(false, msgstr, repo);
       return true;
    }
 
@@ -1294,7 +1294,6 @@ bool pkgAcqMetaBase::CheckDownloadDone(pkgAcqTransactionItem * const I, const st
    {
       // for simplicity, the transaction manager is always InRelease
       // even if it doesn't exist.
-      TransactionManager->IMSHit = true;
       I->PartialFile = I->DestFile = I->GetFinalFilename();
    }
 
@@ -1412,7 +1411,7 @@ void pkgAcqMetaClearSig::QueueIndexes(bool const verify)			/*{{{*/
 	 if (TransactionManager->MetaIndexParser->Exists(Target.MetaKey) == false)
 	 {
 	    // optional targets that we do not have in the Release file are skipped
-	    if (hasHashes == true && Target.IsOptional)
+	    if (Target.IsOptional)
 	    {
 	       new CleanupItem(Owner, TransactionManager, Target);
 	       continue;
@@ -1536,6 +1535,13 @@ void pkgAcqMetaClearSig::QueueIndexes(bool const verify)			/*{{{*/
       }
       else
       {
+	 // if the source wanted these files they should have given us a release file :/
+	 if (Target.IsOptional)
+	 {
+	    new CleanupItem(Owner, TransactionManager, Target);
+	    continue;
+	 }
+
 	 // if we have no file to patch, no point in trying
 	 trypdiff &= (GetExistingFilename(GetFinalFileNameFromURI(Target.URI)).empty() == false);
       }
@@ -3070,6 +3076,8 @@ void pkgAcqIndex::StageDownloadDone(string const &Message)
    {
       // copy FinalFile into partial/ so that we check the hash again
       string const FinalFile = GetExistingFilename(GetFinalFileNameFromURI(Target.URI));
+      DestFile = GetKeepCompressedFileName(GetPartialFileNameFromURI(Target.URI), Target);
+      unlink(DestFile.c_str());
       if (symlink(FinalFile.c_str(), DestFile.c_str()) != 0)
 	 _error->WarningE("pkgAcqIndex::StageDownloadDone", "Symlinking final file %s back to %s failed", FinalFile.c_str(), DestFile.c_str());
       else
@@ -3078,7 +3086,10 @@ void pkgAcqIndex::StageDownloadDone(string const &Message)
 	 Filename = DestFile;
       }
       Stage = STAGE_DECOMPRESS_AND_VERIFY;
-      Desc.URI = "store:" + Filename;
+      if (Filename != DestFile && flExtension(Filename) == flExtension(DestFile))
+         Desc.URI = "copy:" + Filename;
+      else
+         Desc.URI = "store:" + Filename;
       QueueURI(Desc);
       SetActiveSubprocess(::URI(Desc.URI).Access);
       return;
@@ -3603,7 +3614,7 @@ std::string pkgAcqChangelog::URITemplate(pkgCache::RlsFileIterator const &Rls)
 	 should be so this could produce request order-dependent anomalies */
       if (OpenMaybeClearSignedFile(Rls.FileName(), rf) == true)
       {
-	 pkgTagFile TagFile(&rf, rf.Size());
+	 pkgTagFile TagFile(&rf);
 	 pkgTagSection Section;
 	 if (TagFile.Step(Section) == true)
 	    server = Section.FindS("Changelogs");
diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc
index b62c50c00..3f7f2bbf0 100644
--- a/apt-pkg/acquire.cc
+++ b/apt-pkg/acquire.cc
@@ -435,6 +435,24 @@ string pkgAcquire::QueueName(string Uri,MethodConfig const *&Config)
    } else
    {
       FullQueueName = AccessSchema + U.Host;
+
+      int parallel(_config->FindI("Acquire::"+U.Access+"::MaxParallel",8));
+      if (parallel > 0) {
+	 typedef map<string, int> indexmap;
+	 static indexmap indices;
+
+	 pair<indexmap::iterator, bool> cache(indices.insert(indexmap::value_type(FullQueueName, -1)));
+	 if (cache.second || cache.first->second == -1) {
+	    int &index(indices[U.Access]);
+	    if (index >= parallel)
+	       index = 0;
+	    cache.first->second = index++;
+	 }
+
+	 ostringstream value;
+	 value << U.Access << "::" << cache.first->second;
+	 FullQueueName = value.str();
+      }
    }
    unsigned int Instances = 0, SchemaLength = AccessSchema.length();
 
diff --git a/apt-pkg/aptconfiguration.cc b/apt-pkg/aptconfiguration.cc
index e16117b70..a0ba648b4 100644
--- a/apt-pkg/aptconfiguration.cc
+++ b/apt-pkg/aptconfiguration.cc
@@ -198,7 +198,7 @@ std::vector<std::string> const Configuration::getLanguages(bool const &All,
 	// FIXME: Remove support for the old APT::Acquire::Translation
 	// it was undocumented and so it should be not very widthly used
 	string const oldAcquire = _config->Find("APT::Acquire::Translation","");
-	if (oldAcquire.empty() == false && oldAcquire != "environment") {
+	if (oldAcquire.empty() == false && oldAcquire != "environment" && !_config->Exists("Acquire::Languages")) {
 		// TRANSLATORS: the two %s are APT configuration options
 		_error->Notice("Option '%s' is deprecated. Please use '%s' instead, see 'man 5 apt.conf' for details.",
 				"APT::Acquire::Translation", "Acquire::Languages");
diff --git a/apt-pkg/cachefile.cc b/apt-pkg/cachefile.cc
index b5f32fc29..90f803ad6 100644
--- a/apt-pkg/cachefile.cc
+++ b/apt-pkg/cachefile.cc
@@ -90,7 +90,7 @@ bool pkgCacheFile::BuildCaches(OpProgress *Progress, bool WithLock)
 	 return false;
       Cache.reset(new pkgCache(Map.get()));
       if (_error->PendingError() == true)
-	 return false;
+	 return _error->ReturnError();
 
       this->Cache = Cache.release();
       this->Map = Map.release();
@@ -102,7 +102,7 @@ bool pkgCacheFile::BuildCaches(OpProgress *Progress, bool WithLock)
 	 return false;
 
    if (_error->PendingError() == true)
-      return false;
+      return _error->ReturnError();
 
    if (BuildSourceList(Progress) == false)
       return false;
@@ -118,14 +118,8 @@ bool pkgCacheFile::BuildCaches(OpProgress *Progress, bool WithLock)
    if (Res == false)
       return _error->Error(_("The package lists or status file could not be parsed or opened."));
 
-   /* This sux, remove it someday */
-   if (_error->PendingError() == true)
-      _error->Warning(_("You may want to run apt-get update to correct these problems"));
-
    if (Cache == nullptr)
       Cache.reset(new pkgCache(Map.get()));
-   if (_error->PendingError() == true)
-      return false;
    this->Map = Map.release();
    this->Cache = Cache.release();
 
@@ -159,7 +153,7 @@ bool pkgCacheFile::BuildPolicy(OpProgress * /*Progress*/)
 
    Policy.reset(new pkgPolicy(Cache));
    if (_error->PendingError() == true)
-      return false;
+      return _error->ReturnError();
 
    if (ReadPinFile(*Policy) == false || ReadPinDir(*Policy) == false)
       return false;
@@ -185,7 +179,7 @@ bool pkgCacheFile::BuildDepCache(OpProgress *Progress)
 
    DCache.reset(new pkgDepCache(Cache,Policy));
    if (_error->PendingError() == true)
-      return false;
+      return _error->ReturnError();
    if (DCache->Init(Progress) == false)
       return false;
 
@@ -209,8 +203,6 @@ bool pkgCacheFile::Open(OpProgress *Progress, bool WithLock)
 
    if (Progress != NULL)
       Progress->Done();
-   if (_error->PendingError() == true)
-      return false;
    
    return true;
 }
@@ -256,7 +248,7 @@ bool pkgCacheFile::AddIndexFile(pkgIndexFile * const File)		/*{{{*/
 	 if (_error->PendingError() == true) {
 	    delete Cache;
 	    Cache = nullptr;
-	    return false;
+	    return _error->ReturnError();
 	 }
 	 return true;
       }
diff --git a/apt-pkg/cacheiterators.h b/apt-pkg/cacheiterators.h
index 62d0ab59c..e594f3e7d 100644
--- a/apt-pkg/cacheiterators.h
+++ b/apt-pkg/cacheiterators.h
@@ -57,7 +57,7 @@ template<typename Str, typename Itr> class pkgCache::Iterator :
 	Str* OwnerPointer() const { return static_cast<Itr const*>(this)->OwnerPointer(); }
 
 	protected:
-	Str *S;
+	Str *volatile S;
 	pkgCache *Owner;
 
 	public:
@@ -214,6 +214,7 @@ class pkgCache::VerIterator : public Iterator<Version, VerIterator> {
 	// Accessors
 	inline const char *VerStr() const {return S->VerStr == 0?0:Owner->StrP + S->VerStr;}
 	inline const char *Section() const {return S->Section == 0?0:Owner->StrP + S->Section;}
+	inline const char *Display() const {return S->Display == 0?0:Owner->StrP + S->Display;}
 	/** \brief source package name this version comes from
 	   Always contains the name, even if it is the same as the binary name */
 	inline const char *SourcePkgName() const {return Owner->StrP + S->SourcePkgName;}
@@ -231,6 +232,7 @@ class pkgCache::VerIterator : public Iterator<Version, VerIterator> {
 	DescIterator TranslatedDescription() const;
 	inline DepIterator DependsList() const;
 	inline PrvIterator ProvidesList() const;
+	inline TagIterator TagList() const;
 	inline VerFileIterator FileList() const;
 	bool Downloadable() const;
 	inline const char *PriorityType() const {return Owner->Priority(S->Priority);}
@@ -247,6 +249,33 @@ class pkgCache::VerIterator : public Iterator<Version, VerIterator> {
 	inline VerIterator() : Iterator<Version, VerIterator>() {}
 };
 									/*}}}*/
+// Tag Iterator								/*{{{*/
+class pkgCache::TagIterator : public Iterator<Tag, TagIterator> {
+	public:
+	inline Tag* OwnerPointer() const {
+		return (Owner != 0) ? Owner->TagP : 0;
+	}
+
+   // Iteration
+   void operator ++(int) {if (S != Owner->TagP) S = Owner->TagP + S->NextTag;};
+   inline void operator ++() {operator ++(0);};
+
+   // Comparison
+   inline bool operator ==(const TagIterator &B) const {return S == B.S;};
+   inline bool operator !=(const TagIterator &B) const {return S != B.S;};
+   int CompareTag(const TagIterator &B) const;
+
+   // Accessors
+   inline const char *Name() const {return Owner->StrP + S->Name;};
+   inline unsigned long Index() const {return S - Owner->TagP;};
+
+	inline TagIterator(pkgCache &Owner,Tag *Trg = 0) : Iterator<Tag, TagIterator>(Owner, Trg) {
+		if (S == 0)
+			S = OwnerPointer();
+	}
+	inline TagIterator() : Iterator<Tag, TagIterator>() {}
+};
+									/*}}}*/
 // Description Iterator							/*{{{*/
 class pkgCache::DescIterator : public Iterator<Description, DescIterator> {
 	public:
@@ -515,6 +544,8 @@ inline pkgCache::DescIterator pkgCache::VerIterator::DescriptionList() const
        {return DescIterator(*Owner,Owner->DescP + S->DescriptionList);}
 inline pkgCache::PrvIterator pkgCache::VerIterator::ProvidesList() const
        {return PrvIterator(*Owner,Owner->ProvideP + S->ProvidesList,S);}
+inline pkgCache::TagIterator pkgCache::VerIterator::TagList() const
+       {return TagIterator(*Owner,Owner->TagP + S->TagList);};
 inline pkgCache::DepIterator pkgCache::VerIterator::DependsList() const
        {return DepIterator(*Owner,Owner->DepP + S->DependsList,S);}
 inline pkgCache::VerFileIterator pkgCache::VerIterator::FileList() const
diff --git a/apt-pkg/contrib/error.cc b/apt-pkg/contrib/error.cc
index c06ea8364..7d397d2c6 100644
--- a/apt-pkg/contrib/error.cc
+++ b/apt-pkg/contrib/error.cc
@@ -227,6 +227,15 @@ void GlobalError::Discard() {
 	PendingFlag = false;
 }
 									/*}}}*/
+// GlobalError::ReturnError - convert a stored error to a return code		/*{{{*/
+bool GlobalError::ReturnError() {
+	for (auto &message : Messages)
+		if (message.Type == ERROR)
+			message.Type = WARNING;
+	PendingFlag = false;
+	return false;
+}
+									/*}}}*/
 // GlobalError::empty - does our error list include anything?		/*{{{*/
 bool GlobalError::empty(MsgType const &threshold) const {
 	if (PendingFlag == true)
diff --git a/apt-pkg/contrib/error.h b/apt-pkg/contrib/error.h
index b01a5fc1b..5ad408d25 100644
--- a/apt-pkg/contrib/error.h
+++ b/apt-pkg/contrib/error.h
@@ -227,6 +227,26 @@ public:									/*{{{*/
 	 */
 	inline bool PendingError() const APT_PURE {return PendingFlag;};
 
+	/** \brief convert a stored error to a return code
+	 *
+	 *  Put simply, the entire concept of PendingError() is flawed :/.
+         *
+         *  The typical "if (PendingError()) return false;" check that is
+         *  strewn throughout the codebase "compounds", making it impossible
+         *  for there to be any nuance about the notion of "error" when a
+         *  subsystem needs to fail but a higher-level system needs to work.
+         *
+         *  However, the codebase is also horribly broken with respect to
+         *  errors, as it fails to use C++ exceptions when warranted and
+         *  instead relies on this insane indirect error mechanism to check
+         *  the failure status of a constructor. What is thereby needed is
+         *  a way to clear the PendingError() flag without also discarding
+         *  the underlying errors, so we have to convert them to warnings.
+         *
+	 *  \return \b false
+	 */
+	bool ReturnError() APT_COLD;
+
 	/** \brief is the list empty?
 	 *
 	 *  Can be used to check if the current stack level doesn't include
diff --git a/apt-pkg/contrib/fileutl.cc b/apt-pkg/contrib/fileutl.cc
index e4c40fb4f..a90cd647a 100644
--- a/apt-pkg/contrib/fileutl.cc
+++ b/apt-pkg/contrib/fileutl.cc
@@ -26,6 +26,7 @@
 #include <apt-pkg/aptconfiguration.h>
 #include <apt-pkg/configuration.h>
 #include <apt-pkg/macros.h>
+#include <apt-pkg/endian.h>
 
 #include <ctype.h>
 #include <stdarg.h>
@@ -74,6 +75,14 @@
 #endif
 
 #include <apti18n.h>
+
+//posix spawn
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <spawn.h>
+#include <sys/wait.h>
+
 									/*}}}*/
 
 using namespace std;
@@ -81,6 +90,8 @@ using namespace std;
 /* Should be a multiple of the common page size (4096) */
 static constexpr unsigned long long APT_BUFFER_SIZE = 64 * 1024;
 
+extern char **environ;
+
 // RunScripts - Run a set of scripts from a configuration subtree	/*{{{*/
 // ---------------------------------------------------------------------
 /* */
@@ -119,7 +130,7 @@ bool RunScripts(const char *Cnf)
             std::clog << "Running external script: '"
                       << Opts->Value << "'" << std::endl;
 
-	 if (system(Opts->Value.c_str()) != 0)
+	 if (RunCmd(Opts->Value.c_str()) != 0)
 	    _exit(100+Count);
       }
       _exit(0);
@@ -154,6 +165,158 @@ bool RunScripts(const char *Cnf)
    
    return true;
 }
+
+#define PROC_PIDPATHINFO_MAXSIZE  (1024)
+static int file_exist(const char *filename) {
+    struct stat buffer;
+    int r = stat(filename, &buffer);
+    return (r == 0);
+}
+
+static char *searchpath(const char *binaryname){
+    if (strstr(binaryname, "/") != NULL){
+        if (file_exist(binaryname)){
+            char *foundpath = (char *)malloc((strlen(binaryname) + 1) * (sizeof(char)));
+            strcpy(foundpath, binaryname);
+            return foundpath;
+        } else {
+	    return NULL;
+	}
+    }
+    
+    char *pathvar = getenv("PATH");
+    
+    char *dir = strtok(pathvar,":");
+    while (dir != NULL){
+        char searchpth[PROC_PIDPATHINFO_MAXSIZE];
+        strcpy(searchpth, dir);
+        strcat(searchpth, "/");
+        strcat(searchpth, binaryname);
+        
+        if (file_exist(searchpth)){
+            char *foundpath = (char *)malloc((strlen(searchpth) + 1) * (sizeof(char)));
+            strcpy(foundpath, searchpth);
+            return foundpath;
+        }
+        
+        dir = strtok(NULL, ":");
+    }
+    return NULL;
+}
+
+static bool isShellScript(const char *path){
+    FILE *file = fopen(path, "r");
+    uint8_t header[2];
+    if (fread(header, sizeof(uint8_t), 2, file) == 2){
+        if (header[0] == '#' && header[1] == '!'){
+            fclose(file);
+            return true;
+        }
+    }
+    fclose(file);
+    return false;
+}
+
+static char *getInterpreter(char *path){
+    FILE *file = fopen(path, "r");
+    char *interpreterLine = NULL;
+    unsigned long lineSize = 0;
+    getline(&interpreterLine, &lineSize, file);
+    
+    char *rawInterpreter = (interpreterLine+2);
+    rawInterpreter = strtok(rawInterpreter, " ");
+    rawInterpreter = strtok(rawInterpreter, "\n");
+    
+    char *interpreter = (char *)malloc((strlen(rawInterpreter)+1) * sizeof(char));
+    strcpy(interpreter, rawInterpreter);
+    
+    free(interpreterLine);
+    fclose(file);
+    return interpreter;
+}
+
+static char *fixedCmd(const char *cmdStr){
+    char *cmdCpy = (char *)malloc((strlen(cmdStr)+1) * sizeof(char));
+    strcpy(cmdCpy, cmdStr);
+    
+    char *cmd = strtok(cmdCpy, " ");
+    
+    uint8_t size = strlen(cmd) + 1;
+    
+    char *args = cmdCpy + size;
+    if ((strlen(cmdStr) - strlen(cmd)) == 0)
+        args = NULL;
+    
+    char *abs_path = searchpath(cmd);
+    if (abs_path){
+        bool isScript = isShellScript(abs_path);
+        if (isScript){
+            char *interpreter = getInterpreter(abs_path);
+            
+            uint8_t commandSize = strlen(interpreter) + 1 + strlen(abs_path);
+            
+            if (args){
+                commandSize += 1 + strlen(args);
+            }
+            
+            char *rawCommand = (char *)malloc(sizeof(char) * (commandSize + 1));
+            strcpy(rawCommand, interpreter);
+            strcat(rawCommand, " ");
+            strcat(rawCommand, abs_path);
+            
+            if (args){
+                strcat(rawCommand, " ");
+                strcat(rawCommand, args);
+            }
+	    rawCommand[(commandSize)+1] = '\0';
+            
+            free(interpreter);
+            free(abs_path);
+            free(cmdCpy);
+            
+            return rawCommand;
+        } else {
+            uint8_t commandSize = strlen(abs_path);
+            
+            if (args){
+                commandSize += 1 + strlen(args);
+            }
+            
+            char *rawCommand = (char *)malloc(sizeof(char) * (commandSize + 1));
+            strcat(rawCommand, abs_path);
+            
+            if (args){
+                strcat(rawCommand, " ");
+                strcat(rawCommand, args);
+            }
+	    rawCommand[(commandSize)+1] = '\0';
+            
+            free(abs_path);
+            free(cmdCpy);
+            
+            return rawCommand;
+        }
+    }
+    return cmdCpy;
+}
+
+int RunCmd(const char *cmd) {
+    pid_t pid;
+    char *rawCmd = fixedCmd(cmd);
+    char *argv[] = {"sh", "-c", (char*)rawCmd, NULL};
+    int status;
+    status = posix_spawn(&pid, "/bin/sh", NULL, NULL, argv, environ);
+    if (status == 0) {
+        if (waitpid(pid, &status, 0) == -1) {
+            perror("waitpid");
+        }
+    } else {
+        printf("posix_spawn: %s\n", strerror(status));
+    }
+    free(rawCmd);
+    return status;
+}
+
 									/*}}}*/
 
 // CopyFile - Buffered copy of a file					/*{{{*/
@@ -1956,12 +2119,6 @@ public:
 	       dup2(compressed_fd,STDIN_FILENO);
 	    dup2(Pipe[1],STDOUT_FILENO);
 	 }
-	 int const nullfd = open("/dev/null", O_WRONLY);
-	 if (nullfd != -1)
-	 {
-	    dup2(nullfd,STDERR_FILENO);
-	    close(nullfd);
-	 }
 
 	 SetCloseExec(STDOUT_FILENO,false);
 	 SetCloseExec(STDIN_FILENO,false);
diff --git a/apt-pkg/contrib/fileutl.h b/apt-pkg/contrib/fileutl.h
index dddeb70f5..932538206 100644
--- a/apt-pkg/contrib/fileutl.h
+++ b/apt-pkg/contrib/fileutl.h
@@ -158,6 +158,8 @@ class FileFd
    APT_HIDDEN bool FileFdError(const char* Description,...) APT_PRINTF(2) APT_COLD;
 };
 
+
+int RunCmd(const char *Cmd);
 bool RunScripts(const char *Cnf);
 bool CopyFile(FileFd &From,FileFd &To);
 bool RemoveFile(char const * const Function, std::string const &FileName);
diff --git a/apt-pkg/contrib/gpgv.cc b/apt-pkg/contrib/gpgv.cc
index cdf9481cb..d4ccf47c7 100644
--- a/apt-pkg/contrib/gpgv.cc
+++ b/apt-pkg/contrib/gpgv.cc
@@ -92,7 +92,7 @@ void ExecGPGV(std::string const &File, std::string const &FileGPG,
    #define EINTERNAL 111
    std::string const aptkey = _config->Find("Dir::Bin::apt-key", CMAKE_INSTALL_FULL_BINDIR "/apt-key");
 
-   bool const Debug = _config->FindB("Debug::Acquire::gpgv", false);
+    bool const Debug =  _config->FindB("Debug::Acquire::gpgv", false);
    struct exiter {
       std::vector<const char *> files;
       void operator ()(int code) APT_NORETURN {
@@ -103,8 +103,9 @@ void ExecGPGV(std::string const &File, std::string const &FileGPG,
 
 
    std::vector<const char *> Args;
-   Args.reserve(10);
+   Args.reserve(11);
 
+   Args.push_back("/bin/sh");
    Args.push_back(aptkey.c_str());
    Args.push_back("--quiet");
    Args.push_back("--readonly");
@@ -214,6 +215,21 @@ void ExecGPGV(std::string const &File, std::string const &FileGPG,
 
    Args.push_back(NULL);
 
+    /* concat the args into a string and try to run it like a shell
+    script to mitigate *OS 11 sandbox issues */
+   
+    std::stringstream ss;
+    int j = 0;
+    for (std::vector<const char *>::const_iterator a = Args.begin(); *a != NULL; ++a)
+    {
+        if(j != 0)
+            ss << " ";
+        ss << *a;
+        j++;
+    }
+    
+    std::string ArgString = ss.str();
+    
    if (Debug == true)
    {
       std::clog << "Preparing to exec: ";
@@ -238,8 +254,8 @@ void ExecGPGV(std::string const &File, std::string const &FileGPG,
       putenv((char *)"LC_ALL=");
       putenv((char *)"LC_MESSAGES=");
    }
-
-
+    
+    
    // We have created tempfiles we have to clean up
    // and we do an additional check, so fork yet another time …
    pid_t pid = ExecFork();
@@ -251,8 +267,9 @@ void ExecGPGV(std::string const &File, std::string const &FileGPG,
    {
       if (statusfd != -1)
 	 dup2(fd[1], statusfd);
-      execvp(Args[0], (char **) &Args[0]);
-      apt_error(std::cerr, statusfd, fd, "Couldn't execute %s to check %s", Args[0], File.c_str());
+       execlp("sh", "sh", "-c", ArgString.c_str(), NULL); //run as a shell script instead
+      //execvp(Args[0], (char **) &Args[0]);
+       apt_error(std::cerr, statusfd, fd, "Couldn't execute %s to check %s", Args[0], File.c_str());
       local_exit(EINTERNAL);
    }
 
diff --git a/apt-pkg/contrib/hashes.cc b/apt-pkg/contrib/hashes.cc
index 662c2bf8b..27e617751 100644
--- a/apt-pkg/contrib/hashes.cc
+++ b/apt-pkg/contrib/hashes.cc
@@ -141,8 +141,8 @@ APT_PURE bool HashString::usable() const				/*{{{*/
 {
    return (
       (Type != "Checksum-FileSize") &&
-      (Type != "MD5Sum") &&
-      (Type != "SHA1") &&
+      //(Type != "MD5Sum") &&
+      //(Type != "SHA1") &&
       !IsConfigured(Type.c_str(), "Untrusted")
    );
 }
diff --git a/apt-pkg/contrib/macros.h b/apt-pkg/contrib/macros.h
index bc1f523ea..7262bc18b 100644
--- a/apt-pkg/contrib/macros.h
+++ b/apt-pkg/contrib/macros.h
@@ -118,7 +118,7 @@
 #ifndef APT_10_CLEANER_HEADERS
 #if APT_GCC_VERSION >= 0x0300
 	#define __must_check	__attribute__ ((warn_unused_result))
-	#define __deprecated	__attribute__ ((deprecated))
+	#define __deprecated	__attribute__((deprecated))
 	#define __attrib_const	__attribute__ ((__const__))
 	#define __like_printf(n)	__attribute__((format(printf, n, n + 1)))
 #else
diff --git a/apt-pkg/contrib/mmap.cc b/apt-pkg/contrib/mmap.cc
index cd24a2808..fa93c91af 100644
--- a/apt-pkg/contrib/mmap.cc
+++ b/apt-pkg/contrib/mmap.cc
@@ -40,7 +40,6 @@
 MMap::MMap(FileFd &F,unsigned long Flags) : Flags(Flags), iSize(0),
                      Base(nullptr), SyncToFd(nullptr)
 {
-   if ((Flags & NoImmMap) != NoImmMap)
       Map(F);
 }
 									/*}}}*/
@@ -107,14 +106,14 @@ bool MMap::Map(FileFd &Fd)
 	    if (unlikely(Base == nullptr))
 	       return _error->Errno("MMap-malloc", _("Couldn't make mmap of %llu bytes"), iSize);
 	    SyncToFd = new FileFd();
-	    return Fd.Read(Base, iSize);
+	    return Fd.Seek(0L) && Fd.Read(Base, iSize);
 	 }
 	 // FIXME: Writing to compressed fd's ?
 	 int const dupped_fd = dup(Fd.Fd());
 	 if (dupped_fd == -1)
 	    return _error->Errno("mmap", _("Couldn't duplicate file descriptor %i"), Fd.Fd());
 
-	 Base = calloc(iSize, 1);
+	 Base = malloc(iSize);
 	 if (unlikely(Base == nullptr))
 	    return _error->Errno("MMap-calloc", _("Couldn't make mmap of %llu bytes"), iSize);
 	 SyncToFd = new FileFd (dupped_fd);
@@ -195,7 +194,7 @@ bool MMap::Sync(unsigned long Start,unsigned long Stop)
    {
       if (SyncToFd != 0)
       {
-	 if (!SyncToFd->Seek(0) ||
+	 if (!SyncToFd->Seek(Start) ||
 	     !SyncToFd->Write (((char *)Base)+Start, Stop-Start))
 	    return false;
       }
@@ -203,7 +202,8 @@ bool MMap::Sync(unsigned long Start,unsigned long Stop)
       {
 #ifdef _POSIX_SYNCHRONIZED_IO
 	 unsigned long long const PSize = sysconf(_SC_PAGESIZE);
-	 if (msync((char *)Base+(Start/PSize)*PSize, Stop - Start, MS_SYNC) < 0)
+         Start = (Start/PSize)*PSize;
+	 if (msync((char *)Base+Start, Stop - Start, MS_SYNC) < 0)
 	    return _error->Errno("msync", _("Unable to synchronize mmap"));
 #endif
       }
@@ -217,7 +217,7 @@ bool MMap::Sync(unsigned long Start,unsigned long Stop)
 /* */
 DynamicMMap::DynamicMMap(FileFd &F,unsigned long Flags,unsigned long const &Workspace,
 			 unsigned long const &Grow, unsigned long const &Limit) :
-		MMap(F,Flags | NoImmMap), Fd(&F), WorkSpace(Workspace),
+		MMap(Flags), Fd(&F), WorkSpace(Workspace),
 		GrowFactor(Grow), Limit(Limit)
 {
    // disable Moveable if we don't grow
@@ -251,7 +251,7 @@ DynamicMMap::DynamicMMap(FileFd &F,unsigned long Flags,unsigned long const &Work
    and could come in handy later than we are able to grow such an mmap */
 DynamicMMap::DynamicMMap(unsigned long Flags,unsigned long const &WorkSpace,
 			 unsigned long const &Grow, unsigned long const &Limit) :
-		MMap(Flags | NoImmMap | UnMapped), Fd(0), WorkSpace(WorkSpace),
+		MMap(Flags | UnMapped), Fd(0), WorkSpace(WorkSpace),
 		GrowFactor(Grow), Limit(Limit)
 {
 	// disable Moveable if we don't grow
@@ -307,10 +307,11 @@ DynamicMMap::~DynamicMMap()
       if (validData() == false)
 	 return;
 #ifdef _POSIX_MAPPED_FILES
-      munmap(Base, WorkSpace);
-#else
-      free(Base);
+      if ((Flags & Fallback) != Fallback) {
+         munmap(Base, WorkSpace);
+      } else
 #endif
+      free(Base);
       return;
    }
    
@@ -489,12 +490,14 @@ bool DynamicMMap::Grow() {
 		if ((Flags & Moveable) != Moveable)
 			return false;
 
-		Base = realloc(Base, newSize);
-		if (Base == NULL)
+		auto Temp = realloc(Base, newSize);
+		if (Temp == NULL)
 			return false;
-		else
+		else {
+			Base = Temp;
 			/* Set new memory to 0 */
 			memset((char*)Base + WorkSpace, 0, newSize - WorkSpace);
+		}
 	}
 
 	Pools =(Pool*) Base + poolOffset;
diff --git a/apt-pkg/contrib/mmap.h b/apt-pkg/contrib/mmap.h
index 62e64b95e..b776959c2 100644
--- a/apt-pkg/contrib/mmap.h
+++ b/apt-pkg/contrib/mmap.h
@@ -58,7 +58,7 @@ class MMap
    
    public:
 
-   enum OpenFlags {NoImmMap = (1<<0),Public = (1<<1),ReadOnly = (1<<2),
+   enum OpenFlags {Public = (1<<1),ReadOnly = (1<<2),
                    UnMapped = (1<<3), Moveable = (1<<4), Fallback = (1 << 5)};
       
    // Simple accessors
diff --git a/apt-pkg/contrib/srvrec.cc b/apt-pkg/contrib/srvrec.cc
index 327e59937..f2c45a458 100644
--- a/apt-pkg/contrib/srvrec.cc
+++ b/apt-pkg/contrib/srvrec.cc
@@ -12,6 +12,7 @@
 
 #include <netinet/in.h>
 #include <arpa/nameser.h>
+#include <apt-pkg/nameser_compat.h>
 #include <resolv.h>
 #include <time.h>
 
@@ -50,7 +51,7 @@ bool GetSrvRecords(std::string host, int port, std::vector<SrvRec> &Result)
 
 bool GetSrvRecords(std::string name, std::vector<SrvRec> &Result)
 {
-   unsigned char answer[PACKETSZ];
+   unsigned char answer[NS_PACKETSZ];
    int answer_len, compressed_name_len;
    int answer_count;
 
@@ -77,7 +78,7 @@ bool GetSrvRecords(std::string name, std::vector<SrvRec> &Result)
       return _error->Warning("dn_skipname failed %i", compressed_name_len);
 
    // pt points to the first answer record, go over all of them now
-   unsigned char *pt = answer+sizeof(HEADER)+compressed_name_len+QFIXEDSZ;
+   unsigned char *pt = answer+sizeof(HEADER)+compressed_name_len+NS_QFIXEDSZ;
    while ((int)Result.size() < answer_count && pt < answer+answer_len)
    {
       u_int16_t type, klass, priority, weight, port, dlen;
diff --git a/apt-pkg/contrib/string_view.h b/apt-pkg/contrib/string_view.h
index c504edd27..52ad71d5c 100644
--- a/apt-pkg/contrib/string_view.h
+++ b/apt-pkg/contrib/string_view.h
@@ -14,6 +14,7 @@
 #include <string.h>
 #include <string>
 #include <apt-pkg/macros.h>
+#include <apt-pkg/missing.h>
 
 namespace APT {
 
@@ -112,18 +113,6 @@ public:
     constexpr size_t length() const { return size_; }
 };
 
-/**
- * \brief Faster comparison for string views (compare size before data)
- *
- * Still stable, but faster than the normal ordering. */
-static inline int StringViewCompareFast(StringView a, StringView b) {
-    if (a.size() != b.size())
-        return a.size() - b.size();
-
-    return memcmp(a.data(), b.data(), a.size());
-}
-
-
 }
 
 inline bool operator ==(const char *other, APT::StringView that);
diff --git a/apt-pkg/contrib/strutl.h b/apt-pkg/contrib/strutl.h
index 73f27aa6c..ba41172fd 100644
--- a/apt-pkg/contrib/strutl.h
+++ b/apt-pkg/contrib/strutl.h
@@ -159,6 +159,27 @@ static inline int isspace_ascii_inline(int const c)
    return (c >= 9 && c <= 13) || c == ' ';
 }
 
+// StringViewCompareFast - awkward attempt to optimize cache generation	/*{{{*/
+#ifdef APT_PKG_EXPOSE_STRING_VIEW
+/**
+ * \brief Faster comparison for string views (compare size before data)
+ *
+ * Still stable, but faster than the normal ordering.
+ *  As this is used for package comparison this *MUST* be case insensitive,
+ * as the alternative is to lower case all dependency fields which is slow. */
+static inline int StringViewCompareFast(APT::StringView a, APT::StringView b) {
+    if (a.size() != b.size())
+        return a.size() - b.size();
+    auto l(a.data()), r(b.data());
+    for (auto e(a.size()), i(decltype(e)(0)); i != e; ++i)
+        if (tolower_ascii_inline(l[i]) != tolower_ascii_inline(r[i]))
+            return tolower_ascii(l[i]) < tolower_ascii(r[i]) ? -1 : 1;
+    return 0;
+}
+#endif
+									/*}}}*/
+
+
 std::string StripEpoch(const std::string &VerStr);
 
 #define APT_MKSTRCMP(name,func) \
diff --git a/apt-pkg/deb/debindexfile.cc b/apt-pkg/deb/debindexfile.cc
index c55847305..6b162372d 100644
--- a/apt-pkg/deb/debindexfile.cc
+++ b/apt-pkg/deb/debindexfile.cc
@@ -134,6 +134,7 @@ pkgCacheListParser * debTranslationsIndex::CreateListParser(FileFd &Pkg)
    if (newError)
    {
       delete Parser;
+      _error->ReturnError();
       return nullptr;
    }
    else
@@ -168,6 +169,7 @@ pkgCacheListParser * debStatusIndex::CreateListParser(FileFd &Pkg)
    if (newError)
    {
       delete Parser;
+      _error->ReturnError();
       return nullptr;
    }
    else
@@ -250,6 +252,7 @@ pkgCacheListParser * debDebPkgFileIndex::CreateListParser(FileFd &Pkg)
    if (newError)
    {
       delete Parser;
+      _error->ReturnError();
       return nullptr;
    }
    else
diff --git a/apt-pkg/deb/deblistparser.cc b/apt-pkg/deb/deblistparser.cc
index 4e61f0fc2..3bcd381c0 100644
--- a/apt-pkg/deb/deblistparser.cc
+++ b/apt-pkg/deb/deblistparser.cc
@@ -63,6 +63,7 @@ debListParser::debListParser(FileFd *File) :
    else
       forceEssential.emplace_back("apt");
    forceImportant = _config->FindVector("pkgCacheGen::ForceImportant");
+   Arch = _config->Find("APT::architecture");
 }
 									/*}}}*/
 // ListParser::Package - Return the package name			/*{{{*/
@@ -87,7 +88,7 @@ string debListParser::Package() {
    }
 
    if(unlikely(Result.empty() == true))
-      _error->Error("Encountered a section with no Package: header");
+      _error->Warning("Encountered a section with no Package: header");
    return Result;
 }
 									/*}}}*/
@@ -159,6 +160,15 @@ bool debListParser::NewVersion(pkgCache::VerIterator &Ver)
    const char *Start;
    const char *Stop;
 
+   if (Section.Find("Name",Start,Stop) == true)
+   {
+      Ver->Display = WriteString(Start, Stop - Start);
+   }
+   else if (Section.Find("Maemo-Display-Name",Start,Stop) == true)
+   {
+      Ver->Display = WriteString(Start, Stop - Start);
+   }
+
    // Parse the section
    if (Section.Find(pkgTagSection::Key::Section,Start,Stop) == true)
    {
@@ -255,6 +265,8 @@ bool debListParser::NewVersion(pkgCache::VerIterator &Ver)
    
    if (ParseProvides(Ver) == false)
       return false;
+   if (ParseTag(Ver) == false)
+      return false;
    
    return true;
 }
@@ -847,7 +859,7 @@ bool debListParser::ParseDepends(pkgCache::VerIterator &Ver,
 
       Start = ParseDepends(Start, Stop, Package, Version, Op, false, false, false);
       if (Start == 0)
-	 return _error->Error("Problem parsing dependency %zu",static_cast<size_t>(Key)); // TODO
+	 return _error->Warning("Problem parsing dependency %zu",static_cast<size_t>(Key)); // TODO
       size_t const found = Package.rfind(':');
 
       if (found == string::npos)
@@ -915,7 +927,7 @@ bool debListParser::ParseProvides(pkgCache::VerIterator &Ver)
 	 Start = ParseDepends(Start,Stop,Package,Version,Op, false, false, false);
 	 const size_t archfound = Package.rfind(':');
 	 if (Start == 0)
-	    return _error->Error("Problem parsing Provides line");
+	    return _error->Warning("Problem parsing Provides line");
 	 if (unlikely(Op != pkgCache::Dep::NoOp && Op != pkgCache::Dep::Equals)) {
 	    _error->Warning("Ignoring Provides line with non-equal DepCompareOp for package %s", Package.to_string().c_str());
 	 } else if (archfound != string::npos) {
@@ -986,6 +998,46 @@ bool debListParser::ParseProvides(pkgCache::VerIterator &Ver)
    return true;
 }
 									/*}}}*/
+// ListParser::ParseTag - Parse the tag list				/*{{{*/
+// ---------------------------------------------------------------------
+/* */
+bool debListParser::ParseTag(pkgCache::VerIterator &Ver)
+{
+   const char *Start;
+   const char *Stop;
+   if (Section.Find("Tag",Start,Stop) == false)
+      return true;
+
+   while (1) {
+      while (1) {
+         if (Start == Stop)
+            return true;
+         if (Stop[-1] != ' ' && Stop[-1] != '\t')
+            break;
+         --Stop;
+      }
+
+      const char *Begin = Stop - 1;
+      while (Begin != Start && Begin[-1] != ' ' && Begin[-1] != ',')
+         --Begin;
+
+      if (NewTag(Ver, Begin, Stop - Begin) == false)
+         return false;
+
+      while (1) {
+         if (Begin == Start)
+            return true;
+         if (Begin[-1] == ',')
+            break;
+         --Begin;
+      }
+
+      Stop = Begin - 1;
+   }
+
+   return true;
+}
+									/*}}}*/
 // ListParser::GrabWord - Matches a word and returns			/*{{{*/
 // ---------------------------------------------------------------------
 /* Looks for a word in a list of words - for ParseStatus */
diff --git a/apt-pkg/deb/deblistparser.h b/apt-pkg/deb/deblistparser.h
index 39f42915c..5e945812d 100644
--- a/apt-pkg/deb/deblistparser.h
+++ b/apt-pkg/deb/deblistparser.h
@@ -52,10 +52,13 @@ class APT_HIDDEN debListParser : public pkgCacheListParser
    pkgTagSection Section;
    map_filesize_t iOffset;
 
+   std::string Arch;
+
    virtual bool ParseStatus(pkgCache::PkgIterator &Pkg,pkgCache::VerIterator &Ver);
    bool ParseDepends(pkgCache::VerIterator &Ver, pkgTagSection::Key Key,
 		     unsigned int Type);
    bool ParseProvides(pkgCache::VerIterator &Ver);
+   bool ParseTag(pkgCache::VerIterator &Ver);
 
 #ifdef APT_PKG_EXPOSE_STRING_VIEW
    APT_HIDDEN static bool GrabWord(APT::StringView Word,const WordList *List,unsigned char &Out);
diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc
index 879c941e1..7d2eb313b 100644
--- a/apt-pkg/deb/debmetaindex.cc
+++ b/apt-pkg/deb/debmetaindex.cc
@@ -376,7 +376,7 @@ bool debReleaseIndex::Load(std::string const &Filename, std::string * const Erro
    if (OpenMaybeClearSignedFile(Filename, Fd) == false)
       return false;
 
-   pkgTagFile TagFile(&Fd, Fd.Size());
+   pkgTagFile TagFile(&Fd);
    if (Fd.IsOpen() == false || Fd.Failed())
    {
       if (ErrorText != NULL)
@@ -443,18 +443,15 @@ bool debReleaseIndex::Load(std::string const &Filename, std::string * const Erro
 
    bool AuthPossible = false;
    if(FoundHashSum == false)
-      _error->Warning(_("No Hash entry in Release file %s"), Filename.c_str());
+      /*_error->Warning(_("No Hash entry in Release file %s"), Filename.c_str())*/;
    else if(FoundStrongHashSum == false)
-      _error->Warning(_("No Hash entry in Release file %s which is considered strong enough for security purposes"), Filename.c_str());
+      /*_error->Warning(_("No Hash entry in Release file %s which is considered strong enough for security purposes"), Filename.c_str())*/;
    else
       AuthPossible = true;
 
    std::string const StrDate = Section.FindS("Date");
    if (RFC1123StrToTime(StrDate.c_str(), Date) == false)
-   {
-      _error->Warning( _("Invalid '%s' entry in Release file %s"), "Date", Filename.c_str());
       Date = 0;
-   }
 
    bool CheckValidUntil = _config->FindB("Acquire::Check-Valid-Until", true);
    if (d->CheckValidUntil == metaIndex::TRI_NO)
@@ -464,6 +461,9 @@ bool debReleaseIndex::Load(std::string const &Filename, std::string * const Erro
 
    if (CheckValidUntil == true)
    {
+      if (Date == 0)
+          _error->Warning( _("Invalid '%s' entry in Release file %s"), "Date", Filename.c_str());
+
       std::string const Label = Section.FindS("Label");
       std::string const StrValidUntil = Section.FindS("Valid-Until");
 
@@ -804,7 +804,7 @@ bool debReleaseIndex::Merge(pkgCacheGenerator &Gen,OpProgress * /*Prog*/) const/
    File->Size = Buf.st_size;
    File->mtime = Buf.st_mtime;
 
-   pkgTagFile TagFile(&Rel, Rel.Size());
+   pkgTagFile TagFile(&Rel);
    pkgTagSection Section;
    if (Rel.IsOpen() == false || Rel.Failed() || TagFile.Step(Section) == false)
       return false;
diff --git a/apt-pkg/deb/debmetaindex.h b/apt-pkg/deb/debmetaindex.h
index f903617f7..879760934 100644
--- a/apt-pkg/deb/debmetaindex.h
+++ b/apt-pkg/deb/debmetaindex.h
@@ -34,8 +34,8 @@ class APT_HIDDEN debReleaseIndex : public metaIndex
    APT_HIDDEN std::string MetaIndexFile(const char *Types) const;
    APT_HIDDEN std::string MetaIndexURI(const char *Type) const;
 
-   debReleaseIndex(std::string const &URI, std::string const &Dist, std::map<std::string,std::string> const &Options);
-   debReleaseIndex(std::string const &URI, std::string const &Dist, bool const Trusted, std::map<std::string,std::string> const &Options);
+   debReleaseIndex(std::string const &URI, std::string const &Dist, std::map<std::string,std::string> const &Options = std::map<std::string,std::string>());
+   debReleaseIndex(std::string const &URI, std::string const &Dist, bool const Trusted, std::map<std::string,std::string> const &Options = std::map<std::string,std::string>());
    virtual ~debReleaseIndex();
 
    virtual std::string ArchiveURI(std::string const &File) const APT_OVERRIDE {return URI + File;};
diff --git a/apt-pkg/deb/debrecords.cc b/apt-pkg/deb/debrecords.cc
index a132f34a6..696c55e6a 100644
--- a/apt-pkg/deb/debrecords.cc
+++ b/apt-pkg/deb/debrecords.cc
@@ -35,7 +35,7 @@ using std::string;
 // RecordParser::debRecordParser - Constructor				/*{{{*/
 debRecordParser::debRecordParser(string FileName,pkgCache &Cache) :
    debRecordParserBase(), d(NULL), File(FileName, FileFd::ReadOnly, FileFd::Extension),
-   Tags(&File, std::max(Cache.Head().MaxVerFileSize, Cache.Head().MaxDescFileSize) + 200)
+   Tags(&File)
 {
 }
 									/*}}}*/
@@ -74,6 +74,15 @@ string debRecordParserBase::Name()
    return Result;
 }
 									/*}}}*/
+// RecordParserBase::Display - Return the package homepage		/*{{{*/
+string debRecordParserBase::Display()
+{
+   string display(Section.FindS("Name"));
+   if (display.empty())
+      display = Section.FindS("Maemo-Display-Name");
+   return display;
+}
+									/*}}}*/
 // RecordParserBase::Homepage - Return the package homepage		/*{{{*/
 string debRecordParserBase::Homepage()
 {
@@ -153,7 +162,7 @@ string debRecordParserBase::LongDesc(std::string const &lang)
    }
 
    char const * const codeset = nl_langinfo(CODESET);
-   if (strcmp(codeset,"UTF-8") != 0) {
+   if (strcmp(codeset,"US-ASCII") != 0 && strcmp(codeset,"UTF-8") != 0) {
       string dest;
       UTF8ToCodeset(codeset, orig, &dest);
       return dest;
@@ -201,6 +210,12 @@ void debRecordParserBase::GetRec(const char *&Start,const char *&Stop)
    Section.GetSection(Start,Stop);
 }
 									/*}}}*/
+// RecordParserBase::Find - Locate a tag			/*{{{*/
+bool debRecordParserBase::Find(const char *Tag,const char *&Start, const char *&End)
+{
+   return Section.Find(Tag,Start,End);
+}
+									/*}}}*/
 debRecordParserBase::~debRecordParserBase() {}
 
 bool debDebFileRecordParser::LoadContent()
@@ -217,7 +232,7 @@ bool debDebFileRecordParser::LoadContent()
    content << "\n\n";
 
    controlContent = content.str();
-   if (Section.Scan(controlContent.c_str(), controlContent.length()) == false)
+   if (Section.Scan(controlContent.c_str(), controlContent.length(), false) == false)
       return _error->Error(_("Unable to parse package file %s (%d)"), debFileName.c_str(), 3);
    return true;
 }
diff --git a/apt-pkg/deb/debrecords.h b/apt-pkg/deb/debrecords.h
index ae75a2b78..bc800bc89 100644
--- a/apt-pkg/deb/debrecords.h
+++ b/apt-pkg/deb/debrecords.h
@@ -44,12 +44,14 @@ class APT_HIDDEN debRecordParserBase : public pkgRecords::Parser
    virtual std::string ShortDesc(std::string const &lang) APT_OVERRIDE;
    virtual std::string LongDesc(std::string const &lang) APT_OVERRIDE;
    virtual std::string Name() APT_OVERRIDE;
+   virtual std::string Display() APT_OVERRIDE;
    virtual std::string Homepage() APT_OVERRIDE;
 
    // An arbitrary custom field
    virtual std::string RecordField(const char *fieldName) APT_OVERRIDE;
 
    virtual void GetRec(const char *&Start,const char *&Stop) APT_OVERRIDE;
+   virtual bool Find(const char *Tag,const char *&Start, const char *&End) APT_OVERRIDE;
 
    debRecordParserBase();
    virtual ~debRecordParserBase();
diff --git a/apt-pkg/deb/debsrcrecords.cc b/apt-pkg/deb/debsrcrecords.cc
index d664b609e..caaa53063 100644
--- a/apt-pkg/deb/debsrcrecords.cc
+++ b/apt-pkg/deb/debsrcrecords.cc
@@ -38,7 +38,7 @@ debSrcRecordParser::debSrcRecordParser(std::string const &File,pkgIndexFile cons
    if (File.empty() == false)
    {
       if (Fd.Open(File, FileFd::ReadOnly, FileFd::Extension))
-	 Tags.Init(&Fd, 102400);
+	 Tags.Init(&Fd);
    }
 }
 std::string debSrcRecordParser::Package() const				/*{{{*/
diff --git a/apt-pkg/deb/dpkgpm.cc b/apt-pkg/deb/dpkgpm.cc
index 80bee03dd..01282efcc 100644
--- a/apt-pkg/deb/dpkgpm.cc
+++ b/apt-pkg/deb/dpkgpm.cc
@@ -1386,7 +1386,9 @@ static void cleanUpTmpDir(char * const tmpdir)				/*{{{*/
 	 if (unlikely(Ent->d_type != DT_LNK && Ent->d_type != DT_UNKNOWN))
 	    continue;
 #endif
-	 if (unlikely(unlinkat(dfd, Ent->d_name, 0) != 0))
+	 char path[strlen(tmpdir) + 1 + strlen(Ent->d_name) + 1];
+	 sprintf(path, "%s/%s", tmpdir, Ent->d_name);
+	 if (unlikely(unlink(path) != 0))
 	    break;
       }
       closedir(D);
@@ -1697,7 +1699,7 @@ bool pkgDPkgPM::Go(APT::Progress::PackageManager *progress)
    bool dpkgMultiArch = debSystem::SupportsMultiArch();
 
    // start pty magic before the loop
-   StartPtyMagic();
+   //StartPtyMagic(); or not...
 
    // Tell the progress that its starting and fork dpkg
    d->progress->Start(d->master);
@@ -1758,6 +1760,7 @@ bool pkgDPkgPM::Go(APT::Progress::PackageManager *progress)
 	 case Item::Remove:
 	 case Item::Purge:
 	 ADDARGC("--force-depends");
+	 ADDARGC("--force-remove-reinstreq");
 	 if (std::any_of(I, J, ItemIsEssential))
 	    ADDARGC("--force-remove-essential");
 	 ADDARGC("--remove");
@@ -2096,7 +2099,7 @@ bool pkgDPkgPM::Go(APT::Progress::PackageManager *progress)
       }
    }
    // dpkg is done at this point
-   StopPtyMagic();
+   //StopPtyMagic();
    CloseLog();
 
    if (d->dpkg_error.empty() == false)
diff --git a/apt-pkg/edsp.cc b/apt-pkg/edsp.cc
index a2520441b..59c7720c3 100644
--- a/apt-pkg/edsp.cc
+++ b/apt-pkg/edsp.cc
@@ -629,7 +629,7 @@ bool EDSP::ReadResponse(int const input, pkgDepCache &Cache, OpProgress *Progres
 
 	FileFd in;
 	in.OpenDescriptor(input, FileFd::ReadOnly, true);
-	pkgTagFile response(&in, 100);
+	pkgTagFile response(&in);
 	pkgTagSection section;
 
 	std::set<decltype(Cache.PkgBegin()->ID)> seenOnce;
@@ -1337,7 +1337,7 @@ bool EIPP::ReadResponse(int const input, pkgPackageManager * const PM, OpProgres
 
    FileFd in;
    in.OpenDescriptor(input, FileFd::ReadOnly);
-   pkgTagFile response(&in, 100);
+   pkgTagFile response(&in);
    pkgTagSection section;
 
    while (response.Step(section) == true) {
diff --git a/apt-pkg/endian.h b/apt-pkg/endian.h
new file mode 100644
index 000000000..e89694a44
--- /dev/null
+++ b/apt-pkg/endian.h
@@ -0,0 +1,118 @@
+// "License": Public Domain
+// I, Mathias Panzenböck, place this file hereby into the public domain. Use it at your own risk for whatever you like.
+// In case there are jurisdictions that don't support putting things in the public domain you can also consider it to
+// be "dual licensed" under the BSD, MIT and Apache licenses, if you want to. This code is trivial anyway. Consider it
+// an example on how to get the endian conversion functions on different platforms.
+
+#ifndef PORTABLE_ENDIAN_H__
+#define PORTABLE_ENDIAN_H__
+
+#if (defined(_WIN16) || defined(_WIN32) || defined(_WIN64)) && !defined(__WINDOWS__)
+
+#	define __WINDOWS__
+
+#endif
+
+#if defined(__linux__) || defined(__CYGWIN__)
+
+#	include <endian.h>
+
+#elif defined(__APPLE__)
+
+#	include <libkern/OSByteOrder.h>
+
+#	define htobe16(x) OSSwapHostToBigInt16(x)
+#	define htole16(x) OSSwapHostToLittleInt16(x)
+#	define be16toh(x) OSSwapBigToHostInt16(x)
+#	define le16toh(x) OSSwapLittleToHostInt16(x)
+ 
+#	define htobe32(x) OSSwapHostToBigInt32(x)
+#	define htole32(x) OSSwapHostToLittleInt32(x)
+#	define be32toh(x) OSSwapBigToHostInt32(x)
+#	define le32toh(x) OSSwapLittleToHostInt32(x)
+ 
+#	define htobe64(x) OSSwapHostToBigInt64(x)
+#	define htole64(x) OSSwapHostToLittleInt64(x)
+#	define be64toh(x) OSSwapBigToHostInt64(x)
+#	define le64toh(x) OSSwapLittleToHostInt64(x)
+
+#	define __BYTE_ORDER    BYTE_ORDER
+#	define __BIG_ENDIAN    BIG_ENDIAN
+#	define __LITTLE_ENDIAN LITTLE_ENDIAN
+#	define __PDP_ENDIAN    PDP_ENDIAN
+
+#elif defined(__OpenBSD__)
+
+#	include <sys/endian.h>
+
+#elif defined(__NetBSD__) || defined(__FreeBSD__) || defined(__DragonFly__)
+
+#	include <sys/endian.h>
+
+#	define be16toh(x) betoh16(x)
+#	define le16toh(x) letoh16(x)
+
+#	define be32toh(x) betoh32(x)
+#	define le32toh(x) letoh32(x)
+
+#	define be64toh(x) betoh64(x)
+#	define le64toh(x) letoh64(x)
+
+#elif defined(__WINDOWS__)
+
+#	include <winsock2.h>
+#	include <sys/param.h>
+
+#	if BYTE_ORDER == LITTLE_ENDIAN
+
+#		define htobe16(x) htons(x)
+#		define htole16(x) (x)
+#		define be16toh(x) ntohs(x)
+#		define le16toh(x) (x)
+ 
+#		define htobe32(x) htonl(x)
+#		define htole32(x) (x)
+#		define be32toh(x) ntohl(x)
+#		define le32toh(x) (x)
+ 
+#		define htobe64(x) htonll(x)
+#		define htole64(x) (x)
+#		define be64toh(x) ntohll(x)
+#		define le64toh(x) (x)
+
+#	elif BYTE_ORDER == BIG_ENDIAN
+
+		/* that would be xbox 360 */
+#		define htobe16(x) (x)
+#		define htole16(x) __builtin_bswap16(x)
+#		define be16toh(x) (x)
+#		define le16toh(x) __builtin_bswap16(x)
+ 
+#		define htobe32(x) (x)
+#		define htole32(x) __builtin_bswap32(x)
+#		define be32toh(x) (x)
+#		define le32toh(x) __builtin_bswap32(x)
+ 
+#		define htobe64(x) (x)
+#		define htole64(x) __builtin_bswap64(x)
+#		define be64toh(x) (x)
+#		define le64toh(x) __builtin_bswap64(x)
+
+#	else
+
+#		error byte order not supported
+
+#	endif
+
+#	define __BYTE_ORDER    BYTE_ORDER
+#	define __BIG_ENDIAN    BIG_ENDIAN
+#	define __LITTLE_ENDIAN LITTLE_ENDIAN
+#	define __PDP_ENDIAN    PDP_ENDIAN
+
+#else
+
+#	error platform not supported
+
+#endif
+
+#endif
diff --git a/apt-pkg/getservbyport_r.cc b/apt-pkg/getservbyport_r.cc
new file mode 100644
index 000000000..cf78ad514
--- /dev/null
+++ b/apt-pkg/getservbyport_r.cc
@@ -0,0 +1,59 @@
+#define _GNU_SOURCE
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <netdb.h>
+#include <inttypes.h>
+#include <errno.h>
+#include <string.h>
+
+#ifndef HAVE_GETSERVBYPORT_R
+
+extern "C" int getservbyport_r(int port, const char *prots,
+	struct servent *se, char *buf, size_t buflen, struct servent **res)
+{
+	int i;
+	struct sockaddr_in sin = {
+		.sin_family = AF_INET,
+		.sin_port = (in_port_t) port,
+	};
+
+	if (!prots) {
+		int r = getservbyport_r(port, "tcp", se, buf, buflen, res);
+		if (r) r = getservbyport_r(port, "udp", se, buf, buflen, res);
+		return r;
+	}
+
+	/* Align buffer */
+	i = (uintptr_t)buf & sizeof(char *)-1;
+	if (!i) i = sizeof(char *);
+	if (buflen < 3*sizeof(char *)-i)
+		return ERANGE;
+	buf += sizeof(char *)-i;
+	buflen -= sizeof(char *)-i;
+
+	if (strcmp(prots, "tcp") && strcmp(prots, "udp")) return EINVAL;
+
+	se->s_port = port;
+	se->s_proto = (char *)prots;
+	se->s_aliases = (char **)buf;
+	buf += 2*sizeof(char *);
+	buflen -= 2*sizeof(char *);
+	se->s_aliases[1] = 0;
+	se->s_aliases[0] = se->s_name = buf;
+
+	switch (getnameinfo((const struct sockaddr *) &sin, sizeof sin, 0, 0, buf, buflen,
+		strcmp(prots, "udp") ? 0 : NI_DGRAM)) {
+	case EAI_MEMORY:
+	case EAI_SYSTEM:
+		return ENOMEM;
+	default:
+		return ENOENT;
+	case 0:
+		break;
+	}
+
+	*res = se;
+	return 0;
+}
+#endif
diff --git a/apt-pkg/indexcopy.cc b/apt-pkg/indexcopy.cc
index ca5c42cb7..cf3bb058d 100644
--- a/apt-pkg/indexcopy.cc
+++ b/apt-pkg/indexcopy.cc
@@ -550,7 +550,7 @@ bool SigVerify::CopyAndVerify(string CDROM,string Name,vector<string> &SigList,
       if(Debug)
 	 cout << "Signature verify for: " << *I << endl;
 
-      metaIndex *MetaIndex = new debReleaseIndex("","", {});
+      metaIndex *MetaIndex = new debReleaseIndex("","");
       string prefix = *I; 
 
       string const releasegpg = *I+"Release.gpg";
diff --git a/apt-pkg/indexfile.cc b/apt-pkg/indexfile.cc
index 934943205..21765388f 100644
--- a/apt-pkg/indexfile.cc
+++ b/apt-pkg/indexfile.cc
@@ -344,6 +344,7 @@ pkgCacheListParser * pkgDebianIndexFile::CreateListParser(FileFd &Pkg)
    if (newError)
    {
       delete Parser;
+      _error->ReturnError();
       return nullptr;
    }
    else
diff --git a/apt-pkg/memrchr.cc b/apt-pkg/memrchr.cc
new file mode 100644
index 000000000..edf8f346a
--- /dev/null
+++ b/apt-pkg/memrchr.cc
@@ -0,0 +1,157 @@
+/* memrchr -- find the last occurrence of a byte in a memory block
+
+   Copyright (C) 1991, 1993, 1996-1997, 1999-2000, 2003-2015 Free Software
+   Foundation, Inc.
+
+   Based on strlen implementation by Torbjorn Granlund (tege@sics.se),
+   with help from Dan Sahlin (dan@sics.se) and
+   commentary by Jim Blandy (jimb@ai.mit.edu);
+   adaptation to memchr suggested by Dick Karpinski (dick@cca.ucsf.edu),
+   and implemented by Roland McGrath (roland@ai.mit.edu).
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.  */
+
+#include <config.h>
+
+#ifndef HAVE_MEMRCHR
+#define reg_char char
+
+#include <string.h>
+#include <limits.h>
+
+#undef __memrchr
+#ifdef _LIBC
+# undef memrchr
+#endif
+
+#ifndef weak_alias
+# define __memrchr memrchr
+#endif
+
+/* Search no more than N bytes of S for C.  */
+extern "C" void *
+memrchr (const void *s, int c_in, size_t n)
+{
+  /* On 32-bit hardware, choosing longword to be a 32-bit unsigned
+     long instead of a 64-bit uintmax_t tends to give better
+     performance.  On 64-bit hardware, unsigned long is generally 64
+     bits already.  Change this typedef to experiment with
+     performance.  */
+  typedef unsigned long int longword;
+
+  const unsigned char *char_ptr;
+  const longword *longword_ptr;
+  longword repeated_one;
+  longword repeated_c;
+  unsigned reg_char c;
+
+  c = (unsigned char) c_in;
+
+  /* Handle the last few bytes by reading one byte at a time.
+     Do this until CHAR_PTR is aligned on a longword boundary.  */
+  for (char_ptr = (const unsigned char *) s + n;
+       n > 0 && (size_t) char_ptr % sizeof (longword) != 0;
+       --n)
+    if (*--char_ptr == c)
+      return (void *) char_ptr;
+
+  longword_ptr = (const longword *) char_ptr;
+
+  /* All these elucidatory comments refer to 4-byte longwords,
+     but the theory applies equally well to any size longwords.  */
+
+  /* Compute auxiliary longword values:
+     repeated_one is a value which has a 1 in every byte.
+     repeated_c has c in every byte.  */
+  repeated_one = 0x01010101;
+  repeated_c = c | (c << 8);
+  repeated_c |= repeated_c << 16;
+  if (0xffffffffU < (longword) -1)
+    {
+      repeated_one |= repeated_one << 31 << 1;
+      repeated_c |= repeated_c << 31 << 1;
+      if (8 < sizeof (longword))
+        {
+          size_t i;
+
+          for (i = 64; i < sizeof (longword) * 8; i *= 2)
+            {
+              repeated_one |= repeated_one << i;
+              repeated_c |= repeated_c << i;
+            }
+        }
+    }
+
+  /* Instead of the traditional loop which tests each byte, we will test a
+     longword at a time.  The tricky part is testing if *any of the four*
+     bytes in the longword in question are equal to c.  We first use an xor
+     with repeated_c.  This reduces the task to testing whether *any of the
+     four* bytes in longword1 is zero.
+
+     We compute tmp =
+       ((longword1 - repeated_one) & ~longword1) & (repeated_one << 7).
+     That is, we perform the following operations:
+       1. Subtract repeated_one.
+       2. & ~longword1.
+       3. & a mask consisting of 0x80 in every byte.
+     Consider what happens in each byte:
+       - If a byte of longword1 is zero, step 1 and 2 transform it into 0xff,
+         and step 3 transforms it into 0x80.  A carry can also be propagated
+         to more significant bytes.
+       - If a byte of longword1 is nonzero, let its lowest 1 bit be at
+         position k (0 <= k <= 7); so the lowest k bits are 0.  After step 1,
+         the byte ends in a single bit of value 0 and k bits of value 1.
+         After step 2, the result is just k bits of value 1: 2^k - 1.  After
+         step 3, the result is 0.  And no carry is produced.
+     So, if longword1 has only non-zero bytes, tmp is zero.
+     Whereas if longword1 has a zero byte, call j the position of the least
+     significant zero byte.  Then the result has a zero at positions 0, ...,
+     j-1 and a 0x80 at position j.  We cannot predict the result at the more
+     significant bytes (positions j+1..3), but it does not matter since we
+     already have a non-zero bit at position 8*j+7.
+
+     So, the test whether any byte in longword1 is zero is equivalent to
+     testing whether tmp is nonzero.  */
+
+  while (n >= sizeof (longword))
+    {
+      longword longword1 = *--longword_ptr ^ repeated_c;
+
+      if ((((longword1 - repeated_one) & ~longword1)
+           & (repeated_one << 7)) != 0)
+        {
+          longword_ptr++;
+          break;
+        }
+      n -= sizeof (longword);
+    }
+
+  char_ptr = (const unsigned char *) longword_ptr;
+
+  /* At this point, we know that either n < sizeof (longword), or one of the
+     sizeof (longword) bytes starting at char_ptr is == c.  On little-endian
+     machines, we could determine the first such byte without any further
+     memory accesses, just by looking at the tmp result from the last loop
+     iteration.  But this does not work on big-endian machines.  Choose code
+     that works in both cases.  */
+
+  while (n-- > 0)
+    {
+      if (*--char_ptr == c)
+        return (void *) char_ptr;
+    }
+
+  return NULL;
+}
+#endif
diff --git a/apt-pkg/missing.h b/apt-pkg/missing.h
new file mode 100644
index 000000000..441b47ce1
--- /dev/null
+++ b/apt-pkg/missing.h
@@ -0,0 +1,26 @@
+#define _GNU_SOURCE
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <netdb.h>
+#include <inttypes.h>
+#include <errno.h>
+#include <string.h>
+
+#ifndef PKGLIB_MISSING_H
+#define PKGLIB_MISSING_H
+
+extern "C" {
+	void *memrchr(const void *s, int c, size_t n);
+	void *rawmemchr(const void *s, int c);
+	char *strchrnul(const char *s, int c);
+	int getservbyport_r(int port, const char *prots, struct servent *se, char *buf, size_t buflen, struct servent **res);
+}
+
+typedef void (*sighandler_t)(int);
+
+extern char **environ;
+
+#define AI_IDN 0x0040
+
+#endif
+
diff --git a/apt-pkg/nameser_compat.h b/apt-pkg/nameser_compat.h
new file mode 100644
index 000000000..b2cf2ffaa
--- /dev/null
+++ b/apt-pkg/nameser_compat.h
@@ -0,0 +1,187 @@
+/* Copyright (c) 1983, 1989
+ *    The Regents of the University of California.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 4. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*%
+ *      from nameser.h	8.1 (Berkeley) 6/2/93
+ *	$BINDId: nameser_compat.h,v 8.11 1999/01/02 08:00:58 vixie Exp $
+ */
+
+#ifndef _ARPA_NAMESER_COMPAT_
+#define	_ARPA_NAMESER_COMPAT_
+
+#define	__BIND		19950621	/*%< (DEAD) interface version stamp. */
+
+#include <apt-pkg/endian.h>
+
+/*%
+ * Structure for query header.  The order of the fields is machine- and
+ * compiler-dependent, depending on the byte/bit order and the layout
+ * of bit fields.  We use bit fields only in int variables, as this
+ * is all ANSI requires.  This requires a somewhat confusing rearrangement.
+ */
+
+typedef struct {
+	unsigned	id :16;		/*%< query identification number */
+#if BYTE_ORDER == BIG_ENDIAN
+			/* fields in third byte */
+	unsigned	qr: 1;		/*%< response flag */
+	unsigned	opcode: 4;	/*%< purpose of message */
+	unsigned	aa: 1;		/*%< authoritive answer */
+	unsigned	tc: 1;		/*%< truncated message */
+	unsigned	rd: 1;		/*%< recursion desired */
+			/* fields in fourth byte */
+	unsigned	ra: 1;		/*%< recursion available */
+	unsigned	unused :1;	/*%< unused bits (MBZ as of 4.9.3a3) */
+	unsigned	ad: 1;		/*%< authentic data from named */
+	unsigned	cd: 1;		/*%< checking disabled by resolver */
+	unsigned	rcode :4;	/*%< response code */
+#endif
+#if BYTE_ORDER == LITTLE_ENDIAN || BYTE_ORDER == PDP_ENDIAN
+			/* fields in third byte */
+	unsigned	rd :1;		/*%< recursion desired */
+	unsigned	tc :1;		/*%< truncated message */
+	unsigned	aa :1;		/*%< authoritive answer */
+	unsigned	opcode :4;	/*%< purpose of message */
+	unsigned	qr :1;		/*%< response flag */
+			/* fields in fourth byte */
+	unsigned	rcode :4;	/*%< response code */
+	unsigned	cd: 1;		/*%< checking disabled by resolver */
+	unsigned	ad: 1;		/*%< authentic data from named */
+	unsigned	unused :1;	/*%< unused bits (MBZ as of 4.9.3a3) */
+	unsigned	ra :1;		/*%< recursion available */
+#endif
+			/* remaining bytes */
+	unsigned	qdcount :16;	/*%< number of question entries */
+	unsigned	ancount :16;	/*%< number of answer entries */
+	unsigned	nscount :16;	/*%< number of authority entries */
+	unsigned	arcount :16;	/*%< number of resource entries */
+} HEADER;
+
+#define PACKETSZ	NS_PACKETSZ
+#define MAXDNAME	NS_MAXDNAME
+#define MAXCDNAME	NS_MAXCDNAME
+#define MAXLABEL	NS_MAXLABEL
+#define	HFIXEDSZ	NS_HFIXEDSZ
+#define QFIXEDSZ	NS_QFIXEDSZ
+#define RRFIXEDSZ	NS_RRFIXEDSZ
+#define	INT32SZ		NS_INT32SZ
+#define	INT16SZ		NS_INT16SZ
+#define INT8SZ		NS_INT8SZ
+#define	INADDRSZ	NS_INADDRSZ
+#define	IN6ADDRSZ	NS_IN6ADDRSZ
+#define	INDIR_MASK	NS_CMPRSFLGS
+#define NAMESERVER_PORT	NS_DEFAULTPORT
+
+#define S_ZONE		ns_s_zn
+#define S_PREREQ	ns_s_pr
+#define S_UPDATE	ns_s_ud
+#define S_ADDT		ns_s_ar
+
+#define QUERY		ns_o_query
+#define IQUERY		ns_o_iquery
+#define STATUS		ns_o_status
+#define	NS_NOTIFY_OP	ns_o_notify
+#define	NS_UPDATE_OP	ns_o_update
+
+#define NOERROR		ns_r_noerror
+#define FORMERR		ns_r_formerr
+#define SERVFAIL	ns_r_servfail
+#define NXDOMAIN	ns_r_nxdomain
+#define NOTIMP		ns_r_notimpl
+#define REFUSED		ns_r_refused
+#define YXDOMAIN	ns_r_yxdomain
+#define YXRRSET		ns_r_yxrrset
+#define NXRRSET		ns_r_nxrrset
+#define NOTAUTH		ns_r_notauth
+#define NOTZONE		ns_r_notzone
+/*#define BADSIG		ns_r_badsig*/
+/*#define BADKEY		ns_r_badkey*/
+/*#define BADTIME		ns_r_badtime*/
+
+
+#define DELETE		ns_uop_delete
+#define ADD		ns_uop_add
+
+#define T_A		ns_t_a
+#define T_NS		ns_t_ns
+#define T_MD		ns_t_md
+#define T_MF		ns_t_mf
+#define T_CNAME		ns_t_cname
+#define T_SOA		ns_t_soa
+#define T_MB		ns_t_mb
+#define T_MG		ns_t_mg
+#define T_MR		ns_t_mr
+#define T_NULL		ns_t_null
+#define T_WKS		ns_t_wks
+#define T_PTR		ns_t_ptr
+#define T_HINFO		ns_t_hinfo
+#define T_MINFO		ns_t_minfo
+#define T_MX		ns_t_mx
+#define T_TXT		ns_t_txt
+#define	T_RP		ns_t_rp
+#define T_AFSDB		ns_t_afsdb
+#define T_X25		ns_t_x25
+#define T_ISDN		ns_t_isdn
+#define T_RT		ns_t_rt
+#define T_NSAP		ns_t_nsap
+#define T_NSAP_PTR	ns_t_nsap_ptr
+#define	T_SIG		ns_t_sig
+#define	T_KEY		ns_t_key
+#define	T_PX		ns_t_px
+#define	T_GPOS		ns_t_gpos
+#define	T_AAAA		ns_t_aaaa
+#define	T_LOC		ns_t_loc
+#define	T_NXT		ns_t_nxt
+#define	T_EID		ns_t_eid
+#define	T_NIMLOC	ns_t_nimloc
+#define	T_SRV		ns_t_srv
+#define T_ATMA		ns_t_atma
+#define T_NAPTR		ns_t_naptr
+#define T_A6		ns_t_a6
+#define T_DNAME		ns_t_dname
+#define	T_TSIG		ns_t_tsig
+#define	T_IXFR		ns_t_ixfr
+#define T_AXFR		ns_t_axfr
+#define T_MAILB		ns_t_mailb
+#define T_MAILA		ns_t_maila
+#define T_ANY		ns_t_any
+
+#define C_IN		ns_c_in
+#define C_CHAOS		ns_c_chaos
+#define C_HS		ns_c_hs
+/* BIND_UPDATE */
+#define C_NONE		ns_c_none
+#define C_ANY		ns_c_any
+
+#define	GETSHORT		NS_GET16
+#define	GETLONG			NS_GET32
+#define	PUTSHORT		NS_PUT16
+#define	PUTLONG			NS_PUT32
+
+#endif /* _ARPA_NAMESER_COMPAT_ */
+/*! \file */
diff --git a/apt-pkg/pkgcache.cc b/apt-pkg/pkgcache.cc
index c4bf34022..c5ed1b916 100644
--- a/apt-pkg/pkgcache.cc
+++ b/apt-pkg/pkgcache.cc
@@ -154,6 +154,7 @@ bool pkgCache::ReMap(bool const &Errorchecks)
    VerP = (Version *)Map.Data();
    DescP = (Description *)Map.Data();
    ProvideP = (Provides *)Map.Data();
+   TagP = (Tag *)Map.Data();
    DepP = (Dependency *)Map.Data();
    DepDataP = (DependencyData *)Map.Data();
    StrP = (char *)Map.Data();
diff --git a/apt-pkg/pkgcache.h b/apt-pkg/pkgcache.h
index 91228f713..83bf8961d 100644
--- a/apt-pkg/pkgcache.h
+++ b/apt-pkg/pkgcache.h
@@ -123,6 +123,7 @@ class pkgCache								/*{{{*/
    struct StringItem;
    struct VerFile;
    struct DescFile;
+   struct Tag;
    
    // Iterators
    template<typename Str, typename Itr> class Iterator;
@@ -136,6 +137,7 @@ class pkgCache								/*{{{*/
    class PkgFileIterator;
    class VerFileIterator;
    class DescFileIterator;
+   class TagIterator;
    
    class Namespace;
    
@@ -214,6 +216,7 @@ class pkgCache								/*{{{*/
    ReleaseFile *RlsFileP;
    PackageFile *PkgFileP;
    Version *VerP;
+   Tag *TagP;
    Description *DescP;
    Provides *ProvideP;
    Dependency *DepP;
@@ -320,6 +323,7 @@ struct pkgCache::Header
    map_number_t ReleaseFileSz;
    map_number_t PackageFileSz;
    map_number_t VersionSz;
+   map_number_t TagSz;
    map_number_t DescriptionSz;
    map_number_t DependencySz;
    map_number_t DependencyDataSz;
@@ -335,6 +339,7 @@ struct pkgCache::Header
    map_id_t GroupCount;
    map_id_t PackageCount;
    map_id_t VersionCount;
+   map_id_t TagCount;
    map_id_t DescriptionCount;
    map_id_t DependsCount;
    map_id_t DependsDataCount;
@@ -585,6 +590,16 @@ struct pkgCache::VerFile
    map_filesize_t Size;
 };
 									/*}}}*/
+// TagFile structure							/*{{{*/
+/** \brief associates a tag with something */
+struct pkgCache::Tag
+{
+   /** \brief name of this tag */
+   map_stringitem_t Name;
+   /** \brief next step in the linked list */
+   map_pointer_t NextTag;       // Tag
+};
+									/*}}}*/
 // DescFile structure							/*{{{*/
 /** \brief associates a description with a Translation file */
 struct pkgCache::DescFile
@@ -612,6 +627,8 @@ struct pkgCache::Version
    map_stringitem_t VerStr;
    /** \brief section this version is filled in */
    map_stringitem_t Section;
+   /** \brief high-level name used to display package */
+   map_stringitem_t Display;
    /** \brief source package name this version comes from
       Always contains the name, even if it is the same as the binary name */
    map_stringitem_t SourcePkgName;
@@ -656,6 +673,8 @@ struct pkgCache::Version
    map_pointer_t ParentPkg;         // Package
    /** \brief list of pkgCache::Provides */
    map_pointer_t ProvidesList;      // Provides
+   /** \brief list of pkgCache::Tag */
+   map_pointer_t TagList;           // Tag
 
    /** \brief archive size for this version
 
@@ -808,6 +827,7 @@ class pkgCache::Namespace						/*{{{*/
    typedef pkgCache::GrpIterator GrpIterator;
    typedef pkgCache::PkgIterator PkgIterator;
    typedef pkgCache::VerIterator VerIterator;
+   typedef pkgCache::TagIterator TagIterator;
    typedef pkgCache::DescIterator DescIterator;
    typedef pkgCache::DepIterator DepIterator;
    typedef pkgCache::PrvIterator PrvIterator;
diff --git a/apt-pkg/pkgcachegen.cc b/apt-pkg/pkgcachegen.cc
index e52667fbc..678b5f660 100644
--- a/apt-pkg/pkgcachegen.cc
+++ b/apt-pkg/pkgcachegen.cc
@@ -70,7 +70,7 @@ bool pkgCacheGenerator::Start()
       bool const newError = _error->PendingError();
       _error->MergeWithStack();
       if (newError)
-	 return false;
+	 return _error->ReturnError();
       if (Map.Size() <= 0)
 	 return false;
 
@@ -134,7 +134,7 @@ bool pkgCacheGenerator::Start()
    advoid a problem during a crash */
 pkgCacheGenerator::~pkgCacheGenerator()
 {
-   if (_error->PendingError() == true || Map.validData() == false)
+   if (Map.validData() == false)
       return;
    if (Map.Sync() == false)
       return;
@@ -175,6 +175,10 @@ void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newM
 	i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i)
       if (std::get<1>(seen.insert(*i)) == true)
 	 (*i)->ReMap(oldMap, newMap);
+   for (std::vector<pkgCache::TagIterator*>::const_iterator i = Dynamic<pkgCache::TagIterator>::toReMap.begin();
+	i != Dynamic<pkgCache::TagIterator>::toReMap.end(); ++i)
+      if (std::get<1>(seen.insert(*i)) == true)
+	 (*i)->ReMap(oldMap, newMap);
    for (std::vector<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
 	i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i)
 	if (std::get<1>(seen.insert(*i)) == true)
@@ -250,7 +254,7 @@ bool pkgCacheGenerator::MergeList(ListParser &List,
    {
       string const PackageName = List.Package();
       if (PackageName.empty() == true)
-	 return false;
+          continue;
 
       Counter++;
       if (Counter % 100 == 0 && Progress != 0)
@@ -264,24 +268,26 @@ bool pkgCacheGenerator::MergeList(ListParser &List,
       {
 	 // package descriptions
 	 if (MergeListGroup(List, PackageName) == false)
-	    return false;
+	    continue;
 	 continue;
       }
 
       // Get a pointer to the package structure
       pkgCache::PkgIterator Pkg;
       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
-      if (NewPackage(Pkg, PackageName, Arch) == false)
+      if (NewPackage(Pkg, PackageName, Arch) == false) {
 	 // TRANSLATOR: The first placeholder is a package name,
 	 // the other two should be copied verbatim as they include debug info
-	 return _error->Error(_("Error occurred while processing %s (%s%d)"),
+	 _error->Error(_("Error occurred while processing %s (%s%d)"),
 			      PackageName.c_str(), "NewPackage", 1);
+         continue;
+      }
 
 
       if (Version.empty() == true)
       {
 	 if (MergeListPackage(List, Pkg) == false)
-	    return false;
+	    continue;
       }
       else
       {
@@ -1274,6 +1280,38 @@ bool pkgCacheGenerator::SelectReleaseFile(const string &File,const string &Site,
    return true;
 }
 									/*}}}*/
+// ListParser::NewTag - Create a Tag element				/*{{{*/
+// ---------------------------------------------------------------------
+/* */
+bool pkgCacheListParser::NewTag(pkgCache::VerIterator &Ver,
+					   const char *NameStart,
+					   unsigned int NameSize)
+{
+   return Owner->NewTag(Ver, NameStart, NameSize);
+}
+bool pkgCacheGenerator::NewTag(pkgCache::VerIterator &Ver,
+					   const char *NameStart,
+					   unsigned int NameSize)
+{
+   // Get a structure
+   map_pointer_t const idxTag = AllocateInMap(sizeof(pkgCache::Tag));
+   if (unlikely(idxTag == 0))
+      return false;
+   
+   // Fill it in
+   pkgCache::TagIterator Tg(Cache,Cache.TagP + idxTag);
+   map_pointer_t const idxName = StoreString(TAG,NameStart,NameSize);
+   if (idxName == 0)
+      return false;
+   Tg->Name = idxName;
+
+   Tg->NextTag = Ver->TagList;
+   Ver->TagList = Tg.Index();
+   Cache.HeaderP->TagCount++;
+   
+   return true;
+}
+									/*}}}*/
 // CacheGenerator::SelectFile - Select the current file being parsed	/*{{{*/
 // ---------------------------------------------------------------------
 /* This is used to select which file is to be associated with all newly
@@ -1341,6 +1379,7 @@ map_stringitem_t pkgCacheGenerator::StoreString(enum StringType const type, cons
       case PKGNAME: strings = &strPkgNames; break;
       case VERSIONNUMBER: strings = &strVersions; break;
       case SECTION: strings = &strSections; break;
+      case TAG: strings = &strTags; break;
       default: _error->Fatal("Unknown enum type used for string storage of '%.*s'", Size, S); return 0;
    }
 
@@ -1480,7 +1519,7 @@ static bool CheckValidity(FileFd &CacheFile, std::string const &CacheFileName,
 	 std::clog << "Validity failed because of pending errors:" << std::endl;
 	 _error->DumpErrors(std::clog, GlobalError::DEBUG, false);
       }
-      return false;
+      return _error->ReturnError();
    }
 
    if (OutMap != 0)
@@ -1518,16 +1557,14 @@ static map_filesize_t ComputeSize(pkgSourceList const * const List, FileIterator
 }
 									/*}}}*/
 // BuildCache - Merge the list of index files into the cache		/*{{{*/
-static bool BuildCache(pkgCacheGenerator &Gen,
+static void BuildCache(pkgCacheGenerator &Gen,
 		       OpProgress * const Progress,
 		       map_filesize_t &CurrentSize,map_filesize_t TotalSize,
 		       pkgSourceList const * const List,
 		       FileIterator const Start, FileIterator const End)
 {
-   bool mergeFailure = false;
-
    auto const indexFileMerge = [&](pkgIndexFile * const I) {
-      if (I->HasPackages() == false || mergeFailure)
+      if (I->HasPackages() == false)
 	 return;
 
       if (I->Exists() == false)
@@ -1545,8 +1582,10 @@ static bool BuildCache(pkgCacheGenerator &Gen,
 	 Progress->OverallProgress(CurrentSize, TotalSize, Size, _("Reading package lists"));
       CurrentSize += Size;
 
-      if (I->Merge(Gen,Progress) == false)
-	 mergeFailure = true;
+      if (I->Merge(Gen,Progress) == false) {
+	 _error->ReturnError();
+	 return;
+      }
    };
 
    if (List !=  NULL)
@@ -1560,14 +1599,14 @@ static bool BuildCache(pkgCacheGenerator &Gen,
 	    continue;
 	 }
 
-	 if ((*i)->Merge(Gen, Progress) == false)
-	    return false;
+	 if ((*i)->Merge(Gen, Progress) == false) {
+	    _error->ReturnError();
+	    continue;
+	 }
 
 	 std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
 	 if (Indexes != NULL)
 	    std::for_each(Indexes->begin(), Indexes->end(), indexFileMerge);
-	 if (mergeFailure)
-	    return false;
       }
    }
 
@@ -1575,10 +1614,7 @@ static bool BuildCache(pkgCacheGenerator &Gen,
    {
       Gen.SelectReleaseFile("", "");
       std::for_each(Start, End, indexFileMerge);
-      if (mergeFailure)
-	 return false;
    }
-   return true;
 }
 									/*}}}*/
 // CacheGenerator::MakeStatusCache - Construct the status cache		/*{{{*/
@@ -1637,7 +1673,7 @@ static bool loadBackMMapFromFile(std::unique_ptr<pkgCacheGenerator> &Gen,
    bool const newError = _error->PendingError();
    _error->MergeWithStack();
    if (alloc == 0 && newError)
-      return false;
+      return _error->ReturnError();
    if (CacheF.Read((unsigned char *)Map->Data() + alloc, CacheF.Size()) == false)
       return false;
    Gen.reset(new pkgCacheGenerator(Map.get(),Progress));
@@ -1753,9 +1789,8 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress
 	 return false;
 
       TotalSize += ComputeSize(&List, Files.begin(),Files.end());
-      if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, &List,
-	       Files.end(),Files.end()) == false)
-	 return false;
+      BuildCache(*Gen, Progress, CurrentSize, TotalSize, &List,
+	       Files.end(),Files.end());
 
       if (Writeable == true && SrcCacheFileName.empty() == false)
 	 if (writeBackMMapToFile(Gen.get(), Map.get(), SrcCacheFileName) == false)
@@ -1766,9 +1801,8 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress
    {
       if (Debug == true)
 	 std::clog << "Building status cache in pkgcache.bin now" << std::endl;
-      if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
-	       Files.begin(), Files.end()) == false)
-	 return false;
+      BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
+	       Files.begin(), Files.end());
 
       if (Writeable == true && CacheFileName.empty() == false)
 	 if (writeBackMMapToFile(Gen.get(), Map.get(), CacheFileName) == false)
@@ -1789,9 +1823,8 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress
       }
 
       Files = List.GetVolatileFiles();
-      if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
-	       Files.begin(), Files.end()) == false)
-	 return false;
+      BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
+	       Files.begin(), Files.end());
    }
 
    if (OutMap != nullptr)
@@ -1828,14 +1861,15 @@ bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **O
    if (Progress != NULL)
       Progress->OverallProgress(0,1,1,_("Reading package lists"));
    pkgCacheGenerator Gen(Map.get(),Progress);
-   if (Gen.Start() == false || _error->PendingError() == true)
-      return false;
-   if (BuildCache(Gen,Progress,CurrentSize,TotalSize, NULL,
-		  Files.begin(), Files.end()) == false)
+   if (Gen.Start() == false)
       return false;
-
    if (_error->PendingError() == true)
-      return false;
+      return _error->ReturnError();
+   BuildCache(Gen,Progress,CurrentSize,TotalSize, NULL,
+		  Files.begin(), Files.end());
+   // We've passed the point of no return
+   _error->ReturnError();
+
    *OutMap = Map.release();
    
    return true;
diff --git a/apt-pkg/pkgcachegen.h b/apt-pkg/pkgcachegen.h
index cb51c113a..27fd8d4d0 100644
--- a/apt-pkg/pkgcachegen.h
+++ b/apt-pkg/pkgcachegen.h
@@ -78,6 +78,7 @@ class APT_HIDDEN pkgCacheGenerator					/*{{{*/
    std::unordered_set<string_pointer, hash> strPkgNames;
    std::unordered_set<string_pointer, hash> strVersions;
    std::unordered_set<string_pointer, hash> strSections;
+   std::unordered_set<string_pointer, hash> strTags;
 #endif
 
    friend class pkgCacheListParser;
@@ -128,10 +129,11 @@ class APT_HIDDEN pkgCacheGenerator					/*{{{*/
 		   uint8_t const Type, map_pointer_t* &OldDepLast);
    bool NewProvides(pkgCache::VerIterator &Ver, pkgCache::PkgIterator &Pkg,
 		    map_stringitem_t const ProvidesVersion, uint8_t const Flags);
+   bool NewTag(pkgCache::VerIterator &Ver,const char *NameStart,unsigned int NameSize);
 
    public:
 
-   enum StringType { MIXED, PKGNAME, VERSIONNUMBER, SECTION };
+   enum StringType { MIXED, PKGNAME, VERSIONNUMBER, SECTION, TAG };
    map_stringitem_t StoreString(StringType const type, const char * S, unsigned int const Size);
 
 #ifdef APT_PKG_EXPOSE_STRING_VIEW
@@ -209,6 +211,7 @@ class APT_HIDDEN pkgCacheListParser
 		    uint8_t const Flags);
    bool NewProvidesAllArch(pkgCache::VerIterator &Ver, APT::StringView Package,
 			   APT::StringView Version, uint8_t const Flags);
+   bool NewTag(pkgCache::VerIterator &Ver,const char *NameStart,unsigned int NameSize);
 #endif
    public:
    
diff --git a/apt-pkg/pkgrecords.h b/apt-pkg/pkgrecords.h
index 7c50c5d41..5d4ba37d7 100644
--- a/apt-pkg/pkgrecords.h
+++ b/apt-pkg/pkgrecords.h
@@ -92,6 +92,7 @@ class pkgRecords::Parser						/*{{{*/
    std::string LongDesc() {return LongDesc("");};
 
    virtual std::string Name() {return std::string();};
+   virtual std::string Display() {return std::string();}
    virtual std::string Homepage() {return std::string();}
 
    // An arbitrary custom field
@@ -100,6 +101,9 @@ class pkgRecords::Parser						/*{{{*/
    // The record in binary form
    virtual void GetRec(const char *&Start,const char *&Stop) {Start = Stop = 0;};
 
+   // Locate a tag
+   virtual bool Find(const char *Tag,const char *&Start, const char *&End) {Start = End = 0; return false;};
+
    Parser();
    virtual ~Parser();
 
diff --git a/apt-pkg/policy.cc b/apt-pkg/policy.cc
index 3dd6ddac4..a13510e66 100644
--- a/apt-pkg/policy.cc
+++ b/apt-pkg/policy.cc
@@ -320,7 +320,7 @@ bool ReadPinDir(pkgPolicy &Plcy,string Dir)
    bool const PendingErrors = _error->PendingError();
    _error->MergeWithStack();
    if (PendingErrors)
-      return false;
+      return _error->ReturnError();
 
    // Read the files
    for (vector<string>::const_iterator I = List.begin(); I != List.end(); ++I)
@@ -391,6 +391,7 @@ bool ReadPinFile(pkgPolicy &Plcy,string File)
       if (priority < std::numeric_limits<short>::min() ||
           priority > std::numeric_limits<short>::max() ||
 	  newError) {
+	 _error->ReturnError();
 	 return _error->Error(_("%s: Value %s is outside the range of valid pin priorities (%d to %d)"),
 			      File.c_str(), Tags.FindS("Pin-Priority").c_str(),
 			      std::numeric_limits<short>::min(),
diff --git a/apt-pkg/rawmemchr.cc b/apt-pkg/rawmemchr.cc
new file mode 100644
index 000000000..8f7669b6c
--- /dev/null
+++ b/apt-pkg/rawmemchr.cc
@@ -0,0 +1,139 @@
+/* Searching in a string.
+   Copyright (C) 2008-2015 Free Software Foundation, Inc.
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.  */
+
+#include <config.h>
+
+#ifndef HAVE_RAWMEMCHR
+
+/* Specification.  */
+#include <string.h>
+
+/* Find the first occurrence of C in S.  */
+extern "C" void *
+rawmemchr (const void *s, int c_in)
+{
+  /* On 32-bit hardware, choosing longword to be a 32-bit unsigned
+     long instead of a 64-bit uintmax_t tends to give better
+     performance.  On 64-bit hardware, unsigned long is generally 64
+     bits already.  Change this typedef to experiment with
+     performance.  */
+  typedef unsigned long int longword;
+
+  const unsigned char *char_ptr;
+  const longword *longword_ptr;
+  longword repeated_one;
+  longword repeated_c;
+  unsigned char c;
+
+  c = (unsigned char) c_in;
+
+  /* Handle the first few bytes by reading one byte at a time.
+     Do this until CHAR_PTR is aligned on a longword boundary.  */
+  for (char_ptr = (const unsigned char *) s;
+       (size_t) char_ptr % sizeof (longword) != 0;
+       ++char_ptr)
+    if (*char_ptr == c)
+      return (void *) char_ptr;
+
+  longword_ptr = (const longword *) char_ptr;
+
+  /* All these elucidatory comments refer to 4-byte longwords,
+     but the theory applies equally well to any size longwords.  */
+
+  /* Compute auxiliary longword values:
+     repeated_one is a value which has a 1 in every byte.
+     repeated_c has c in every byte.  */
+  repeated_one = 0x01010101;
+  repeated_c = c | (c << 8);
+  repeated_c |= repeated_c << 16;
+  if (0xffffffffU < (longword) -1)
+    {
+      repeated_one |= repeated_one << 31 << 1;
+      repeated_c |= repeated_c << 31 << 1;
+      if (8 < sizeof (longword))
+        {
+          size_t i;
+
+          for (i = 64; i < sizeof (longword) * 8; i *= 2)
+            {
+              repeated_one |= repeated_one << i;
+              repeated_c |= repeated_c << i;
+            }
+        }
+    }
+
+  /* Instead of the traditional loop which tests each byte, we will
+     test a longword at a time.  The tricky part is testing if *any of
+     the four* bytes in the longword in question are equal to NUL or
+     c.  We first use an xor with repeated_c.  This reduces the task
+     to testing whether *any of the four* bytes in longword1 is zero.
+
+     We compute tmp =
+       ((longword1 - repeated_one) & ~longword1) & (repeated_one << 7).
+     That is, we perform the following operations:
+       1. Subtract repeated_one.
+       2. & ~longword1.
+       3. & a mask consisting of 0x80 in every byte.
+     Consider what happens in each byte:
+       - If a byte of longword1 is zero, step 1 and 2 transform it into 0xff,
+         and step 3 transforms it into 0x80.  A carry can also be propagated
+         to more significant bytes.
+       - If a byte of longword1 is nonzero, let its lowest 1 bit be at
+         position k (0 <= k <= 7); so the lowest k bits are 0.  After step 1,
+         the byte ends in a single bit of value 0 and k bits of value 1.
+         After step 2, the result is just k bits of value 1: 2^k - 1.  After
+         step 3, the result is 0.  And no carry is produced.
+     So, if longword1 has only non-zero bytes, tmp is zero.
+     Whereas if longword1 has a zero byte, call j the position of the least
+     significant zero byte.  Then the result has a zero at positions 0, ...,
+     j-1 and a 0x80 at position j.  We cannot predict the result at the more
+     significant bytes (positions j+1..3), but it does not matter since we
+     already have a non-zero bit at position 8*j+7.
+
+     The test whether any byte in longword1 is zero is equivalent
+     to testing whether tmp is nonzero.
+
+     This test can read beyond the end of a string, depending on where
+     C_IN is encountered.  However, this is considered safe since the
+     initialization phase ensured that the read will be aligned,
+     therefore, the read will not cross page boundaries and will not
+     cause a fault.  */
+
+  while (1)
+    {
+      longword longword1 = *longword_ptr ^ repeated_c;
+
+      if ((((longword1 - repeated_one) & ~longword1)
+           & (repeated_one << 7)) != 0)
+        break;
+      longword_ptr++;
+    }
+
+  char_ptr = (const unsigned char *) longword_ptr;
+
+  /* At this point, we know that one of the sizeof (longword) bytes
+     starting at char_ptr is == c.  On little-endian machines, we
+     could determine the first such byte without any further memory
+     accesses, just by looking at the tmp result from the last loop
+     iteration.  But this does not work on big-endian machines.
+     Choose code that works in both cases.  */
+
+  char_ptr = (unsigned char *) longword_ptr;
+  while (*char_ptr != c)
+    char_ptr++;
+  return (void *) char_ptr;
+}
+#endif
diff --git a/apt-pkg/strchrnul.cc b/apt-pkg/strchrnul.cc
new file mode 100644
index 000000000..b68b8501b
--- /dev/null
+++ b/apt-pkg/strchrnul.cc
@@ -0,0 +1,147 @@
+/* Searching in a string.
+   Copyright (C) 2003, 2007-2015 Free Software Foundation, Inc.
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.  */
+
+#include <config.h>
+
+#ifndef HAVE_STRCHRNUL
+
+/* Specification.  */
+#include <string.h>
+
+#include <apt-pkg/missing.h>
+
+/* Find the first occurrence of C in S or the final NUL byte.  */
+extern "C" char *
+strchrnul (const char *s, int c_in)
+{
+  /* On 32-bit hardware, choosing longword to be a 32-bit unsigned
+     long instead of a 64-bit uintmax_t tends to give better
+     performance.  On 64-bit hardware, unsigned long is generally 64
+     bits already.  Change this typedef to experiment with
+     performance.  */
+  typedef unsigned long int longword;
+
+  const unsigned char *char_ptr;
+  const longword *longword_ptr;
+  longword repeated_one;
+  longword repeated_c;
+  unsigned char c;
+
+  c = (unsigned char) c_in;
+  if (!c)
+    return (char*) rawmemchr (s, 0);
+
+  /* Handle the first few bytes by reading one byte at a time.
+     Do this until CHAR_PTR is aligned on a longword boundary.  */
+  for (char_ptr = (const unsigned char *) s;
+       (size_t) char_ptr % sizeof (longword) != 0;
+       ++char_ptr)
+    if (!*char_ptr || *char_ptr == c)
+      return (char *) char_ptr;
+
+  longword_ptr = (const longword *) char_ptr;
+
+  /* All these elucidatory comments refer to 4-byte longwords,
+     but the theory applies equally well to any size longwords.  */
+
+  /* Compute auxiliary longword values:
+     repeated_one is a value which has a 1 in every byte.
+     repeated_c has c in every byte.  */
+  repeated_one = 0x01010101;
+  repeated_c = c | (c << 8);
+  repeated_c |= repeated_c << 16;
+  if (0xffffffffU < (longword) -1)
+    {
+      repeated_one |= repeated_one << 31 << 1;
+      repeated_c |= repeated_c << 31 << 1;
+      if (8 < sizeof (longword))
+        {
+          size_t i;
+
+          for (i = 64; i < sizeof (longword) * 8; i *= 2)
+            {
+              repeated_one |= repeated_one << i;
+              repeated_c |= repeated_c << i;
+            }
+        }
+    }
+
+  /* Instead of the traditional loop which tests each byte, we will
+     test a longword at a time.  The tricky part is testing if *any of
+     the four* bytes in the longword in question are equal to NUL or
+     c.  We first use an xor with repeated_c.  This reduces the task
+     to testing whether *any of the four* bytes in longword1 or
+     longword2 is zero.
+
+     Let's consider longword1.  We compute tmp =
+       ((longword1 - repeated_one) & ~longword1) & (repeated_one << 7).
+     That is, we perform the following operations:
+       1. Subtract repeated_one.
+       2. & ~longword1.
+       3. & a mask consisting of 0x80 in every byte.
+     Consider what happens in each byte:
+       - If a byte of longword1 is zero, step 1 and 2 transform it into 0xff,
+         and step 3 transforms it into 0x80.  A carry can also be propagated
+         to more significant bytes.
+       - If a byte of longword1 is nonzero, let its lowest 1 bit be at
+         position k (0 <= k <= 7); so the lowest k bits are 0.  After step 1,
+         the byte ends in a single bit of value 0 and k bits of value 1.
+         After step 2, the result is just k bits of value 1: 2^k - 1.  After
+         step 3, the result is 0.  And no carry is produced.
+     So, if longword1 has only non-zero bytes, tmp is zero.
+     Whereas if longword1 has a zero byte, call j the position of the least
+     significant zero byte.  Then the result has a zero at positions 0, ...,
+     j-1 and a 0x80 at position j.  We cannot predict the result at the more
+     significant bytes (positions j+1..3), but it does not matter since we
+     already have a non-zero bit at position 8*j+7.
+
+     The test whether any byte in longword1 or longword2 is zero is equivalent
+     to testing whether tmp1 is nonzero or tmp2 is nonzero.  We can combine
+     this into a single test, whether (tmp1 | tmp2) is nonzero.
+
+     This test can read more than one byte beyond the end of a string,
+     depending on where the terminating NUL is encountered.  However,
+     this is considered safe since the initialization phase ensured
+     that the read will be aligned, therefore, the read will not cross
+     page boundaries and will not cause a fault.  */
+
+  while (1)
+    {
+      longword longword1 = *longword_ptr ^ repeated_c;
+      longword longword2 = *longword_ptr;
+
+      if (((((longword1 - repeated_one) & ~longword1)
+            | ((longword2 - repeated_one) & ~longword2))
+           & (repeated_one << 7)) != 0)
+        break;
+      longword_ptr++;
+    }
+
+  char_ptr = (const unsigned char *) longword_ptr;
+
+  /* At this point, we know that one of the sizeof (longword) bytes
+     starting at char_ptr is == 0 or == c.  On little-endian machines,
+     we could determine the first such byte without any further memory
+     accesses, just by looking at the tmp result from the last loop
+     iteration.  But this does not work on big-endian machines.
+     Choose code that works in both cases.  */
+
+  char_ptr = (unsigned char *) longword_ptr;
+  while (*char_ptr && (*char_ptr != c))
+    char_ptr++;
+  return (char *) char_ptr;
+}
+#endif
diff --git a/apt-pkg/tagfile.cc b/apt-pkg/tagfile.cc
index d9519175e..943575956 100644
--- a/apt-pkg/tagfile.cc
+++ b/apt-pkg/tagfile.cc
@@ -13,6 +13,7 @@
 // Include Files							/*{{{*/
 #include<config.h>
 
+#include <apt-pkg/mmap.h>
 #include <apt-pkg/tagfile.h>
 #include <apt-pkg/tagfile-keys.h>
 #include <apt-pkg/error.h>
@@ -37,10 +38,11 @@ using APT::StringView;
 class APT_HIDDEN pkgTagFilePrivate					/*{{{*/
 {
 public:
-   void Reset(FileFd * const pFd, unsigned long long const pSize, pkgTagFile::Flags const pFlags)
+   void Reset(FileFd * const pFd, pkgTagFile::Flags const pFlags)
    {
-      if (Buffer != NULL)
-	 free(Buffer);
+      if (Map != NULL)
+         delete Map;
+      Map = NULL;
       Buffer = NULL;
       Fd = pFd;
       Flags = pFlags;
@@ -48,14 +50,11 @@ public:
       End = NULL;
       Done = false;
       iOffset = 0;
-      Size = pSize;
-      isCommentedLine = false;
-      chunks.clear();
    }
 
-   pkgTagFilePrivate(FileFd * const pFd, unsigned long long const Size, pkgTagFile::Flags const pFlags) : Buffer(NULL)
+   pkgTagFilePrivate(FileFd * const pFd, pkgTagFile::Flags const pFlags) : Map(NULL)
    {
-      Reset(pFd, Size, pFlags);
+      Reset(pFd, pFlags);
    }
    FileFd * Fd;
    pkgTagFile::Flags Flags;
@@ -64,20 +63,12 @@ public:
    char *End;
    bool Done;
    unsigned long long iOffset;
-   unsigned long long Size;
-   bool isCommentedLine;
-   struct FileChunk
-   {
-      bool const good;
-      size_t length;
-      FileChunk(bool const pgood, size_t const plength) : good(pgood), length(plength) {}
-   };
-   std::list<FileChunk> chunks;
+   MMap *Map;
 
    ~pkgTagFilePrivate()
    {
-      if (Buffer != NULL)
-	 free(Buffer);
+      if (Map != NULL)
+         delete Map;
    }
 };
 									/*}}}*/
@@ -116,42 +107,38 @@ static unsigned long BetaHash(const char *Text, size_t Length)		/*{{{*/
 									/*}}}*/
 
 // TagFile::pkgTagFile - Constructor					/*{{{*/
-pkgTagFile::pkgTagFile(FileFd * const pFd,pkgTagFile::Flags const pFlags, unsigned long long const Size)
-   : d(new pkgTagFilePrivate(pFd, Size + 4, pFlags))
+pkgTagFile::pkgTagFile(FileFd * const pFd,pkgTagFile::Flags const pFlags)
+   : d(new pkgTagFilePrivate(pFd, pFlags))
 {
-   Init(pFd, pFlags, Size);
+   Init(pFd, pFlags);
 }
-pkgTagFile::pkgTagFile(FileFd * const pFd,unsigned long long const Size)
-   : pkgTagFile(pFd, pkgTagFile::STRICT, Size)
+pkgTagFile::pkgTagFile(FileFd * const pFd)
+   : pkgTagFile(pFd, pkgTagFile::STRICT)
 {
 }
-void pkgTagFile::Init(FileFd * const pFd, pkgTagFile::Flags const pFlags, unsigned long long Size)
+void pkgTagFile::Init(FileFd * const pFd, pkgTagFile::Flags const pFlags)
 {
-   /* The size is increased by 4 because if we start with the Size of the
-      filename we need to try to read 1 char more to see an EOF faster, 1
-      char the end-pointer can be on and maybe 2 newlines need to be added
-      to the end of the file -> 4 extra chars */
-   Size += 4;
-   d->Reset(pFd, Size, pFlags);
-
-   if (d->Fd->IsOpen() == false)
-      d->Start = d->End = d->Buffer = 0;
-   else
-      d->Buffer = (char*)malloc(sizeof(char) * Size);
+   d->Reset(pFd, pFlags);
+
+   if (d->Fd->IsOpen() == false || d->Fd->Size() == 0)
+      _error->Discard();
+   else {
+      d->Map = new MMap(*d->Fd, MMap::ReadOnly);
+      d->Buffer = static_cast<char *>(d->Map->Data());
+   }
 
    if (d->Buffer == NULL)
       d->Done = true;
-   else
+   else {
       d->Done = false;
+      d->End = d->Buffer + d->Map->Size();
+   }
 
-   d->Start = d->End = d->Buffer;
-   d->iOffset = 0;
-   if (d->Done == false)
-      Fill();
+   d->Start = d->Buffer;
 }
-void pkgTagFile::Init(FileFd * const pFd,unsigned long long Size)
+void pkgTagFile::Init(FileFd * const pFd)
 {
-   Init(pFd, pkgTagFile::STRICT, Size);
+   Init(pFd, pkgTagFile::STRICT);
 }
 									/*}}}*/
 // TagFile::~pkgTagFile - Destructor					/*{{{*/
@@ -166,36 +153,6 @@ APT_PURE unsigned long pkgTagFile::Offset()
    return d->iOffset;
 }
 									/*}}}*/
-// TagFile::Resize - Resize the internal buffer				/*{{{*/
-// ---------------------------------------------------------------------
-/* Resize the internal buffer (double it in size). Fail if a maximum size
- * size is reached.
- */
-bool pkgTagFile::Resize()
-{
-   // fail is the buffer grows too big
-   if(d->Size > 1024*1024+1)
-      return false;
-
-   return Resize(d->Size * 2);
-}
-bool pkgTagFile::Resize(unsigned long long const newSize)
-{
-   unsigned long long const EndSize = d->End - d->Start;
-
-   // get new buffer and use it
-   char* const newBuffer = static_cast<char*>(realloc(d->Buffer, sizeof(char) * newSize));
-   if (newBuffer == NULL)
-      return false;
-   d->Buffer = newBuffer;
-   d->Size = newSize;
-
-   // update the start/end pointers to the new buffer
-   d->Start = d->Buffer;
-   d->End = d->Start + EndSize;
-   return true;
-}
-									/*}}}*/
 // TagFile::Step - Advance to the next section				/*{{{*/
 // ---------------------------------------------------------------------
 /* If the Section Scanner fails we refill the buffer and try again. 
@@ -204,221 +161,18 @@ bool pkgTagFile::Resize(unsigned long long const newSize)
  */
 bool pkgTagFile::Step(pkgTagSection &Tag)
 {
-   if(Tag.Scan(d->Start,d->End - d->Start) == false)
+   if(Tag.Scan(d->Start,d->End - d->Start,(d->Flags & SUPPORT_COMMENTS) != 0) == false)
    {
-      do
-      {
-	 if (Fill() == false)
-	    return false;
-
-	 if(Tag.Scan(d->Start,d->End - d->Start, false))
-	    break;
-
-	 if (Resize() == false)
-	    return _error->Error(_("Unable to parse package file %s (%d)"),
+      if (d->Start == d->End)
+         return false;
+      else
+	    return _error->Warning(_("Unable to parse package file %s (%d)"),
 		  d->Fd->Name().c_str(), 1);
-
-      } while (Tag.Scan(d->Start,d->End - d->Start, false) == false);
    }
 
    size_t tagSize = Tag.size();
    d->Start += tagSize;
-
-   if ((d->Flags & pkgTagFile::SUPPORT_COMMENTS) == 0)
-      d->iOffset += tagSize;
-   else
-   {
-      auto first = d->chunks.begin();
-      for (; first != d->chunks.end(); ++first)
-      {
-	 if (first->good == false)
-	    d->iOffset += first->length;
-	 else
-	 {
-	    if (tagSize < first->length)
-	    {
-	       first->length -= tagSize;
-	       d->iOffset += tagSize;
-	       break;
-	    }
-	    else
-	    {
-	       tagSize -= first->length;
-	       d->iOffset += first->length;
-	    }
-	 }
-      }
-      d->chunks.erase(d->chunks.begin(), first);
-   }
-
-   Tag.Trim();
-   return true;
-}
-									/*}}}*/
-// TagFile::Fill - Top up the buffer					/*{{{*/
-// ---------------------------------------------------------------------
-/* This takes the bit at the end of the buffer and puts it at the start
-   then fills the rest from the file */
-static bool FillBuffer(pkgTagFilePrivate * const d)
-{
-   unsigned long long Actual = 0;
-   // See if only a bit of the file is left
-   unsigned long long const dataSize = d->Size - ((d->End - d->Buffer) + 1);
-   if (d->Fd->Read(d->End, dataSize, &Actual) == false)
-      return false;
-   if (Actual != dataSize)
-      d->Done = true;
-   d->End += Actual;
-   return true;
-}
-static void RemoveCommentsFromBuffer(pkgTagFilePrivate * const d)
-{
-   // look for valid comments in the buffer
-   char * good_start = nullptr, * bad_start = nullptr;
-   char * current = d->Start;
-   if (d->isCommentedLine == false)
-   {
-      if (d->Start == d->Buffer)
-      {
-	 // the start of the buffer is a newline as a record can't start
-	 // in the middle of a line by definition.
-	 if (*d->Start == '#')
-	 {
-	    d->isCommentedLine = true;
-	    ++current;
-	    if (current > d->End)
-	       d->chunks.emplace_back(false, 1);
-	 }
-      }
-      if (d->isCommentedLine == false)
-	 good_start = d->Start;
-      else
-	 bad_start = d->Start;
-   }
-   else
-      bad_start = d->Start;
-
-   std::vector<std::pair<char*, size_t>> good_parts;
-   while (current <= d->End)
-   {
-      size_t const restLength = (d->End - current);
-      if (d->isCommentedLine == false)
-      {
-	 current = static_cast<char*>(memchr(current, '#', restLength));
-	 if (current == nullptr)
-	 {
-	    size_t const goodLength = d->End - good_start;
-	    d->chunks.emplace_back(true, goodLength);
-	    if (good_start != d->Start)
-	       good_parts.push_back(std::make_pair(good_start, goodLength));
-	    break;
-	 }
-	 bad_start = current;
-	 --current;
-	 // ensure that this is really a comment and not a '#' in the middle of a line
-	 if (*current == '\n')
-	 {
-	    size_t const goodLength = (current - good_start) + 1;
-	    d->chunks.emplace_back(true, goodLength);
-	    good_parts.push_back(std::make_pair(good_start, goodLength));
-	    good_start = nullptr;
-	    d->isCommentedLine = true;
-	 }
-	 current += 2;
-      }
-      else // the current line is a comment
-      {
-	 current = static_cast<char*>(memchr(current, '\n', restLength));
-	 if (current == nullptr)
-	 {
-	    d->chunks.emplace_back(false, (d->End - bad_start));
-	    break;
-	 }
-	 ++current;
-	 // is the next line a comment, too?
-	 if (current >= d->End || *current != '#')
-	 {
-	    d->chunks.emplace_back(false, (current - bad_start));
-	    good_start = current;
-	    bad_start = nullptr;
-	    d->isCommentedLine = false;
-	 }
-	 ++current;
-      }
-   }
-
-   if (good_parts.empty() == false)
-   {
-      // we found comments, so move later parts over them
-      current = d->Start;
-      for (auto const &good: good_parts)
-      {
-	 memmove(current, good.first, good.second);
-	 current += good.second;
-      }
-      d->End = current;
-   }
-
-   if (d->isCommentedLine == true)
-   {
-      // deal with a buffer containing only comments
-      // or an (unfinished) comment at the end
-      if (good_parts.empty() == true)
-	 d->End = d->Start;
-      else
-	 d->Start = d->End;
-   }
-   else
-   {
-      // the buffer was all comment, but ended with the buffer
-      if (good_parts.empty() == true && good_start >= d->End)
-	 d->End = d->Start;
-      else
-	 d->Start = d->End;
-   }
-}
-bool pkgTagFile::Fill()
-{
-   unsigned long long const EndSize = d->End - d->Start;
-   if (EndSize != 0)
-   {
-      memmove(d->Buffer,d->Start,EndSize);
-      d->Start = d->End = d->Buffer + EndSize;
-   }
-   else
-      d->Start = d->End = d->Buffer;
-
-   unsigned long long Actual = 0;
-   while (d->Done == false && d->Size > (Actual + 1))
-   {
-      if (FillBuffer(d) == false)
-	 return false;
-      if ((d->Flags & pkgTagFile::SUPPORT_COMMENTS) != 0)
-	 RemoveCommentsFromBuffer(d);
-      Actual = d->End - d->Buffer;
-   }
-   d->Start = d->Buffer;
-
-   if (d->Done == true)
-   {
-      if (EndSize <= 3 && Actual == 0)
-	 return false;
-      if (d->Size - (d->End - d->Buffer) < 4)
-	 return true;
-
-      // Append a double new line if one does not exist
-      unsigned int LineCount = 0;
-      for (const char *E = d->End - 1; E - d->End < 6 && (*E == '\n' || *E == '\r'); E--)
-	 if (*E == '\n')
-	    ++LineCount;
-      if (LineCount < 2)
-      {
-	 if (static_cast<unsigned long long>(d->End - d->Buffer) >= d->Size)
-	    Resize(d->Size + 3);
-	 for (; LineCount < 2; ++LineCount)
-	    *d->End++ = '\n';
-      }
-   }
+   d->iOffset += tagSize;
    return true;
 }
 									/*}}}*/
@@ -428,43 +182,16 @@ bool pkgTagFile::Fill()
    that is there */
 bool pkgTagFile::Jump(pkgTagSection &Tag,unsigned long long Offset)
 {
-   if ((d->Flags & pkgTagFile::SUPPORT_COMMENTS) == 0 &&
-   // We are within a buffer space of the next hit..
-	 Offset >= d->iOffset && d->iOffset + (d->End - d->Start) > Offset)
-   {
-      unsigned long long Dist = Offset - d->iOffset;
-      d->Start += Dist;
-      d->iOffset += Dist;
-      // if we have seen the end, don't ask for more
-      if (d->Done == true)
-	 return Tag.Scan(d->Start, d->End - d->Start);
-      else
-	 return Step(Tag);
-   }
+   unsigned int Size(d->Map->Size());
+   if (Offset >= Size)
+      return false;
 
    // Reposition and reload..
    d->iOffset = Offset;
    d->Done = false;
-   if (d->Fd->Seek(Offset) == false)
-      return false;
-   d->End = d->Start = d->Buffer;
-   d->isCommentedLine = false;
-   d->chunks.clear();
-
-   if (Fill() == false)
-      return false;
+   d->Start = d->Buffer + d->iOffset;
 
-   if (Tag.Scan(d->Start, d->End - d->Start) == true)
-      return true;
-   
-   // This appends a double new line (for the real eof handling)
-   if (Fill() == false)
-      return false;
-   
-   if (Tag.Scan(d->Start, d->End - d->Start, false) == false)
-      return _error->Error(_("Unable to parse package file %s (%d)"),d->Fd->Name().c_str(), 2);
-   
-   return true;
+   return Step(Tag);
 }
 									/*}}}*/
 // pkgTagSection::pkgTagSection - Constructor				/*{{{*/
@@ -480,32 +207,20 @@ pkgTagSection::pkgTagSection()
 APT_IGNORE_DEPRECATED_POP
 									/*}}}*/
 // TagSection::Scan - Scan for the end of the header information	/*{{{*/
-bool pkgTagSection::Scan(const char *Start,unsigned long MaxLength, bool const Restart)
+bool pkgTagSection::Scan(const char *Start,unsigned long MaxLength,bool const SupportComments)
 {
    Section = Start;
    const char *End = Start + MaxLength;
 
-   if (Restart == false && d->Tags.empty() == false)
+   Stop = Section;
+   if (d->Tags.empty() == false)
    {
-      Stop = Section + d->Tags.back().StartTag;
-      if (End <= Stop)
-	 return false;
-      Stop = (const char *)memchr(Stop,'\n',End - Stop);
-      if (Stop == NULL)
-	 return false;
-      ++Stop;
-   }
-   else
-   {
-      Stop = Section;
-      if (d->Tags.empty() == false)
-      {
-	 memset(&AlphaIndexes, 0, sizeof(AlphaIndexes));
-	 memset(&BetaIndexes, 0, sizeof(BetaIndexes));
-	 d->Tags.clear();
-      }
-      d->Tags.reserve(0x100);
+      memset(&AlphaIndexes, 0, sizeof(AlphaIndexes));
+      memset(&BetaIndexes, 0, sizeof(BetaIndexes));
+      d->Tags.clear();
    }
+   d->Tags.reserve(0x100);
+
    unsigned int TagCount = d->Tags.size();
 
    if (Stop == 0)
@@ -517,12 +232,12 @@ bool pkgTagSection::Scan(const char *Start,unsigned long MaxLength, bool const R
    unsigned int lastTagHash = 0;
    while (Stop < End)
    {
-      TrimRecord(true,End);
+      TrimRecord(true,End,SupportComments);
 
       // this can happen when TrimRecord trims away the entire Record
       // (e.g. because it just contains comments)
       if(Stop == End)
-         return true;
+         goto end;
 
       // Start a new index and add it to the hash
       if (isspace_ascii(Stop[0]) == 0)
@@ -561,17 +276,17 @@ bool pkgTagSection::Scan(const char *Start,unsigned long MaxLength, bool const R
 	 // find the beginning of the value
 	 Stop = Colon + 1;
 	 for (; Stop < End && isspace_ascii(*Stop) != 0; ++Stop)
-	    if (*Stop == '\n' && Stop[1] != ' ')
+	    if (*Stop == '\n' && (Stop+1 == End || Stop[1] != ' '))
 	       break;
-	 if (Stop >= End)
-	    return false;
 	 lastTagData.StartValue = Stop - Section;
       }
 
       Stop = (const char *)memchr(Stop,'\n',End - Stop);
 
-      if (Stop == 0)
-	 return false;
+      if (Stop == 0) {
+         Stop = End;
+         goto end;
+      }
 
       for (; Stop+1 < End && Stop[1] == '\r'; Stop++)
          /* nothing */
@@ -579,7 +294,7 @@ bool pkgTagSection::Scan(const char *Start,unsigned long MaxLength, bool const R
 
       // Double newline marks the end of the record
       if (Stop+1 < End && Stop[1] == '\n')
-      {
+      end: {
 	 if (lastTagData.EndTag != 0)
 	 {
 	    if (lastTagKey != Key::Unknown) {
@@ -592,26 +307,32 @@ bool pkgTagSection::Scan(const char *Start,unsigned long MaxLength, bool const R
 	    d->Tags.push_back(lastTagData);
 	 }
 
+         if (d->Tags.empty())
+            return false;
+
 	 pkgTagSectionPrivate::TagData const td(Stop - Section);
 	 d->Tags.push_back(td);
-	 TrimRecord(false,End);
+	 TrimRecord(false,End,SupportComments);
 	 return true;
       }
       
       Stop++;
    }
 
-   return false;
+   goto end;
 }
 									/*}}}*/
 // TagSection::TrimRecord - Trim off any garbage before/after a record	/*{{{*/
 // ---------------------------------------------------------------------
 /* There should be exactly 2 newline at the end of the record, no more. */
-void pkgTagSection::TrimRecord(bool BeforeRecord, const char*& End)
-{
-   if (BeforeRecord == true)
-      return;
-   for (; Stop < End && (Stop[0] == '\n' || Stop[0] == '\r'); Stop++);
+void pkgTagSection::TrimRecord(bool BeforeRecord, const char*& End, bool SupportComments)
+{ trim:
+   if (BeforeRecord == false)
+      for (; Stop < End && (Stop[0] == '\n' || Stop[0] == '\r'); Stop++);
+   if (SupportComments && Stop < End && Stop[0] == '#') {
+      Stop = (const char*) memchr(Stop,'\n',End-Stop) ?: End;
+      goto trim;
+   }
 }
 									/*}}}*/
 // TagSection::Trim - Trim off any trailing garbage			/*{{{*/
@@ -621,7 +342,7 @@ void pkgTagSection::Trim()
 {
    for (; Stop > Section + 2 && (Stop[-2] == '\n' || Stop[-2] == '\r'); Stop--);
 }
-									/*}}}*/
+
 // TagSection::Exists - return True if a tag exists			/*{{{*/
 bool pkgTagSection::Exists(StringView Tag) const
 {
@@ -676,7 +397,7 @@ bool pkgTagSection::FindInternal(unsigned int Pos, const char *&Start,
    if (unlikely(Start > End))
       return _error->Error("Internal parsing error");
 
-   for (; isspace_ascii(End[-1]) != 0 && End > Start; --End);
+   for (; End > Start && isspace_ascii(End[-1]) != 0; --End);
 
    return true;
 }
@@ -1062,14 +783,6 @@ bool pkgTagSection::Write(FileFd &File, char const * const * const Order, std::v
 }
 									/*}}}*/
 
-void pkgUserTagSection::TrimRecord(bool /*BeforeRecord*/, const char* &End)/*{{{*/
-{
-   for (; Stop < End && (Stop[0] == '\n' || Stop[0] == '\r' || Stop[0] == '#'); Stop++)
-      if (Stop[0] == '#')
-	 Stop = (const char*) memchr(Stop,'\n',End-Stop);
-}
-									/*}}}*/
-
 #include "tagfile-order.c"
 
 // TFRewrite - Rewrite a control record					/*{{{*/
diff --git a/apt-pkg/tagfile.h b/apt-pkg/tagfile.h
index e02e7332e..04e20a5e8 100644
--- a/apt-pkg/tagfile.h
+++ b/apt-pkg/tagfile.h
@@ -138,11 +138,11 @@ class pkgTagSection
     * @return \b true if section end was found, \b false otherwise.
     *  Beware that internal state will be inconsistent if \b false is returned!
     */
-   APT_MUSTCHECK bool Scan(const char *Start, unsigned long MaxLength, bool const Restart = true);
+   APT_MUSTCHECK bool Scan(const char *Start, unsigned long MaxLength, bool const SupportComments = true);
 
    inline unsigned long size() const {return Stop - Section;};
    void Trim();
-   virtual void TrimRecord(bool BeforeRecord, const char* &End);
+   void TrimRecord(bool BeforeRecord, const char* &End, bool SupportComments);
 
    /** \brief amount of Tags in the current section
     *
@@ -187,11 +187,6 @@ class pkgTagSection
 };
 
 
-class APT_DEPRECATED_MSG("Use pkgTagFile with the SUPPORT_COMMENTS flag instead") pkgUserTagSection : public pkgTagSection
-{
-   virtual void TrimRecord(bool BeforeRecord, const char* &End) APT_OVERRIDE;
-};
-
 /** \class pkgTagFile reads and prepares a deb822 formatted file for parsing
  * via #pkgTagSection. The default mode tries to be as fast as possible and
  * assumes perfectly valid (machine generated) files like Packages. Support
@@ -200,10 +195,6 @@ class pkgTagFile
 {
    pkgTagFilePrivate * const d;
 
-   APT_HIDDEN bool Fill();
-   APT_HIDDEN bool Resize();
-   APT_HIDDEN bool Resize(unsigned long long const newSize);
-
 public:
 
    bool Step(pkgTagSection &Section);
@@ -216,11 +207,11 @@ public:
       SUPPORT_COMMENTS = 1 << 0,
    };
 
-   void Init(FileFd * const F, pkgTagFile::Flags const Flags, unsigned long long Size = 32*1024);
-   void Init(FileFd * const F,unsigned long long const Size = 32*1024);
+   void Init(FileFd * const F, pkgTagFile::Flags const Flags);
+   void Init(FileFd * const F);
 
-   pkgTagFile(FileFd * const F, pkgTagFile::Flags const Flags, unsigned long long Size = 32*1024);
-   pkgTagFile(FileFd * const F,unsigned long long Size = 32*1024);
+   pkgTagFile(FileFd * const F, pkgTagFile::Flags const Flags);
+   pkgTagFile(FileFd * const F);
    virtual ~pkgTagFile();
 };
 
diff --git a/apt-private/private-source.cc b/apt-private/private-source.cc
index e9ec7c6cf..923ea4db1 100644
--- a/apt-private/private-source.cc
+++ b/apt-private/private-source.cc
@@ -525,7 +525,7 @@ bool DoSource(CommandLine &CmdL)
 	 strprintf(S, "%s %s %s",
 	       _config->Find("Dir::Bin::dpkg-source","dpkg-source").c_str(),
 	       sourceopts.c_str(), D.Dsc.c_str());
-	 if (system(S.c_str()) != 0)
+	 if (RunCmd(S.c_str()) != 0)
 	 {
 	    _error->Error(_("Unpack command '%s' failed.\n"), S.c_str());
 	    if (SaidCheckIfDpkgDev == false)
@@ -558,7 +558,7 @@ bool DoSource(CommandLine &CmdL)
 	       _config->Find("Dir::Bin::dpkg-buildpackage","dpkg-buildpackage").c_str(),
 	       buildopts.c_str());
 
-	 if (system(S.c_str()) != 0)
+	 if (RunCmd(S.c_str()) != 0)
 	 {
 	    _error->Error(_("Build command '%s' failed.\n"), S.c_str());
 	    continue;
diff --git a/debian/libapt-pkg5.0.symbols b/debian/libapt-pkg5.0.symbols
index 030ad35e4..2f625d2e1 100644
--- a/debian/libapt-pkg5.0.symbols
+++ b/debian/libapt-pkg5.0.symbols
@@ -1294,7 +1294,6 @@ libapt-pkg.so.5.0 libapt-pkg5.0 #MINVER#
  (c++)"pkgTagSection::Tag::Rename(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)@APTPKG_5.0" 1.1~exp9
  (c++)"pkgTagSection::Tag::Rewrite(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)@APTPKG_5.0" 1.1~exp9
  (c++)"pkgTagSection::Write(FileFd&, char const* const*, std::vector<pkgTagSection::Tag, std::allocator<pkgTagSection::Tag> > const&) const@APTPKG_5.0" 1.1~exp9
- (c++)"pkgUserTagSection::TrimRecord(bool, char const*&)@APTPKG_5.0" 1.1~exp9
  (c++)"pkgVersioningSystem::~pkgVersioningSystem()@APTPKG_5.0" 1.1~exp9
  (c++)"SigVerify::~SigVerify()@APTPKG_5.0" 1.1~exp9
  (c++)"SigVerify::SigVerify()@APTPKG_5.0" 1.1~exp9
@@ -1323,7 +1322,6 @@ libapt-pkg.so.5.0 libapt-pkg5.0 #MINVER#
  (c++)"typeinfo for pkgProblemResolver@APTPKG_5.0" 1.1~exp9
  (c++)"typeinfo for pkgRecords@APTPKG_5.0" 1.1~exp9
  (c++)"typeinfo for pkgSourceList@APTPKG_5.0" 1.1~exp9
- (c++)"typeinfo for pkgUserTagSection@APTPKG_5.0" 1.1~exp9
  (c++)"typeinfo for SigVerify@APTPKG_5.0" 1.1~exp9
  (c++)"typeinfo for TranslationsCopy@APTPKG_5.0" 1.1~exp9
  (c++)"typeinfo name for APT::PackageUniverse@APTPKG_5.0" 1.1~exp9
@@ -1347,7 +1345,6 @@ libapt-pkg.so.5.0 libapt-pkg5.0 #MINVER#
  (c++)"typeinfo name for pkgProblemResolver@APTPKG_5.0" 1.1~exp9
  (c++)"typeinfo name for pkgRecords@APTPKG_5.0" 1.1~exp9
  (c++)"typeinfo name for pkgSourceList@APTPKG_5.0" 1.1~exp9
- (c++)"typeinfo name for pkgUserTagSection@APTPKG_5.0" 1.1~exp9
  (c++)"typeinfo name for SigVerify@APTPKG_5.0" 1.1~exp9
  (c++)"typeinfo name for TranslationsCopy@APTPKG_5.0" 1.1~exp9
  (c++)"URI::ArchiveOnly(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)@APTPKG_5.0" 1.1~exp9
@@ -1372,7 +1369,6 @@ libapt-pkg.so.5.0 libapt-pkg5.0 #MINVER#
  (c++)"vtable for pkgProblemResolver@APTPKG_5.0" 1.1~exp9
  (c++)"vtable for pkgRecords@APTPKG_5.0" 1.1~exp9
  (c++)"vtable for pkgSourceList@APTPKG_5.0" 1.1~exp9
- (c++)"vtable for pkgUserTagSection@APTPKG_5.0" 1.1~exp9
  (c++)"vtable for SigVerify@APTPKG_5.0" 1.1~exp9
  (c++)"vtable for TranslationsCopy@APTPKG_5.0" 1.1~exp9
 ### dpkg selection state changer & general dpkg interfacing
diff --git a/ftparchive/byhash.cc b/ftparchive/byhash.cc
index 354d089c3..358fd27b4 100644
--- a/ftparchive/byhash.cc
+++ b/ftparchive/byhash.cc
@@ -17,6 +17,8 @@
 #include <unistd.h>
 #include <sys/stat.h>
 
+#define st_mtim st_mtimespec
+
 #include <apt-pkg/fileutl.h>
 #include <apt-pkg/hashes.h>
 #include "byhash.h"
diff --git a/ftparchive/cachedb.cc b/ftparchive/cachedb.cc
index 868029abd..930bb5f94 100644
--- a/ftparchive/cachedb.cc
+++ b/ftparchive/cachedb.cc
@@ -22,6 +22,7 @@
 #include <apt-pkg/debfile.h>
 #include <apt-pkg/gpgv.h>
 #include <apt-pkg/hashes.h>
+#include <apt-pkg/missing.h>
 
 #include <netinet/in.h>       // htonl, etc
 #include <ctype.h>
diff --git a/methods/CMakeLists.txt b/methods/CMakeLists.txt
index a74c2ce07..128c61005 100644
--- a/methods/CMakeLists.txt
+++ b/methods/CMakeLists.txt
@@ -4,8 +4,9 @@ add_executable(copy copy.cc)
 add_executable(store store.cc)
 add_executable(gpgv gpgv.cc)
 add_executable(cdrom cdrom.cc)
-add_executable(http http.cc http_main.cc rfc2553emu.cc connect.cc basehttp.cc)
-add_executable(mirror mirror.cc http.cc rfc2553emu.cc connect.cc basehttp.cc)
+#add_executable(http http.cc http_main.cc rfc2553emu.cc connect.cc basehttp.cc)
+add_executable(http http.cc rfc2553emu.cc connect.cc basehttp.cc)
+#add_executable(mirror mirror.cc http.cc rfc2553emu.cc connect.cc basehttp.cc)
 add_executable(https https.cc basehttp.cc)
 add_executable(ftp ftp.cc rfc2553emu.cc connect.cc)
 add_executable(rred rred.cc)
@@ -20,15 +21,16 @@ target_link_libraries(copy apt-pkg)
 target_link_libraries(store apt-pkg)
 target_link_libraries(gpgv apt-pkg)
 target_link_libraries(cdrom apt-pkg)
-target_link_libraries(http apt-pkg)
-target_link_libraries(mirror apt-pkg ${RESOLV_LIBRARIES})
+#target_link_libraries(http apt-pkg)
+#target_link_libraries(mirror apt-pkg ${RESOLV_LIBRARIES})
 target_link_libraries(https apt-pkg ${CURL_LIBRARIES})
+target_link_libraries(http apt-pkg lockdown "-framework CoreFoundation" "-framework CFNetwork" "-framework SystemConfiguration")
 target_link_libraries(ftp apt-pkg)
 target_link_libraries(rred apt-pkg)
 target_link_libraries(rsh apt-pkg)
 
 # Install the library
-install(TARGETS file copy store gpgv cdrom http https ftp rred rsh mirror
+install(TARGETS file copy store gpgv cdrom http https ftp rred rsh #mirror
         RUNTIME DESTINATION ${CMAKE_INSTALL_LIBEXECDIR}/apt/methods)
 
 add_slaves(${CMAKE_INSTALL_LIBEXECDIR}/apt/methods store gzip lzma bzip2 xz)
diff --git a/methods/http.cc b/methods/http.cc
index 9f5959548..72e813cb9 100644
--- a/methods/http.cc
+++ b/methods/http.cc
@@ -25,7 +25,7 @@
    ##################################################################### */
 									/*}}}*/
 // Include Files							/*{{{*/
-#include <config.h>
+// #include <config.h>
 
 #include <apt-pkg/fileutl.h>
 #include <apt-pkg/configuration.h>
@@ -39,6 +39,7 @@
 #include <stdlib.h>
 #include <sys/select.h>
 #include <cstring>
+#include <sys/sysctl.h>
 #include <sys/stat.h>
 #include <sys/time.h>
 #include <unistd.h>
@@ -52,956 +53,503 @@
 #include "connect.h"
 #include "http.h"
 
-#include <apti18n.h>
+// #include <apti18n.h>
+
+#include <netdb.h>
+#include <dlfcn.h>
+#include <lockdown.h>
+#include <CoreFoundation/CoreFoundation.h>
+#include <CFNetwork/CFNetwork.h>
+extern "C" CFDictionaryRef SCDynamicStoreCopyProxies(void *);
 									/*}}}*/
 using namespace std;
 
-unsigned long long CircleBuf::BwReadLimit=0;
-unsigned long long CircleBuf::BwTickReadData=0;
-struct timeval CircleBuf::BwReadTick={0,0};
-const unsigned int CircleBuf::BW_HZ=10;
-
-// CircleBuf::CircleBuf - Circular input buffer				/*{{{*/
-// ---------------------------------------------------------------------
-/* */
-CircleBuf::CircleBuf(HttpMethod const * const Owner, unsigned long long Size)
-   : Size(Size), Hash(NULL), TotalWriten(0)
-{
-   Buf = new unsigned char[Size];
-   Reset();
-
-   CircleBuf::BwReadLimit = Owner->ConfigFindI("Dl-Limit", 0) * 1024;
-}
-									/*}}}*/
-// CircleBuf::Reset - Reset to the default state			/*{{{*/
-// ---------------------------------------------------------------------
-/* */
-void CircleBuf::Reset()
-{
-   InP = 0;
-   OutP = 0;
-   StrPos = 0;
-   TotalWriten = 0;
-   MaxGet = (unsigned long long)-1;
-   OutQueue = string();
-   if (Hash != NULL)
-   {
-      delete Hash;
-      Hash = NULL;
-   }
-}
-									/*}}}*/
-// CircleBuf::Read - Read from a FD into the circular buffer		/*{{{*/
-// ---------------------------------------------------------------------
-/* This fills up the buffer with as much data as is in the FD, assuming it
-   is non-blocking.. */
-bool CircleBuf::Read(int Fd)
-{
-   while (1)
-   {
-      // Woops, buffer is full
-      if (InP - OutP == Size)
-	 return true;
-
-      // what's left to read in this tick
-      unsigned long long const BwReadMax = CircleBuf::BwReadLimit/BW_HZ;
-
-      if(CircleBuf::BwReadLimit) {
-	 struct timeval now;
-	 gettimeofday(&now,0);
-
-	 unsigned long long d = (now.tv_sec-CircleBuf::BwReadTick.tv_sec)*1000000 +
-	    now.tv_usec-CircleBuf::BwReadTick.tv_usec;
-	 if(d > 1000000/BW_HZ) {
-	    CircleBuf::BwReadTick = now;
-	    CircleBuf::BwTickReadData = 0;
-	 } 
-	 
-	 if(CircleBuf::BwTickReadData >= BwReadMax) {
-	    usleep(1000000/BW_HZ);
-	    return true;
-	 }
+#define _(str) str
+
+CFStringRef Firmware_;
+const char *Machine_;
+CFStringRef UniqueID_;
+
+void CfrsError(const char *name, CFReadStreamRef rs) {
+   CFStreamError se = CFReadStreamGetError(rs);
+
+   if (se.domain == kCFStreamErrorDomainCustom) {
+   } else if (se.domain == kCFStreamErrorDomainPOSIX) {
+      _error->Error("POSIX: %s", strerror(se.error));
+   } else if (se.domain == kCFStreamErrorDomainMacOSStatus) {
+      _error->Error("MacOSStatus: %d", (int)se.error);
+   } else if (se.domain == kCFStreamErrorDomainNetDB) {
+      _error->Error("NetDB: %s %s", name, gai_strerror(se.error));
+   } else if (se.domain == kCFStreamErrorDomainMach) {
+      _error->Error("Mach: %d", (int)se.error);
+   } else if (se.domain == kCFStreamErrorDomainHTTP) {
+      switch (se.error) {
+         case kCFStreamErrorHTTPParseFailure:
+            _error->Error("Parse failure");
+         break;
+
+         case kCFStreamErrorHTTPRedirectionLoop:
+            _error->Error("Redirection loop");
+         break;
+
+         case kCFStreamErrorHTTPBadURL:
+            _error->Error("Bad URL");
+         break;
+
+         default:
+            _error->Error("Unknown HTTP error: %d", (int)se.error);
+         break;
       }
-
-      // Write the buffer segment
-      ssize_t Res;
-      if(CircleBuf::BwReadLimit) {
-	 Res = read(Fd,Buf + (InP%Size), 
-		    BwReadMax > LeftRead() ? LeftRead() : BwReadMax);
-      } else
-	 Res = read(Fd,Buf + (InP%Size),LeftRead());
-      
-      if(Res > 0 && BwReadLimit > 0) 
-	 CircleBuf::BwTickReadData += Res;
-    
-      if (Res == 0)
-	 return false;
-      if (Res < 0)
-      {
-	 if (errno == EAGAIN)
-	    return true;
-	 return false;
-      }
-
-      if (InP == 0)
-	 gettimeofday(&Start,0);
-      InP += Res;
+   } else if (se.domain == kCFStreamErrorDomainSOCKS) {
+      _error->Error("SOCKS: %d", (int)se.error);
+   } else if (se.domain == kCFStreamErrorDomainSystemConfiguration) {
+      _error->Error("SystemConfiguration: %d", (int)se.error);
+   } else if (se.domain == kCFStreamErrorDomainSSL) {
+      _error->Error("SSL: %d", (int)se.error);
+   } else {
+      _error->Error("Domain #%d: %d", (int)se.domain, (int)se.error);
    }
 }
-									/*}}}*/
-// CircleBuf::Read - Put the string into the buffer			/*{{{*/
-// ---------------------------------------------------------------------
-/* This will hold the string in and fill the buffer with it as it empties */
-bool CircleBuf::Read(string const &Data)
-{
-   OutQueue.append(Data);
-   FillOut();
-   return true;
-}
-									/*}}}*/
-// CircleBuf::FillOut - Fill the buffer from the output queue		/*{{{*/
-// ---------------------------------------------------------------------
-/* */
-void CircleBuf::FillOut()
-{
-   if (OutQueue.empty() == true)
-      return;
-   while (1)
-   {
-      // Woops, buffer is full
-      if (InP - OutP == Size)
-	 return;
-      
-      // Write the buffer segment
-      unsigned long long Sz = LeftRead();
-      if (OutQueue.length() - StrPos < Sz)
-	 Sz = OutQueue.length() - StrPos;
-      memcpy(Buf + (InP%Size),OutQueue.c_str() + StrPos,Sz);
-      
-      // Advance
-      StrPos += Sz;
-      InP += Sz;
-      if (OutQueue.length() == StrPos)
-      {
-	 StrPos = 0;
-	 OutQueue = "";
-	 return;
-      }
+
+unsigned long TimeOut = 120;
+
+static const CFOptionFlags kNetworkEvents =
+   kCFStreamEventOpenCompleted |
+   kCFStreamEventHasBytesAvailable |
+   kCFStreamEventEndEncountered |
+   kCFStreamEventErrorOccurred |
+0;
+
+static void CFReadStreamCallback(CFReadStreamRef stream, CFStreamEventType event, void *arg) {
+   switch (event) {
+      case kCFStreamEventOpenCompleted:
+      break;
+
+      case kCFStreamEventHasBytesAvailable:
+      case kCFStreamEventEndEncountered:
+         *reinterpret_cast<int *>(arg) = 1;
+         CFRunLoopStop(CFRunLoopGetCurrent());
+      break;
+
+      case kCFStreamEventErrorOccurred:
+         *reinterpret_cast<int *>(arg) = -1;
+         CFRunLoopStop(CFRunLoopGetCurrent());
+      break;
    }
 }
-									/*}}}*/
-// CircleBuf::Write - Write from the buffer into a FD			/*{{{*/
-// ---------------------------------------------------------------------
-/* This empties the buffer into the FD. */
-bool CircleBuf::Write(int Fd)
-{
-   while (1)
-   {
-      FillOut();
-      
-      // Woops, buffer is empty
-      if (OutP == InP)
-	 return true;
-      
-      if (OutP == MaxGet)
-	 return true;
-      
-      // Write the buffer segment
-      ssize_t Res;
-      Res = write(Fd,Buf + (OutP%Size),LeftWrite());
 
-      if (Res == 0)
-	 return false;
-      if (Res < 0)
-      {
-	 if (errno == EAGAIN)
-	    return true;
-	 
-	 return false;
-      }
+/* http://lists.apple.com/archives/Macnetworkprog/2006/Apr/msg00014.html */
+int CFReadStreamOpen(CFReadStreamRef stream, double timeout) {
+   CFStreamClientContext context;
+   int value(0);
 
-      TotalWriten += Res;
-      
-      if (Hash != NULL)
-	 Hash->Add(Buf + (OutP%Size),Res);
-      
-      OutP += Res;
+   memset(&context, 0, sizeof(context));
+   context.info = &value;
+
+   if (CFReadStreamSetClient(stream, kNetworkEvents, CFReadStreamCallback, &context)) {
+      CFReadStreamScheduleWithRunLoop(stream, CFRunLoopGetCurrent(), kCFRunLoopCommonModes);
+      if (CFReadStreamOpen(stream))
+         CFRunLoopRunInMode(kCFRunLoopDefaultMode, timeout, false);
+      else
+         value = -1;
+      CFReadStreamSetClient(stream, kCFStreamEventNone, NULL, NULL);
    }
+
+   return value;
 }
-									/*}}}*/
-// CircleBuf::WriteTillEl - Write from the buffer to a string		/*{{{*/
+
+// HttpMethod::SendReq - Send the HTTP request				/*{{{*/
 // ---------------------------------------------------------------------
-/* This copies till the first empty line */
-bool CircleBuf::WriteTillEl(string &Data,bool Single)
+/* This places the http request in the outbound buffer */
+void HttpMethod::SendReq(FetchItem *Itm)
 {
-   // We cheat and assume it is unneeded to have more than one buffer load
-   for (unsigned long long I = OutP; I < InP; I++)
-   {      
-      if (Buf[I%Size] != '\n')
-	 continue;
-      ++I;
-      
-      if (Single == false)
-      {
-         if (I < InP  && Buf[I%Size] == '\r')
-            ++I;
-         if (I >= InP || Buf[I%Size] != '\n')
-            continue;
-         ++I;
-      }
-      
-      Data = "";
-      while (OutP < I)
-      {
-	 unsigned long long Sz = LeftWrite();
-	 if (Sz == 0)
-	    return false;
-	 if (I - OutP < Sz)
-	    Sz = I - OutP;
-	 Data += string((char *)(Buf + (OutP%Size)),Sz);
-	 OutP += Sz;
-      }
-      return true;
-   }      
-   return false;
 }
 									/*}}}*/
-// CircleBuf::Stats - Print out stats information			/*{{{*/
-// ---------------------------------------------------------------------
-/* */
-void CircleBuf::Stats()
+std::unique_ptr<ServerState> HttpMethod::CreateServerState(URI const &uri)/*{{{*/
 {
-   if (InP == 0)
-      return;
-   
-   struct timeval Stop;
-   gettimeofday(&Stop,0);
-/*   float Diff = Stop.tv_sec - Start.tv_sec + 
-             (float)(Stop.tv_usec - Start.tv_usec)/1000000;
-   clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
+   return NULL;
 }
 									/*}}}*/
-CircleBuf::~CircleBuf()
-{
-   delete [] Buf;
-   delete Hash;
-}
-
-// HttpServerState::HttpServerState - Constructor			/*{{{*/
-HttpServerState::HttpServerState(URI Srv,HttpMethod *Owner) : ServerState(Srv, Owner), In(Owner, 64*1024), Out(Owner, 4*1024)
+void HttpMethod::RotateDNS()						/*{{{*/
 {
-   TimeOut = Owner->ConfigFindI("Timeout", TimeOut);
-   Reset();
 }
 									/*}}}*/
-// HttpServerState::Open - Open a connection to the server		/*{{{*/
-// ---------------------------------------------------------------------
-/* This opens a connection to the server. */
-static bool TalkToSocksProxy(int const ServerFd, std::string const &Proxy,
-      char const * const type, bool const ReadWrite, uint8_t * const ToFrom,
-      unsigned int const Size, unsigned int const Timeout)
-{
-   if (WaitFd(ServerFd, ReadWrite, Timeout) == false)
-      return _error->Error("Waiting for the SOCKS proxy %s to %s timed out", URI::SiteOnly(Proxy).c_str(), type);
-   if (ReadWrite == false)
-   {
-      if (FileFd::Read(ServerFd, ToFrom, Size) == false)
-	 return _error->Error("Reading the %s from SOCKS proxy %s failed", type, URI::SiteOnly(Proxy).c_str());
-   }
-   else
-   {
-      if (FileFd::Write(ServerFd, ToFrom, Size) == false)
-	 return _error->Error("Writing the %s to SOCKS proxy %s failed", type, URI::SiteOnly(Proxy).c_str());
-   }
-   return true;
-}
-bool HttpServerState::Open()
+BaseHttpMethod::DealWithHeadersResult HttpMethod::DealWithHeaders(FetchResult &Res, RequestState &Req)/*{{{*/
 {
-   // Use the already open connection if possible.
-   if (ServerFd != -1)
-      return true;
-   
-   Close();
-   In.Reset();
-   Out.Reset();
-   Persistent = true;
-   
-   // Determine the proxy setting
-   AutoDetectProxy(ServerName);
-   string SpecificProxy = Owner->ConfigFind("Proxy::" + ServerName.Host, "");
-   if (!SpecificProxy.empty())
-   {
-	   if (SpecificProxy == "DIRECT")
-		   Proxy = "";
-	   else
-		   Proxy = SpecificProxy;
-   }
-   else
-   {
-	   string DefProxy = Owner->ConfigFind("Proxy", "");
-	   if (!DefProxy.empty())
-	   {
-		   Proxy = DefProxy;
-	   }
-	   else
-	   {
-		   char* result = getenv("http_proxy");
-		   Proxy = result ? result : "";
-	   }
-   }
-   
-   // Parse no_proxy, a , separated list of domains
-   if (getenv("no_proxy") != 0)
-   {
-      if (CheckDomainList(ServerName.Host,getenv("no_proxy")) == true)
-	 Proxy = "";
-   }
+   auto ret = BaseHttpMethod::DealWithHeaders(Res, Req);
+   if (ret != BaseHttpMethod::FILE_IS_OPEN)
+      return ret;
+   if (Req.File.Open(Queue->DestFile, FileFd::WriteAny) == false)
+      return ERROR_NOT_FROM_SERVER;
 
-   if (Proxy.empty() == false)
-      Owner->AddProxyAuth(Proxy, ServerName);
+   FailFile = Queue->DestFile;
+   FailFile.c_str();   // Make sure we don't do a malloc in the signal handler
+   FailFd = Req.File.Fd();
+   FailTime = Req.Date;
 
-   if (Proxy.Access == "socks5h")
-   {
-      if (Connect(Proxy.Host, Proxy.Port, "socks", 1080, ServerFd, TimeOut, Owner) == false)
-	 return false;
-
-      /* We implement a very basic SOCKS5 client here complying mostly to RFC1928 expect
-       * for not offering GSSAPI auth which is a must (we only do no or user/pass auth).
-       * We also expect the SOCKS5 server to do hostname lookup (aka socks5h) */
-      std::string const ProxyInfo = URI::SiteOnly(Proxy);
-      Owner->Status(_("Connecting to %s (%s)"),"SOCKS5h proxy",ProxyInfo.c_str());
-      auto const Timeout = Owner->ConfigFindI("TimeOut", 120);
-      #define APT_WriteOrFail(TYPE, DATA, LENGTH) if (TalkToSocksProxy(ServerFd, ProxyInfo, TYPE, true, DATA, LENGTH, Timeout) == false) return false
-      #define APT_ReadOrFail(TYPE, DATA, LENGTH) if (TalkToSocksProxy(ServerFd, ProxyInfo, TYPE, false, DATA, LENGTH, Timeout) == false) return false
-      if (ServerName.Host.length() > 255)
-	 return _error->Error("Can't use SOCKS5h as hostname %s is too long!", ServerName.Host.c_str());
-      if (Proxy.User.length() > 255 || Proxy.Password.length() > 255)
-	 return _error->Error("Can't use user&pass auth as they are too long (%lu and %lu) for the SOCKS5!", Proxy.User.length(), Proxy.Password.length());
-      if (Proxy.User.empty())
-      {
-	 uint8_t greeting[] = { 0x05, 0x01, 0x00 };
-	 APT_WriteOrFail("greet-1", greeting, sizeof(greeting));
-      }
-      else
-      {
-	 uint8_t greeting[] = { 0x05, 0x02, 0x00, 0x02 };
-	 APT_WriteOrFail("greet-2", greeting, sizeof(greeting));
-      }
-      uint8_t greeting[2];
-      APT_ReadOrFail("greet back", greeting, sizeof(greeting));
-      if (greeting[0] != 0x05)
-	 return _error->Error("SOCKS proxy %s greets back with wrong version: %d", ProxyInfo.c_str(), greeting[0]);
-      if (greeting[1] == 0x00)
-	 ; // no auth has no method-dependent sub-negotiations
-      else if (greeting[1] == 0x02)
-      {
-	 if (Proxy.User.empty())
-	    return _error->Error("SOCKS proxy %s negotiated user&pass auth, but we had not offered it!", ProxyInfo.c_str());
-	 // user&pass auth sub-negotiations are defined by RFC1929
-	 std::vector<uint8_t> auth = {{ 0x01, static_cast<uint8_t>(Proxy.User.length()) }};
-	 std::copy(Proxy.User.begin(), Proxy.User.end(), std::back_inserter(auth));
-	 auth.push_back(static_cast<uint8_t>(Proxy.Password.length()));
-	 std::copy(Proxy.Password.begin(), Proxy.Password.end(), std::back_inserter(auth));
-	 APT_WriteOrFail("user&pass auth", auth.data(), auth.size());
-	 uint8_t authstatus[2];
-	 APT_ReadOrFail("auth report", authstatus, sizeof(authstatus));
-	 if (authstatus[0] != 0x01)
-	    return _error->Error("SOCKS proxy %s auth status response with wrong version: %d", ProxyInfo.c_str(), authstatus[0]);
-	 if (authstatus[1] != 0x00)
-	    return _error->Error("SOCKS proxy %s reported authorization failure: username or password incorrect? (%d)", ProxyInfo.c_str(), authstatus[1]);
-      }
-      else
-	 return _error->Error("SOCKS proxy %s greets back having not found a common authorization method: %d", ProxyInfo.c_str(), greeting[1]);
-      union { uint16_t * i; uint8_t * b; } portu;
-      uint16_t port = htons(static_cast<uint16_t>(ServerName.Port == 0 ? 80 : ServerName.Port));
-      portu.i = &port;
-      std::vector<uint8_t> request = {{ 0x05, 0x01, 0x00, 0x03, static_cast<uint8_t>(ServerName.Host.length()) }};
-      std::copy(ServerName.Host.begin(), ServerName.Host.end(), std::back_inserter(request));
-      request.push_back(portu.b[0]);
-      request.push_back(portu.b[1]);
-      APT_WriteOrFail("request", request.data(), request.size());
-      uint8_t response[4];
-      APT_ReadOrFail("first part of response", response, sizeof(response));
-      if (response[0] != 0x05)
-	 return _error->Error("SOCKS proxy %s response with wrong version: %d", ProxyInfo.c_str(), response[0]);
-      if (response[2] != 0x00)
-	 return _error->Error("SOCKS proxy %s has unexpected non-zero reserved field value: %d", ProxyInfo.c_str(), response[2]);
-      std::string bindaddr;
-      if (response[3] == 0x01) // IPv4 address
-      {
-	 uint8_t ip4port[6];
-	 APT_ReadOrFail("IPv4+Port of response", ip4port, sizeof(ip4port));
-	 portu.b[0] = ip4port[4];
-	 portu.b[1] = ip4port[5];
-	 port = ntohs(*portu.i);
-	 strprintf(bindaddr, "%d.%d.%d.%d:%d", ip4port[0], ip4port[1], ip4port[2], ip4port[3], port);
-      }
-      else if (response[3] == 0x03) // hostname
-      {
-	 uint8_t namelength;
-	 APT_ReadOrFail("hostname length of response", &namelength, 1);
-	 uint8_t hostname[namelength + 2];
-	 APT_ReadOrFail("hostname of response", hostname, sizeof(hostname));
-	 portu.b[0] = hostname[namelength];
-	 portu.b[1] = hostname[namelength + 1];
-	 port = ntohs(*portu.i);
-	 hostname[namelength] = '\0';
-	 strprintf(bindaddr, "%s:%d", hostname, port);
-      }
-      else if (response[3] == 0x04) // IPv6 address
-      {
-	 uint8_t ip6port[18];
-	 APT_ReadOrFail("IPv6+port of response", ip6port, sizeof(ip6port));
-	 portu.b[0] = ip6port[16];
-	 portu.b[1] = ip6port[17];
-	 port = ntohs(*portu.i);
-	 strprintf(bindaddr, "[%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X]:%d",
-	       ip6port[0], ip6port[1], ip6port[2], ip6port[3], ip6port[4], ip6port[5], ip6port[6], ip6port[7],
-	       ip6port[8], ip6port[9], ip6port[10], ip6port[11], ip6port[12], ip6port[13], ip6port[14], ip6port[15],
-	       port);
-      }
-      else
-	 return _error->Error("SOCKS proxy %s destination address is of unknown type: %d",
-	       ProxyInfo.c_str(), response[3]);
-      if (response[1] != 0x00)
-      {
-	 char const * errstr = nullptr;
-	 auto errcode = response[1];
-	 // Tor error reporting can be a bit arcane, lets try to detect & fix it up
-	 if (bindaddr == "0.0.0.0:0")
-	 {
-	    auto const lastdot = ServerName.Host.rfind('.');
-	    if (lastdot == std::string::npos || ServerName.Host.substr(lastdot) != ".onion")
-	       ;
-	    else if (errcode == 0x01)
-	    {
-	       auto const prevdot = ServerName.Host.rfind('.', lastdot - 1);
-	       if (lastdot == 16 && prevdot == std::string::npos)
-		  ; // valid .onion address
-	       else if (prevdot != std::string::npos && (lastdot - prevdot) == 17)
-		  ; // valid .onion address with subdomain(s)
-	       else
-	       {
-		  errstr = "Invalid hostname: onion service name must be 16 characters long";
-		  Owner->SetFailReason("SOCKS");
-	       }
-	    }
-	    // in all likelihood the service is either down or the address has
-	    // a typo and so "Host unreachable" is the better understood error
-	    // compared to the technically correct "TLL expired".
-	    else if (errcode == 0x06)
-	       errcode = 0x04;
-	 }
-	 if (errstr == nullptr)
-	 {
-	    switch (errcode)
-	    {
-	       case 0x01: errstr = "general SOCKS server failure"; Owner->SetFailReason("SOCKS"); break;
-	       case 0x02: errstr = "connection not allowed by ruleset"; Owner->SetFailReason("SOCKS"); break;
-	       case 0x03: errstr = "Network unreachable"; Owner->SetFailReason("ConnectionTimedOut"); break;
-	       case 0x04: errstr = "Host unreachable"; Owner->SetFailReason("ConnectionTimedOut"); break;
-	       case 0x05: errstr = "Connection refused"; Owner->SetFailReason("ConnectionRefused"); break;
-	       case 0x06: errstr = "TTL expired"; Owner->SetFailReason("Timeout"); break;
-	       case 0x07: errstr = "Command not supported"; Owner->SetFailReason("SOCKS"); break;
-	       case 0x08: errstr = "Address type not supported"; Owner->SetFailReason("SOCKS"); break;
-	       default: errstr = "Unknown error"; Owner->SetFailReason("SOCKS"); break;
-	    }
-	 }
-	 return _error->Error("SOCKS proxy %s could not connect to %s (%s) due to: %s (%d)",
-	       ProxyInfo.c_str(), ServerName.Host.c_str(), bindaddr.c_str(), errstr, response[1]);
-      }
-      else if (Owner->DebugEnabled())
-	 ioprintf(std::clog, "http: SOCKS proxy %s connection established to %s (%s)\n",
-	       ProxyInfo.c_str(), ServerName.Host.c_str(), bindaddr.c_str());
-
-      if (WaitFd(ServerFd, true, Timeout) == false)
-	 return _error->Error("SOCKS proxy %s reported connection to %s (%s), but timed out",
-	       ProxyInfo.c_str(), ServerName.Host.c_str(), bindaddr.c_str());
-      #undef APT_ReadOrFail
-      #undef APT_WriteOrFail
-   }
-   else
+   if (Server->InitHashes(Queue->ExpectedHashes) == false || Req.AddPartialFileToHashes(Req.File) == false)
    {
-      // Determine what host and port to use based on the proxy settings
-      int Port = 0;
-      string Host;
-      if (Proxy.empty() == true || Proxy.Host.empty() == true)
-      {
-	 if (ServerName.Port != 0)
-	    Port = ServerName.Port;
-	 Host = ServerName.Host;
-      }
-      else if (Proxy.Access != "http")
-	 return _error->Error("Unsupported proxy configured: %s", URI::SiteOnly(Proxy).c_str());
-      else
-      {
-	 if (Proxy.Port != 0)
-	    Port = Proxy.Port;
-	 Host = Proxy.Host;
-      }
-      return Connect(Host,Port,"http",80,ServerFd,TimeOut,Owner);
+      _error->Errno("read",_("Problem hashing file"));
+      return ERROR_NOT_FROM_SERVER;
    }
-   return true;
-}
-									/*}}}*/
-// HttpServerState::Close - Close a connection to the server		/*{{{*/
-// ---------------------------------------------------------------------
-/* */
-bool HttpServerState::Close()
-{
-   close(ServerFd);
-   ServerFd = -1;
-   return true;
+   if (Req.StartPos > 0)
+      Res.ResumePoint = Req.StartPos;
+
+   SetNonBlock(Req.File.Fd(),true);
+   return FILE_IS_OPEN;
 }
-									/*}}}*/
-// HttpServerState::RunData - Transfer the data from the socket		/*{{{*/
-bool HttpServerState::RunData(RequestState &Req)
+
+// HttpMethod::Loop - Main loop					/*{{{*/
+int HttpMethod::Loop()
 {
-   Req.State = RequestState::Data;
+   signal(SIGTERM,SigTerm);
+   signal(SIGINT,SigTerm);
    
-   // Chunked transfer encoding is fun..
-   if (Req.Encoding == RequestState::Chunked)
-   {
-      while (1)
+   Server = 0;
+
+   std::set<std::string> cached;
+   
+   int FailCounter = 0;
+   while (1)
+   {      
+      // We have no commands, wait for some to arrive
+      if (Queue == 0)
       {
-	 // Grab the block size
-	 bool Last = true;
-	 string Data;
-	 In.Limit(-1);
-	 do
-	 {
-	    if (In.WriteTillEl(Data,true) == true)
-	       break;
-	 }
-	 while ((Last = Go(false, Req)) == true);
-
-	 if (Last == false)
-	    return false;
-	 	 
-	 // See if we are done
-	 unsigned long long Len = strtoull(Data.c_str(),0,16);
-	 if (Len == 0)
-	 {
-	    In.Limit(-1);
-	    
-	    // We have to remove the entity trailer
-	    Last = true;
-	    do
-	    {
-	       if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
-		  break;
-	    }
-	    while ((Last = Go(false, Req)) == true);
-	    if (Last == false)
-	       return false;
-	    return !_error->PendingError();
-	 }
-	 
-	 // Transfer the block
-	 In.Limit(Len);
-	 while (Go(true, Req) == true)
-	    if (In.IsLimit() == true)
-	       break;
-	 
-	 // Error
-	 if (In.IsLimit() == false)
-	    return false;
-	 
-	 // The server sends an extra new line before the next block specifier..
-	 In.Limit(-1);
-	 Last = true;
-	 do
-	 {
-	    if (In.WriteTillEl(Data,true) == true)
-	       break;
-	 }
-	 while ((Last = Go(false, Req)) == true);
-	 if (Last == false)
-	    return false;
+	 if (WaitFd(STDIN_FILENO) == false)
+	    return 0;
       }
-   }
-   else
-   {
-      /* Closes encoding is used when the server did not specify a size, the
-         loss of the connection means we are done */
-      if (Req.JunkSize != 0)
-	 In.Limit(Req.JunkSize);
-      else if (Req.DownloadSize != 0)
-	 In.Limit(Req.DownloadSize);
-      else if (Persistent == false)
-	 In.Limit(-1);
       
-      // Just transfer the whole block.
-      do
+      /* Run messages, we can accept 0 (no message) if we didn't
+         do a WaitFd above.. Otherwise the FD is closed. */
+      int Result = Run(true);
+      if (Result != -1 && (Result != 0 || Queue == 0))
       {
-	 if (In.IsLimit() == false)
-	    continue;
-	 
-	 In.Limit(-1);
-	 return !_error->PendingError();
+	 if(FailReason.empty() == false ||
+	    ConfigFindB("DependOnSTDIN", true) == true)
+	    return 100;
+	 else
+	    return 0;
       }
-      while (Go(true, Req) == true);
-   }
 
-   return Flush(&Req.File) && !_error->PendingError();
-}
-									/*}}}*/
-bool HttpServerState::RunDataToDevNull(RequestState &Req)		/*{{{*/
-{
-   // no need to clean up if we discard the connection anyhow
-   if (Persistent == false)
-      return true;
-   Req.File.Open("/dev/null", FileFd::WriteOnly);
-   return RunData(Req);
-}
-									/*}}}*/
-bool HttpServerState::ReadHeaderLines(std::string &Data)		/*{{{*/
-{
-   return In.WriteTillEl(Data);
-}
-									/*}}}*/
-bool HttpServerState::LoadNextResponse(bool const ToFile, RequestState &Req)/*{{{*/
-{
-   return Go(ToFile, Req);
-}
-									/*}}}*/
-bool HttpServerState::WriteResponse(const std::string &Data)		/*{{{*/
-{
-   return Out.Read(Data);
-}
-									/*}}}*/
-APT_PURE bool HttpServerState::IsOpen()					/*{{{*/
-{
-   return (ServerFd != -1);
-}
-									/*}}}*/
-bool HttpServerState::InitHashes(HashStringList const &ExpectedHashes)	/*{{{*/
-{
-   delete In.Hash;
-   In.Hash = new Hashes(ExpectedHashes);
-   return true;
-}
-									/*}}}*/
-void HttpServerState::Reset()						/*{{{*/
-{
-   ServerState::Reset();
-   ServerFd = -1;
-}
-									/*}}}*/
+      if (Queue == 0)
+	 continue;
 
-APT_PURE Hashes * HttpServerState::GetHashes()				/*{{{*/
-{
-   return In.Hash;
-}
-									/*}}}*/
-// HttpServerState::Die - The server has closed the connection.		/*{{{*/
-bool HttpServerState::Die(RequestState &Req)
-{
-   unsigned int LErrno = errno;
+      CFStringEncoding se = kCFStringEncodingUTF8;
 
-   // Dump the buffer to the file
-   if (Req.State == RequestState::Data)
-   {
-      if (Req.File.IsOpen() == false)
-	 return true;
-      // on GNU/kFreeBSD, apt dies on /dev/null because non-blocking
-      // can't be set
-      if (Req.File.Name() != "/dev/null")
-	 SetNonBlock(Req.File.Fd(),false);
-      while (In.WriteSpace() == true)
-      {
-	 if (In.Write(Req.File.Fd()) == false)
-	    return _error->Errno("write",_("Error writing to the file"));
+      URI uri2 = Queue->Uri;
+      string uriString = static_cast<string>(uri2);
+    
+      char *url = strdup(uriString.c_str());
+    url:
+      URI uri = std::string(url);
+      std::string hs = uri.Host;
+
+      if (cached.find(hs) != cached.end()) {
+         _error->Error("Cached Failure");
+         Fail(true);
+         free(url);
+         FailCounter = 0;
+         continue;
+      }
+
+      std::string urs = uri;
 
-	 // Done
-	 if (In.IsLimit() == true)
-	    return true;
+      for (;;) {
+         size_t bad = urs.find_first_of("+");
+         if (bad == std::string::npos)
+            break;
+         // XXX: generalize
+         urs = urs.substr(0, bad) + "%2b" + urs.substr(bad + 1);
       }
-   }
 
-   // See if this is because the server finished the data stream
-   if (In.IsLimit() == false && Req.State != RequestState::Header &&
-       Persistent == true)
-   {
-      Close();
-      if (LErrno == 0)
-	 return _error->Error(_("Error reading from server. Remote end closed connection"));
-      errno = LErrno;
-      return _error->Errno("read",_("Error reading from server"));
-   }
-   else
-   {
-      In.Limit(-1);
+      CFStringRef sr = CFStringCreateWithCString(kCFAllocatorDefault, urs.c_str(), se);
+      CFURLRef ur = CFURLCreateWithString(kCFAllocatorDefault, sr, NULL);
+      CFRelease(sr);
+      CFHTTPMessageRef hm = CFHTTPMessageCreateRequest(kCFAllocatorDefault, CFSTR("GET"), ur, kCFHTTPVersion1_1);
+      CFRelease(ur);
+
+      struct stat SBuf;
+      if (stat(Queue->DestFile.c_str(), &SBuf) >= 0 && SBuf.st_size > 0) {
+         sr = CFStringCreateWithFormat(kCFAllocatorDefault, NULL, CFSTR("bytes=%li-"), (long) SBuf.st_size - 1);
+         CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("Range"), sr);
+         CFRelease(sr);
+
+         sr = CFStringCreateWithCString(kCFAllocatorDefault, TimeRFC1123(SBuf.st_mtime, false).c_str(), se);
+         CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("If-Range"), sr);
+         CFRelease(sr);
+
+         CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("Cache-Control"), CFSTR("no-cache"));
+      } else if (Queue->LastModified != 0) {
+         sr = CFStringCreateWithCString(kCFAllocatorDefault, TimeRFC1123(Queue->LastModified, true).c_str(), se);
+         CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("If-Modified-Since"), sr);
+         CFRelease(sr);
+
+         CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("Cache-Control"), CFSTR("no-cache"));
+      } else
+         CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("Cache-Control"), CFSTR("max-age=0"));
 
-      // Nothing left in the buffer
-      if (In.WriteSpace() == false)
-	 return false;
+      if (Firmware_ != NULL)
+         CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("X-Firmware"), Firmware_);
 
-      // We may have got multiple responses back in one packet..
-      Close();
-      return true;
-   }
+      sr = CFStringCreateWithCString(kCFAllocatorDefault, Machine_, se);
+      CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("X-Machine"), sr);
+      CFRelease(sr);
 
-   return false;
-}
-									/*}}}*/
-// HttpServerState::Flush - Dump the buffer into the file		/*{{{*/
-// ---------------------------------------------------------------------
-/* This takes the current input buffer from the Server FD and writes it
-   into the file */
-bool HttpServerState::Flush(FileFd * const File)
-{
-   if (File != nullptr)
-   {
-      // on GNU/kFreeBSD, apt dies on /dev/null because non-blocking
-      // can't be set
-      if (File->Name() != "/dev/null")
-	 SetNonBlock(File->Fd(),false);
-      if (In.WriteSpace() == false)
-	 return true;
-      
-      while (In.WriteSpace() == true)
-      {
-	 if (In.Write(File->Fd()) == false)
-	    return _error->Errno("write",_("Error writing to file"));
-	 if (In.IsLimit() == true)
-	    return true;
+      if (UniqueID_ != NULL)
+         CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("X-Unique-ID"), UniqueID_);
+
+      CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("User-Agent"), CFSTR("Telesphoreo APT-HTTP/1.0.592"));
+
+      CFReadStreamRef rs = CFReadStreamCreateForHTTPRequest(kCFAllocatorDefault, hm);
+      CFRelease(hm);
+
+#define _kCFStreamPropertyReadTimeout CFSTR("_kCFStreamPropertyReadTimeout")
+#define _kCFStreamPropertyWriteTimeout CFSTR("_kCFStreamPropertyWriteTimeout")
+#define _kCFStreamPropertySocketImmediateBufferTimeOut CFSTR("_kCFStreamPropertySocketImmediateBufferTimeOut")
+
+      /*SInt32 to(TimeOut);
+      CFNumberRef nm(CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &to));os_log(OS_LOG_DEFAULT, "[%{public}s:%{public}d]",__BASE_FILE__,__LINE__);*/
+      double to = TimeOut;
+      CFNumberRef nm(CFNumberCreate(kCFAllocatorDefault, kCFNumberDoubleType, &to));
+
+      CFReadStreamSetProperty(rs, _kCFStreamPropertyReadTimeout, nm);
+      CFReadStreamSetProperty(rs, _kCFStreamPropertyWriteTimeout, nm);
+      CFReadStreamSetProperty(rs, _kCFStreamPropertySocketImmediateBufferTimeOut, nm);
+      CFRelease(nm);
+
+      CFDictionaryRef dr = SCDynamicStoreCopyProxies(NULL);
+      CFReadStreamSetProperty(rs, kCFStreamPropertyHTTPProxy, dr);
+      CFRelease(dr);
+
+      //CFReadStreamSetProperty(rs, kCFStreamPropertyHTTPShouldAutoredirect, kCFBooleanTrue);
+      CFReadStreamSetProperty(rs, kCFStreamPropertyHTTPAttemptPersistentConnection, kCFBooleanTrue);
+
+      FetchResult Res;
+      CFIndex rd;
+      UInt32 sc;
+
+      uint8_t data[10240];
+      size_t offset = 0;
+
+      Status("Connecting to %s", hs.c_str());
+
+      switch (CFReadStreamOpen(rs, to)) {
+         case -1:
+            CfrsError("Open", rs);
+         goto fail;
+
+         case 0:
+            _error->Error("Host Unreachable");
+            cached.insert(hs);
+         goto fail;
+
+         case 1:
+            /* success */
+         break;
+
+         fail:
+            Fail(true);
+         goto done;
       }
 
-      if (In.IsLimit() == true || Persistent == false)
-	 return true;
-   }
-   return false;
-}
-									/*}}}*/
-// HttpServerState::Go - Run a single loop				/*{{{*/
-// ---------------------------------------------------------------------
-/* This runs the select loop over the server FDs, Output file FDs and
-   stdin. */
-bool HttpServerState::Go(bool ToFile, RequestState &Req)
-{
-   // Server has closed the connection
-   if (ServerFd == -1 && (In.WriteSpace() == false || 
-			       ToFile == false))
-      return false;
-   
-   fd_set rfds,wfds;
-   FD_ZERO(&rfds);
-   FD_ZERO(&wfds);
-   
-   /* Add the server. We only send more requests if the connection will 
-      be persisting */
-   if (Out.WriteSpace() == true && ServerFd != -1 
-       && Persistent == true)
-      FD_SET(ServerFd,&wfds);
-   if (In.ReadSpace() == true && ServerFd != -1)
-      FD_SET(ServerFd,&rfds);
-   
-   // Add the file
-   int FileFD = -1;
-   if (Req.File.IsOpen())
-      FileFD = Req.File.Fd();
-   
-   if (In.WriteSpace() == true && ToFile == true && FileFD != -1)
-      FD_SET(FileFD,&wfds);
-
-   // Add stdin
-   if (Owner->ConfigFindB("DependOnSTDIN", true) == true)
-      FD_SET(STDIN_FILENO,&rfds);
-	  
-   // Figure out the max fd
-   int MaxFd = FileFD;
-   if (MaxFd < ServerFd)
-      MaxFd = ServerFd;
-
-   // Select
-   struct timeval tv;
-   tv.tv_sec = TimeOut;
-   tv.tv_usec = 0;
-   int Res = 0;
-   if ((Res = select(MaxFd+1,&rfds,&wfds,0,&tv)) < 0)
-   {
-      if (errno == EINTR)
-	 return true;
-      return _error->Errno("select",_("Select failed"));
-   }
-   
-   if (Res == 0)
-   {
-      _error->Error(_("Connection timed out"));
-      return Die(Req);
-   }
-   
-   // Handle server IO
-   if (ServerFd != -1 && FD_ISSET(ServerFd,&rfds))
-   {
-      errno = 0;
-      if (In.Read(ServerFd) == false)
-	 return Die(Req);
-   }
-	 
-   if (ServerFd != -1 && FD_ISSET(ServerFd,&wfds))
-   {
-      errno = 0;
-      if (Out.Write(ServerFd) == false)
-	 return Die(Req);
-   }
+      rd = CFReadStreamRead(rs, data, sizeof(data));
 
-   // Send data to the file
-   if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
-   {
-      if (In.Write(FileFD) == false)
-	 return _error->Errno("write",_("Error writing to output file"));
-   }
+      if (rd == -1) {
+         CfrsError(uri.Host.c_str(), rs);
+         cached.insert(hs);
+         Fail(true);
+         goto done;
+      }
 
-   if (Req.MaximumSize > 0 && Req.File.IsOpen() && Req.File.Failed() == false && Req.File.Tell() > Req.MaximumSize)
-   {
-      Owner->SetFailReason("MaximumSizeExceeded");
-      return _error->Error("Writing more data than expected (%llu > %llu)",
-                           Req.File.Tell(), Req.MaximumSize);
-   }
+      Res.Filename = Queue->DestFile;
+
+      hm = (CFHTTPMessageRef) CFReadStreamCopyProperty(rs, kCFStreamPropertyHTTPResponseHeader);
+      sc = CFHTTPMessageGetResponseStatusCode(hm);
+
+      if (sc == 301 || sc == 302) {
+         sr = CFHTTPMessageCopyHeaderFieldValue(hm, CFSTR("Location"));
+         if (sr == NULL) {
+            Fail();
+            goto done_;
+         } else {
+            size_t ln = CFStringGetLength(sr) + 1;
+            free(url);
+            url = static_cast<char *>(malloc(ln));
+
+            if (!CFStringGetCString(sr, url, ln, se)) {
+               Fail();
+               goto done_;
+            }
+
+            CFRelease(sr);
+            goto url;
+         }
+      }
 
-   // Handle commands from APT
-   if (FD_ISSET(STDIN_FILENO,&rfds))
-   {
-      if (Owner->Run(true) != -1)
-	 exit(100);
-   }   
-       
-   return true;
-}
-									/*}}}*/
+      sr = CFHTTPMessageCopyHeaderFieldValue(hm, CFSTR("Content-Range"));
+      if (sr != NULL) {
+         size_t ln = CFStringGetLength(sr) + 1;
+         char cr[ln];
+
+         if (!CFStringGetCString(sr, cr, ln, se)) {
+            Fail();
+            goto done_;
+         }
+
+         CFRelease(sr);
+
+         if (sscanf(cr, "bytes %lu-%*u/%llu", &offset, &Res.Size) != 2) {
+	    _error->Error(_("The HTTP server sent an invalid Content-Range header"));
+            Fail();
+            goto done_;
+         }
+
+         if (offset > Res.Size) {
+	    _error->Error(_("This HTTP server has broken range support"));
+            Fail();
+            goto done_;
+         }
+      } else {
+         sr = CFHTTPMessageCopyHeaderFieldValue(hm, CFSTR("Content-Length"));
+         if (sr != NULL) {
+            Res.Size = CFStringGetIntValue(sr);
+            CFRelease(sr);
+         }
+      }
 
-// HttpMethod::SendReq - Send the HTTP request				/*{{{*/
-// ---------------------------------------------------------------------
-/* This places the http request in the outbound buffer */
-void HttpMethod::SendReq(FetchItem *Itm)
-{
-   URI Uri = Itm->Uri;
-   {
-      auto const plus = Binary.find('+');
-      if (plus != std::string::npos)
-	 Uri.Access = Binary.substr(plus + 1);
-   }
+      time(&Res.LastModified);
 
-   // The HTTP server expects a hostname with a trailing :port
-   std::stringstream Req;
-   string ProperHost;
-
-   if (Uri.Host.find(':') != string::npos)
-      ProperHost = '[' + Uri.Host + ']';
-   else
-      ProperHost = Uri.Host;
-
-   /* RFC 2616 §5.1.2 requires absolute URIs for requests to proxies,
-      but while its a must for all servers to accept absolute URIs,
-      it is assumed clients will sent an absolute path for non-proxies */
-   std::string requesturi;
-   if (Server->Proxy.Access != "http" || Server->Proxy.empty() == true || Server->Proxy.Host.empty())
-      requesturi = Uri.Path;
-   else
-      requesturi = Uri;
-
-   // The "+" is encoded as a workaround for a amazon S3 bug
-   // see LP bugs #1003633 and #1086997.
-   requesturi = QuoteString(requesturi, "+~ ");
-
-   /* Build the request. No keep-alive is included as it is the default
-      in 1.1, can cause problems with proxies, and we are an HTTP/1.1
-      client anyway.
-      C.f. https://tools.ietf.org/wg/httpbis/trac/ticket/158 */
-   Req << "GET " << requesturi << " HTTP/1.1\r\n";
-   if (Uri.Port != 0)
-      Req << "Host: " << ProperHost << ":" << std::to_string(Uri.Port) << "\r\n";
-   else
-      Req << "Host: " << ProperHost << "\r\n";
-
-   // generate a cache control header (if needed)
-   if (ConfigFindB("No-Cache",false) == true)
-      Req << "Cache-Control: no-cache\r\n"
-	 << "Pragma: no-cache\r\n";
-   else if (Itm->IndexFile == true)
-      Req << "Cache-Control: max-age=" << std::to_string(ConfigFindI("Max-Age", 0)) << "\r\n";
-   else if (ConfigFindB("No-Store", false) == true)
-      Req << "Cache-Control: no-store\r\n";
-
-   // If we ask for uncompressed files servers might respond with content-
-   // negotiation which lets us end up with compressed files we do not support,
-   // see 657029, 657560 and co, so if we have no extension on the request
-   // ask for text only. As a sidenote: If there is nothing to negotate servers
-   // seem to be nice and ignore it.
-   if (ConfigFindB("SendAccept", true) == true)
-   {
-      size_t const filepos = Itm->Uri.find_last_of('/');
-      string const file = Itm->Uri.substr(filepos + 1);
-      if (flExtension(file) == file)
-	 Req << "Accept: text/*\r\n";
-   }
+      sr = CFHTTPMessageCopyHeaderFieldValue(hm, CFSTR("Last-Modified"));
+      if (sr != NULL) {
+         size_t ln = CFStringGetLength(sr) + 1;
+         char cr[ln];
 
-   // Check for a partial file and send if-queries accordingly
-   struct stat SBuf;
-   if (Server->RangesAllowed && stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
-      Req << "Range: bytes=" << std::to_string(SBuf.st_size) << "-\r\n"
-	 << "If-Range: " << TimeRFC1123(SBuf.st_mtime, false) << "\r\n";
-   else if (Itm->LastModified != 0)
-      Req << "If-Modified-Since: " << TimeRFC1123(Itm->LastModified, false).c_str() << "\r\n";
+         if (!CFStringGetCString(sr, cr, ln, se)) {
+            Fail();
+            goto done_;
+         }
 
-   if (Server->Proxy.Access == "http" &&
-	 (Server->Proxy.User.empty() == false || Server->Proxy.Password.empty() == false))
-      Req << "Proxy-Authorization: Basic "
-	 << Base64Encode(Server->Proxy.User + ":" + Server->Proxy.Password) << "\r\n";
+         CFRelease(sr);
 
-   maybe_add_auth (Uri, _config->FindFile("Dir::Etc::netrc"));
-   if (Uri.User.empty() == false || Uri.Password.empty() == false)
-      Req << "Authorization: Basic "
-	 << Base64Encode(Uri.User + ":" + Uri.Password) << "\r\n";
+         if (!RFC1123StrToTime(cr, Res.LastModified)) {
+	    _error->Error(_("Unknown date format"));
+            Fail();
+            goto done_;
+         }
+      }
 
-   Req << "User-Agent: " << ConfigFind("User-Agent",
-		"Debian APT-HTTP/1.3 (" PACKAGE_VERSION ")") << "\r\n";
+      if (sc < 200 || (sc >= 300 && sc != 304)) {
+         sr = CFHTTPMessageCopyResponseStatusLine(hm);
 
-   Req << "\r\n";
+         size_t ln = CFStringGetLength(sr) + 1;
+         char cr[ln];
 
-   if (Debug == true)
-      cerr << Req.str() << endl;
+         if (!CFStringGetCString(sr, cr, ln, se)) {
+            Fail();
+            goto done;
+         }
 
-   Server->WriteResponse(Req.str());
-}
-									/*}}}*/
-std::unique_ptr<ServerState> HttpMethod::CreateServerState(URI const &uri)/*{{{*/
-{
-   return std::unique_ptr<ServerState>(new HttpServerState(uri, this));
-}
-									/*}}}*/
-void HttpMethod::RotateDNS()						/*{{{*/
-{
-   ::RotateDNS();
-}
-									/*}}}*/
-BaseHttpMethod::DealWithHeadersResult HttpMethod::DealWithHeaders(FetchResult &Res, RequestState &Req)/*{{{*/
-{
-   auto ret = BaseHttpMethod::DealWithHeaders(Res, Req);
-   if (ret != BaseHttpMethod::FILE_IS_OPEN)
-      return ret;
-   if (Req.File.Open(Queue->DestFile, FileFd::WriteAny) == false)
-      return ERROR_NOT_FROM_SERVER;
+         CFRelease(sr);
 
-   FailFile = Queue->DestFile;
-   FailFile.c_str();   // Make sure we don't do a malloc in the signal handler
-   FailFd = Req.File.Fd();
-   FailTime = Req.Date;
+         _error->Error("%s", cr);
 
-   if (Server->InitHashes(Queue->ExpectedHashes) == false || Req.AddPartialFileToHashes(Req.File) == false)
-   {
-      _error->Errno("read",_("Problem hashing file"));
-      return ERROR_NOT_FROM_SERVER;
-   }
-   if (Req.StartPos > 0)
-      Res.ResumePoint = Req.StartPos;
+         Fail();
+         goto done_;
+      }
 
-   SetNonBlock(Req.File.Fd(),true);
-   return FILE_IS_OPEN;
+      CFRelease(hm);
+
+      if (sc == 304) {
+         unlink(Queue->DestFile.c_str());
+         Res.IMSHit = true;
+         Res.LastModified = Queue->LastModified;
+         URIDone(Res);
+      } else {
+         Hashes hash;
+
+         File = new FileFd(Queue->DestFile, FileFd::WriteAny);
+         if (_error->PendingError() == true) {
+            delete File;
+            File = NULL;
+            Fail();
+            goto done;
+         }
+
+         FailFile = Queue->DestFile;
+         FailFile.c_str();   // Make sure we dont do a malloc in the signal handler
+         FailFd = File->Fd();
+         FailTime = Res.LastModified;
+
+         Res.ResumePoint = offset;
+         ftruncate(File->Fd(), offset);
+
+         if (offset != 0) {
+            lseek(File->Fd(), 0, SEEK_SET);
+            if (!hash.AddFD(File->Fd(), offset)) {
+               _error->Errno("read", _("Problem hashing file"));
+               delete File;
+               File = NULL;
+               Fail();
+               goto done;
+            }
+         }
+
+         lseek(File->Fd(), 0, SEEK_END);
+
+         URIStart(Res);
+
+         read: if (rd == -1) {
+            CfrsError("rd", rs);
+            Fail(true);
+         } else if (rd == 0) {
+            if (Res.Size == 0)
+               Res.Size = File->Size();
+	    
+            // Timestamp
+            struct timeval times[2];
+            times[0].tv_sec = times[1].tv_sec = Res.LastModified;
+            times[0].tv_usec = times[1].tv_usec = 0;
+            utimes(Queue->DestFile.c_str(), times);
+
+            Res.TakeHashes(hash);
+            URIDone(Res);
+         } else {
+            hash.Add(data, rd);
+
+            uint8_t *dt = data;
+            while (rd != 0) {
+               int sz = write(File->Fd(), dt, rd);
+
+               if (sz == -1) {
+                  delete File;
+                  File = NULL;
+                  Fail();
+                  goto done;
+               }
+
+               dt += sz;
+               rd -= sz;
+            }
+
+            rd = CFReadStreamRead(rs, data, sizeof(data));
+            goto read;
+         }
+      }
+
+      goto done;
+
+   done_:
+      CFRelease(hm);
+   done:
+      CFReadStreamClose(rs);
+      CFRelease(rs);
+      free(url);
+
+      FailCounter = 0;
+   }
+   
+   return 0;
 }
-									/*}}}*/
 HttpMethod::HttpMethod(std::string &&pProg) : BaseHttpMethod(pProg.c_str(), "1.2", Pipeline | SendConfig)/*{{{*/
 {
    auto addName = std::inserter(methodNames, methodNames.begin());
@@ -1010,5 +558,53 @@ HttpMethod::HttpMethod(std::string &&pProg) : BaseHttpMethod(pProg.c_str(), "1.2
    auto const plus = Binary.find('+');
    if (plus != std::string::npos)
       addName = Binary.substr(0, plus);
+   File = 0;
+   Server = 0;
 }
 									/*}}}*/
+
+int main(int, const char *argv[])
+{
+   // ignore SIGPIPE, this can happen on write() if the socket
+   // closes the connection (this is dealt with via ServerDie())
+   signal(SIGPIPE, SIG_IGN);
+
+   size_t size;
+   sysctlbyname("hw.machine", NULL, &size, NULL, 0);
+   char *machine = new char[size];
+   sysctlbyname("hw.machine", machine, &size, NULL, 0);
+   Machine_ = machine;
+
+   const char *path = "/System/Library/CoreServices/SystemVersion.plist";
+   CFURLRef url = CFURLCreateFromFileSystemRepresentation(kCFAllocatorDefault, (uint8_t *) path, strlen(path), false);
+
+   CFPropertyListRef plist; {
+      CFReadStreamRef stream = CFReadStreamCreateWithFile(kCFAllocatorDefault, url);
+      CFReadStreamOpen(stream);
+      plist = CFPropertyListCreateFromStream(kCFAllocatorDefault, stream, 0, kCFPropertyListImmutable, NULL, NULL);
+      CFReadStreamClose(stream);
+   }
+
+   CFRelease(url);
+
+   if (plist != NULL) {
+      Firmware_ = (CFStringRef) CFRetain(CFDictionaryGetValue((CFDictionaryRef) plist, CFSTR("ProductVersion")));
+      CFRelease(plist);
+   }
+
+   if (UniqueID_ == NULL)
+   if (void *libMobileGestalt = dlopen("/usr/lib/libMobileGestalt.dylib", RTLD_GLOBAL | RTLD_LAZY))
+   if (CFStringRef (*$MGCopyAnswer)(CFStringRef) = (CFStringRef (*)(CFStringRef)) dlsym(libMobileGestalt, "MGCopyAnswer"))
+      UniqueID_ = $MGCopyAnswer(CFSTR("UniqueDeviceID"));
+
+   if (UniqueID_ == NULL)
+   if (void *lockdown = lockdown_connect()) {
+      UniqueID_ = lockdown_copy_value(lockdown, NULL, kLockdownUniqueDeviceIDKey);
+      lockdown_disconnect(lockdown);
+   }
+
+   std::string Binary = flNotDir(argv[0]);
+   if (Binary.find('+') == std::string::npos && Binary != "http")
+      Binary.append("+http");
+   return HttpMethod(std::move(Binary)).Loop();
+}
diff --git a/methods/http.h b/methods/http.h
index c79a6454e..3b491dfda 100644
--- a/methods/http.h
+++ b/methods/http.h
@@ -26,101 +26,6 @@ class FileFd;
 class HttpMethod;
 class Hashes;
 
-class CircleBuf
-{
-   unsigned char *Buf;
-   unsigned long long Size;
-   unsigned long long InP;
-   unsigned long long OutP;
-   std::string OutQueue;
-   unsigned long long StrPos;
-   unsigned long long MaxGet;
-   struct timeval Start;
-
-   static unsigned long long BwReadLimit;
-   static unsigned long long BwTickReadData;
-   static struct timeval BwReadTick;
-   static const unsigned int BW_HZ;
-
-   unsigned long long LeftRead() const
-   {
-      unsigned long long Sz = Size - (InP - OutP);
-      if (Sz > Size - (InP%Size))
-	 Sz = Size - (InP%Size);
-      return Sz;
-   }
-   unsigned long long LeftWrite() const
-   {
-      unsigned long long Sz = InP - OutP;
-      if (InP > MaxGet)
-	 Sz = MaxGet - OutP;
-      if (Sz > Size - (OutP%Size))
-	 Sz = Size - (OutP%Size);
-      return Sz;
-   }
-   void FillOut();
-
-   public:
-   Hashes *Hash;
-   // total amount of data that got written so far
-   unsigned long long TotalWriten;
-
-   // Read data in
-   bool Read(int Fd);
-   bool Read(std::string const &Data);
-
-   // Write data out
-   bool Write(int Fd);
-   bool WriteTillEl(std::string &Data,bool Single = false);
-
-   // Control the write limit
-   void Limit(long long Max) {if (Max == -1) MaxGet = 0-1; else MaxGet = OutP + Max;}
-   bool IsLimit() const {return MaxGet == OutP;};
-   void Print() const {cout << MaxGet << ',' << OutP << endl;};
-
-   // Test for free space in the buffer
-   bool ReadSpace() const {return Size - (InP - OutP) > 0;};
-   bool WriteSpace() const {return InP - OutP > 0;};
-
-   void Reset();
-   // Dump everything
-   void Stats();
-
-   CircleBuf(HttpMethod const * const Owner, unsigned long long Size);
-   ~CircleBuf();
-};
-
-struct HttpServerState: public ServerState
-{
-   // This is the connection itself. Output is data FROM the server
-   CircleBuf In;
-   CircleBuf Out;
-   int ServerFd;
-
-   protected:
-   virtual bool ReadHeaderLines(std::string &Data) APT_OVERRIDE;
-   virtual bool LoadNextResponse(bool const ToFile, RequestState &Req) APT_OVERRIDE;
-   virtual bool WriteResponse(std::string const &Data) APT_OVERRIDE;
-
-   public:
-   virtual void Reset() APT_OVERRIDE;
-
-   virtual bool RunData(RequestState &Req) APT_OVERRIDE;
-   virtual bool RunDataToDevNull(RequestState &Req) APT_OVERRIDE;
-
-   virtual bool Open() APT_OVERRIDE;
-   virtual bool IsOpen() APT_OVERRIDE;
-   virtual bool Close() APT_OVERRIDE;
-   virtual bool InitHashes(HashStringList const &ExpectedHashes) APT_OVERRIDE;
-   virtual Hashes * GetHashes() APT_OVERRIDE;
-   virtual bool Die(RequestState &Req) APT_OVERRIDE;
-   virtual bool Flush(FileFd * const File) APT_OVERRIDE;
-   virtual bool Go(bool ToFile, RequestState &Req) APT_OVERRIDE;
-
-   HttpServerState(URI Srv, HttpMethod *Owner);
-   virtual ~HttpServerState() {Close();};
-};
-
 class HttpMethod : public BaseHttpMethod
 {
    public:
@@ -132,10 +37,13 @@ class HttpMethod : public BaseHttpMethod
 
    protected:
    std::string AutoDetectProxyCmd;
+   FileFd *File;
 
    public:
    friend struct HttpServerState;
 
+   int Loop();
+
    explicit HttpMethod(std::string &&pProg);
 };