From 895417ef99bb1371d8970da1afe87c6d64382f67 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Mon, 28 Apr 2014 10:02:27 +0200 Subject: reenable pipelining via hashsum reordering support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now that methods have the expected hashes available they can check if the response from the server is what they expected. Pipelining is one of those areas in which servers can mess up by not supporting it properly, which forced us to disable it for the time being. Now, we check if we got a response out of order, which we can not only use to disable pipelining automatically for the next requests, but we can fix it up just like the server responded in proper order for the current requests. To ensure that this little trick works pipelining is only attempt if we have hashsums for all the files in the chain which in theory reduces the use of pipelining usage even on the many servers which work properly, but in practice only the InRelease file (or similar such) will be requested without a hashsum – and as it is the only file requested in that stage it can't be pipelined even if we wanted to. Some minor annoyances remain: The display of the progress we have doesn't reflect this change, so it looks like the same package gets downloaded multiple times while others aren't at all. Further more, partial files are not supported in this recovery as the received data was appended to the wrong file, so the hashsum doesn't match. Both seem to be minor enough to reenable pipelining by default until further notice through to test if it really solves the problem. This therefore reverts commit 8221431757c775ee875a061b184b5f6f2330f928. --- methods/server.cc | 46 ++++++++++++++++++++++++++++++++++++++++++---- methods/server.h | 2 +- 2 files changed, 43 insertions(+), 5 deletions(-) (limited to 'methods') diff --git a/methods/server.cc b/methods/server.cc index 5a13f18a7..c91d3b218 100644 --- a/methods/server.cc +++ b/methods/server.cc @@ -392,9 +392,16 @@ bool ServerMethod::Fetch(FetchItem *) for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth; I = I->Next, Depth++) { - // If pipelining is disabled, we only queue 1 request - if (Server->Pipeline == false && Depth >= 0) - break; + if (Depth >= 0) + { + // If pipelining is disabled, we only queue 1 request + if (Server->Pipeline == false) + break; + // if we have no hashes, do at most one such request + // as we can't fixup pipeling misbehaviors otherwise + else if (I->ExpectedHashes.usable() == false) + break; + } // Make sure we stick with the same server if (Server->Comp(I->Uri) == false) @@ -546,7 +553,38 @@ int ServerMethod::Loop() // Send status to APT if (Result == true) { - Res.TakeHashes(*Server->GetHashes()); + Hashes * const resultHashes = Server->GetHashes(); + HashStringList const hashList = resultHashes->GetHashStringList(); + if (PipelineDepth != 0 && Queue->ExpectedHashes.usable() == true && Queue->ExpectedHashes != hashList) + { + // we did not get the expected hash… mhhh: + // could it be that server/proxy messed up pipelining? + FetchItem * BeforeI = Queue; + for (FetchItem *I = Queue->Next; I != 0 && I != QueueBack; I = I->Next) + { + if (I->ExpectedHashes.usable() == true && I->ExpectedHashes == hashList) + { + // yes, he did! Disable pipelining and rewrite queue + if (Server->Pipeline == true) + { + // FIXME: fake a warning message as we have no proper way of communicating here + std::string out; + strprintf(out, _("Automatically disabled %s due to incorrect response from server/proxy. (man 5 apt.conf)"), "Acquire::http::PipelineDepth"); + std::cerr << "W: " << out << std::endl; + Server->Pipeline = false; + // we keep the PipelineDepth value so that the rest of the queue can be fixed up as well + } + Rename(Res.Filename, I->DestFile); + Res.Filename = I->DestFile; + BeforeI->Next = I->Next; + I->Next = Queue; + Queue = I; + break; + } + BeforeI = I; + } + } + Res.TakeHashes(*resultHashes); URIDone(Res); } else diff --git a/methods/server.h b/methods/server.h index 0f45ab994..5299b3954 100644 --- a/methods/server.h +++ b/methods/server.h @@ -140,7 +140,7 @@ class ServerMethod : public pkgAcqMethod virtual ServerState * CreateServerState(URI uri) = 0; virtual void RotateDNS() = 0; - ServerMethod(const char *Ver,unsigned long Flags = 0) : pkgAcqMethod(Ver, Flags), Server(NULL), File(NULL), PipelineDepth(0), AllowRedirect(false), Debug(false) {}; + ServerMethod(const char *Ver,unsigned long Flags = 0) : pkgAcqMethod(Ver, Flags), Server(NULL), File(NULL), PipelineDepth(10), AllowRedirect(false), Debug(false) {}; virtual ~ServerMethod() {}; }; -- cgit v1.2.3 From 7b734b09f6bd9356e4622aee64bd2e5e43554570 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 24 Jun 2014 15:45:09 +0200 Subject: methods/http.cc: use Req.str() in debug output --- methods/http.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'methods') diff --git a/methods/http.cc b/methods/http.cc index c734d3799..7c7949eac 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -744,7 +744,7 @@ void HttpMethod::SendReq(FetchItem *Itm) Req << "\r\n"; if (Debug == true) - cerr << Req << endl; + cerr << Req.str() << endl; Server->WriteResponse(Req.str()); } -- cgit v1.2.3 From 2737f28a1cb2d03c66d2a7edd04215566903dbf1 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 21 Jul 2014 11:19:37 +0200 Subject: Download Release first, then Release.gpg The old way of handling this was that pkgAcqMetaIndex was responsible to check/move both Release and Release.gpg in place. This breaks the assumption of the transaction that each pkgAcquire::Item has a single File that its responsible for. --- methods/gpgv.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'methods') diff --git a/methods/gpgv.cc b/methods/gpgv.cc index ae521a2ed..30fd217bd 100644 --- a/methods/gpgv.cc +++ b/methods/gpgv.cc @@ -159,7 +159,7 @@ string GPGVMethod::VerifyGetSigners(const char *file, const char *outfile, waitpid(pid, &status, 0); if (Debug == true) { - std::clog << "gpgv exited\n"; + ioprintf(std::clog, "gpgv exited with status %i\n", WEXITSTATUS(status)); } if (WEXITSTATUS(status) == 0) -- cgit v1.2.3 From 5f6c6c6e085bcba021a29d88859e70b1e03ff153 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Sun, 24 Aug 2014 16:00:36 -0700 Subject: make compressed-indexes test pass again --- methods/copy.cc | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) (limited to 'methods') diff --git a/methods/copy.cc b/methods/copy.cc index d59f032ff..8c797ff1f 100644 --- a/methods/copy.cc +++ b/methods/copy.cc @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -27,12 +28,28 @@ class CopyMethod : public pkgAcqMethod { virtual bool Fetch(FetchItem *Itm); + void CalculateHashes(FetchResult &Res); public: - CopyMethod() : pkgAcqMethod("1.0",SingleInstance) {}; + CopyMethod() : pkgAcqMethod("1.0",SingleInstance|SendConfig) {}; }; +void CopyMethod::CalculateHashes(FetchResult &Res) +{ + // For gzip indexes we need to look inside the gzip for the hash + // We can not use the extension here as its not used in partial + // on a IMS hit + FileFd::OpenMode OpenMode = FileFd::ReadOnly; + if (_config->FindB("Acquire::GzipIndexes", false) == true) + OpenMode = FileFd::ReadOnlyGzip; + + Hashes Hash; + FileFd Fd(Res.Filename, OpenMode); + Hash.AddFD(Fd); + Res.TakeHashes(Hash); +} + // CopyMethod::Fetch - Fetch a file /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -53,6 +70,14 @@ bool CopyMethod::Fetch(FetchItem *Itm) Res.LastModified = Buf.st_mtime; Res.IMSHit = false; URIStart(Res); + + // when the files are identical, just compute the hashes + if(File == Itm->DestFile) + { + CalculateHashes(Res); + URIDone(Res); + return true; + } // See if the file exists FileFd From(File,FileFd::ReadOnly); @@ -82,10 +107,7 @@ bool CopyMethod::Fetch(FetchItem *Itm) if (utimes(Res.Filename.c_str(), times) != 0) return _error->Errno("utimes",_("Failed to set modification time")); - Hashes Hash; - FileFd Fd(Res.Filename, FileFd::ReadOnly); - Hash.AddFD(Fd); - Res.TakeHashes(Hash); + CalculateHashes(Res); URIDone(Res); return true; -- cgit v1.2.3 From dcd5856b11c685ca6d4629212d2978ce196ea65c Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 26 Aug 2014 19:08:37 -0700 Subject: Pass ExpectedSize to tthe backend method This ensures that we can stop downloading if the server send too much data by accident (or by a malicious attempt) --- methods/http.cc | 10 +++++++++- methods/http.h | 4 +++- methods/server.cc | 5 +++++ methods/server.h | 4 +++- 4 files changed, 20 insertions(+), 3 deletions(-) (limited to 'methods') diff --git a/methods/http.cc b/methods/http.cc index c734d3799..916fa464f 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -63,7 +63,8 @@ const unsigned int CircleBuf::BW_HZ=10; // CircleBuf::CircleBuf - Circular input buffer /*{{{*/ // --------------------------------------------------------------------- /* */ -CircleBuf::CircleBuf(unsigned long long Size) : Size(Size), Hash(0) +CircleBuf::CircleBuf(unsigned long long Size) + : Size(Size), Hash(0), TotalWriten(0) { Buf = new unsigned char[Size]; Reset(); @@ -79,6 +80,7 @@ void CircleBuf::Reset() InP = 0; OutP = 0; StrPos = 0; + TotalWriten = 0; MaxGet = (unsigned long long)-1; OutQueue = string(); if (Hash != 0) @@ -216,6 +218,8 @@ bool CircleBuf::Write(int Fd) return false; } + + TotalWriten += Res; if (Hash != 0) Hash->Add(Buf + (OutP%Size),Res); @@ -649,6 +653,10 @@ bool HttpServerState::Go(bool ToFile, FileFd * const File) return _error->Errno("write",_("Error writing to output file")); } + if (ExpectedSize > 0 && In.TotalWriten > ExpectedSize) + return _error->Error("Writing more data than expected (%llu > %llu)", + In.TotalWriten, ExpectedSize); + // Handle commands from APT if (FD_ISSET(STDIN_FILENO,&rfds)) { diff --git a/methods/http.h b/methods/http.h index 5406ce4a7..c98fe8e5f 100644 --- a/methods/http.h +++ b/methods/http.h @@ -63,6 +63,8 @@ class CircleBuf public: Hashes *Hash; + // total amount of data that got written so far + unsigned long long TotalWriten; // Read data in bool Read(int Fd); @@ -81,8 +83,8 @@ class CircleBuf bool ReadSpace() const {return Size - (InP - OutP) > 0;}; bool WriteSpace() const {return InP - OutP > 0;}; - // Dump everything void Reset(); + // Dump everything void Stats(); CircleBuf(unsigned long long Size); diff --git a/methods/server.cc b/methods/server.cc index c91d3b218..1b6511c59 100644 --- a/methods/server.cc +++ b/methods/server.cc @@ -531,6 +531,11 @@ int ServerMethod::Loop() // Run the data bool Result = true; + + // ensure we don't fetch too much + if (Queue->ExpectedSize > 0) + Server->ExpectedSize = Queue->ExpectedSize; + if (Server->HaveContent) Result = Server->RunData(File); diff --git a/methods/server.h b/methods/server.h index 5299b3954..0d7333140 100644 --- a/methods/server.h +++ b/methods/server.h @@ -49,6 +49,8 @@ struct ServerState URI Proxy; unsigned long TimeOut; + unsigned long long ExpectedSize; + protected: ServerMethod *Owner; @@ -73,7 +75,7 @@ struct ServerState bool Comp(URI Other) const {return Other.Host == ServerName.Host && Other.Port == ServerName.Port;}; virtual void Reset() {Major = 0; Minor = 0; Result = 0; Code[0] = '\0'; Size = 0; StartPos = 0; Encoding = Closes; time(&Date); HaveContent = false; - State = Header; Persistent = false; Pipeline = true;}; + State = Header; Persistent = false; Pipeline = true; ExpectedSize = 0;}; virtual bool WriteResponse(std::string const &Data) = 0; /** \brief Transfer the data from the socket */ -- cgit v1.2.3 From ffd2dd93a640b47663ebdccc4fda00b426b3db71 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 26 Aug 2014 19:20:04 -0700 Subject: make https honor ExpectedSize as well --- methods/https.cc | 6 ++++++ methods/https.h | 3 ++- 2 files changed, 8 insertions(+), 1 deletion(-) (limited to 'methods') diff --git a/methods/https.cc b/methods/https.cc index e0348ab58..cacd8a6bc 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -81,6 +81,12 @@ HttpsMethod::write_data(void *buffer, size_t size, size_t nmemb, void *userp) if(me->File->Write(buffer, size*nmemb) != true) return false; + me->TotalWritten += size*nmemb; + if(me->TotalWritten > me->Queue->ExpectedSize) + return _error->Error("Writing more data than expected (%llu > %llu)", + me->TotalWritten, me->Queue->ExpectedSize); + + return size*nmemb; } diff --git a/methods/https.h b/methods/https.h index faac8a3cd..2335559fb 100644 --- a/methods/https.h +++ b/methods/https.h @@ -66,11 +66,12 @@ class HttpsMethod : public pkgAcqMethod CURL *curl; FetchResult Res; HttpsServerState *Server; + unsigned long long TotalWritten; public: FileFd *File; - HttpsMethod() : pkgAcqMethod("1.2",Pipeline | SendConfig), File(NULL) + HttpsMethod() : pkgAcqMethod("1.2",Pipeline | SendConfig), TotalWritten(0), File(NULL) { File = 0; curl = curl_easy_init(); -- cgit v1.2.3 From c6ee61eab54edf6cc3fbe118d304d72a860e1451 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 2 Sep 2014 15:50:19 +0200 Subject: Make Proxy-Auto-Detect check for each host When doing Acquire::http{,s}::Proxy-Auto-Detect, run the auto-detect command for each host instead of only once. This should make using "proxy" from libproxy-tools feasible which can then be used for PAC style or other proxy configurations. Closes: #759264 --- methods/http.cc | 62 ++------------------------------------------------------ methods/http.h | 3 --- methods/https.cc | 4 ++++ 3 files changed, 6 insertions(+), 63 deletions(-) (limited to 'methods') diff --git a/methods/http.cc b/methods/http.cc index 7c7949eac..f2a4a4db6 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -34,6 +34,7 @@ #include #include #include +#include #include #include @@ -304,6 +305,7 @@ bool HttpServerState::Open() Persistent = true; // Determine the proxy setting + AutoDetectProxy(ServerName); string SpecificProxy = _config->Find("Acquire::http::Proxy::" + ServerName.Host); if (!SpecificProxy.empty()) { @@ -762,66 +764,6 @@ bool HttpMethod::Configuration(string Message) PipelineDepth); Debug = _config->FindB("Debug::Acquire::http",false); - // Get the proxy to use - AutoDetectProxy(); - - return true; -} - /*}}}*/ -// HttpMethod::AutoDetectProxy - auto detect proxy /*{{{*/ -// --------------------------------------------------------------------- -/* */ -bool HttpMethod::AutoDetectProxy() -{ - // option is "Acquire::http::Proxy-Auto-Detect" but we allow the old - // name without the dash ("-") - AutoDetectProxyCmd = _config->Find("Acquire::http::Proxy-Auto-Detect", - _config->Find("Acquire::http::ProxyAutoDetect")); - - if (AutoDetectProxyCmd.empty()) - return true; - - if (Debug) - clog << "Using auto proxy detect command: " << AutoDetectProxyCmd << endl; - - int Pipes[2] = {-1,-1}; - if (pipe(Pipes) != 0) - return _error->Errno("pipe", "Failed to create Pipe"); - - pid_t Process = ExecFork(); - if (Process == 0) - { - close(Pipes[0]); - dup2(Pipes[1],STDOUT_FILENO); - SetCloseExec(STDOUT_FILENO,false); - - const char *Args[2]; - Args[0] = AutoDetectProxyCmd.c_str(); - Args[1] = 0; - execv(Args[0],(char **)Args); - cerr << "Failed to exec method " << Args[0] << endl; - _exit(100); - } - char buf[512]; - int InFd = Pipes[0]; - close(Pipes[1]); - int res = read(InFd, buf, sizeof(buf)-1); - ExecWait(Process, "ProxyAutoDetect", true); - - if (res < 0) - return _error->Errno("read", "Failed to read"); - if (res == 0) - return _error->Warning("ProxyAutoDetect returned no data"); - - // add trailing \0 - buf[res] = 0; - - if (Debug) - clog << "auto detect command returned: '" << buf << "'" << endl; - - if (strstr(buf, "http://") == buf) - _config->Set("Acquire::http::proxy", _strstrip(buf)); - return true; } /*}}}*/ diff --git a/methods/http.h b/methods/http.h index 5406ce4a7..1df9fa07d 100644 --- a/methods/http.h +++ b/methods/http.h @@ -124,9 +124,6 @@ class HttpMethod : public ServerMethod public: virtual void SendReq(FetchItem *Itm); - /** \brief Try to AutoDetect the proxy */ - bool AutoDetectProxy(); - virtual bool Configuration(std::string Message); virtual ServerState * CreateServerState(URI uri); diff --git a/methods/https.cc b/methods/https.cc index e0348ab58..0499af0c5 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -107,6 +108,9 @@ void HttpsMethod::SetupProxy() /*{{{*/ { URI ServerName = Queue->Uri; + // Determine the proxy setting + AutoDetectProxy(ServerName); + // Curl should never read proxy settings from the environment, as // we determine which proxy to use. Do this for consistency among // methods and prevent an environment variable overriding a -- cgit v1.2.3 From 9622b2111095c3fc705ec0615d27fe403e18c3b8 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 5 Sep 2014 16:24:32 +0200 Subject: Improve Debug::Acquire::http debug output Prefix all answers with the URL that the answer is for. This helps when debugging and pipeline is enabled. --- methods/server.cc | 7 ++++--- methods/server.h | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) (limited to 'methods') diff --git a/methods/server.cc b/methods/server.cc index 5a13f18a7..92d94e638 100644 --- a/methods/server.cc +++ b/methods/server.cc @@ -44,7 +44,8 @@ time_t ServerMethod::FailTime = 0; // --------------------------------------------------------------------- /* Returns 0 if things are OK, 1 if an IO error occurred and 2 if a header parse error occurred */ -ServerState::RunHeadersResult ServerState::RunHeaders(FileFd * const File) +ServerState::RunHeadersResult ServerState::RunHeaders(FileFd * const File, + const std::string &Uri) { State = Header; @@ -66,7 +67,7 @@ ServerState::RunHeadersResult ServerState::RunHeaders(FileFd * const File) continue; if (Owner->Debug == true) - clog << Data; + clog << "Answer for: " << Uri << endl << Data; for (string::const_iterator I = Data.begin(); I < Data.end(); ++I) { @@ -478,7 +479,7 @@ int ServerMethod::Loop() Fetch(0); // Fetch the next URL header data from the server. - switch (Server->RunHeaders(File)) + switch (Server->RunHeaders(File, Queue->Uri)) { case ServerState::RUN_HEADERS_OK: break; diff --git a/methods/server.h b/methods/server.h index 0f45ab994..f5e68d902 100644 --- a/methods/server.h +++ b/methods/server.h @@ -68,7 +68,7 @@ struct ServerState RUN_HEADERS_PARSE_ERROR }; /** \brief Get the headers before the data */ - RunHeadersResult RunHeaders(FileFd * const File); + RunHeadersResult RunHeaders(FileFd * const File, const std::string &Uri); bool Comp(URI Other) const {return Other.Host == ServerName.Host && Other.Port == ServerName.Port;}; virtual void Reset() {Major = 0; Minor = 0; Result = 0; Code[0] = '\0'; Size = 0; -- cgit v1.2.3 From ca7fd76c2f30c100dcf1c12e717ce397cccd690b Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 16 Sep 2014 20:23:43 +0200 Subject: SECURITY UPDATE for CVE-2014-{0488,0487,0489} incorrect invalidating of unauthenticated data (CVE-2014-0488) incorect verification of 304 reply (CVE-2014-0487) incorrect verification of Acquire::Gzip indexes (CVE-2014-0489) --- methods/copy.cc | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) (limited to 'methods') diff --git a/methods/copy.cc b/methods/copy.cc index d59f032ff..5570f31c8 100644 --- a/methods/copy.cc +++ b/methods/copy.cc @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -27,12 +28,28 @@ class CopyMethod : public pkgAcqMethod { virtual bool Fetch(FetchItem *Itm); + void CalculateHashes(FetchResult &Res); public: - CopyMethod() : pkgAcqMethod("1.0",SingleInstance) {}; + CopyMethod() : pkgAcqMethod("1.0",SingleInstance | SendConfig) {}; }; +void CopyMethod::CalculateHashes(FetchResult &Res) +{ + // For gzip indexes we need to look inside the gzip for the hash + // We can not use the extension here as its not used in partial + // on a IMS hit + FileFd::OpenMode OpenMode = FileFd::ReadOnly; + if (_config->FindB("Acquire::GzipIndexes", false) == true) + OpenMode = FileFd::ReadOnlyGzip; + + Hashes Hash; + FileFd Fd(Res.Filename, OpenMode); + Hash.AddFD(Fd); + Res.TakeHashes(Hash); +} + // CopyMethod::Fetch - Fetch a file /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -54,6 +71,14 @@ bool CopyMethod::Fetch(FetchItem *Itm) Res.IMSHit = false; URIStart(Res); + // just calc the hashes if the source and destination are identical + if (File == Itm->DestFile) + { + CalculateHashes(Res); + URIDone(Res); + return true; + } + // See if the file exists FileFd From(File,FileFd::ReadOnly); FileFd To(Itm->DestFile,FileFd::WriteAtomic); @@ -82,10 +107,7 @@ bool CopyMethod::Fetch(FetchItem *Itm) if (utimes(Res.Filename.c_str(), times) != 0) return _error->Errno("utimes",_("Failed to set modification time")); - Hashes Hash; - FileFd Fd(Res.Filename, FileFd::ReadOnly); - Hash.AddFD(Fd); - Res.TakeHashes(Hash); + CalculateHashes(Res); URIDone(Res); return true; -- cgit v1.2.3 From 9da539c5aff025aab99537be1c75e8c6a853fd83 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 19 Sep 2014 16:41:55 +0200 Subject: Fix regression when copy: is used for a relative path When we do a ReverifyAfterIMS() we use the copy: method to verify the hashes again. If the user uses -o Dir=./something/relative this fails because we use the URI class in copy.cc that strips away the leading relative part. By not using URI this is fixed. Closes: #762160 --- methods/copy.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'methods') diff --git a/methods/copy.cc b/methods/copy.cc index 5570f31c8..b78053d36 100644 --- a/methods/copy.cc +++ b/methods/copy.cc @@ -55,8 +55,8 @@ void CopyMethod::CalculateHashes(FetchResult &Res) /* */ bool CopyMethod::Fetch(FetchItem *Itm) { - URI Get = Itm->Uri; - std::string File = Get.Path; + // this ensures that relative paths work in copy + std::string File = Itm->Uri.substr(Itm->Uri.find(':')+1); // Stat the file and send a start message struct stat Buf; -- cgit v1.2.3 From b0f4b486e6850c5f98520ccf19da71d0ed748ae4 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Sun, 21 Sep 2014 10:18:03 +0200 Subject: generalize Acquire::GzipIndex --- methods/copy.cc | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) (limited to 'methods') diff --git a/methods/copy.cc b/methods/copy.cc index b78053d36..40f8f85ec 100644 --- a/methods/copy.cc +++ b/methods/copy.cc @@ -37,15 +37,12 @@ class CopyMethod : public pkgAcqMethod void CopyMethod::CalculateHashes(FetchResult &Res) { - // For gzip indexes we need to look inside the gzip for the hash - // We can not use the extension here as its not used in partial - // on a IMS hit - FileFd::OpenMode OpenMode = FileFd::ReadOnly; + Hashes Hash; + FileFd::CompressMode CompressMode = FileFd::None; if (_config->FindB("Acquire::GzipIndexes", false) == true) - OpenMode = FileFd::ReadOnlyGzip; + CompressMode = FileFd::Extension; - Hashes Hash; - FileFd Fd(Res.Filename, OpenMode); + FileFd Fd(Res.Filename, FileFd::ReadOnly, CompressMode); Hash.AddFD(Fd); Res.TakeHashes(Hash); } -- cgit v1.2.3 From 3927c6da48c206b6b251661f44680d9883b4f6b4 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 24 Sep 2014 16:22:05 +0200 Subject: Drop Privileges to "Debian-apt" in most acquire methods Add a new "Debian-apt" user that owns the /var/lib/apt/lists and /var/cache/apt/archive directories. The methods http, https, ftp, gpgv, gzip switch to this user when they start. Thanks to Julian and "ioerror" and tors "switch_id()" code. --- methods/copy.cc | 2 ++ methods/ftp.cc | 3 +++ methods/gpgv.cc | 3 +++ methods/gzip.cc | 2 ++ methods/http_main.cc | 4 +++- methods/https.cc | 2 ++ 6 files changed, 15 insertions(+), 1 deletion(-) (limited to 'methods') diff --git a/methods/copy.cc b/methods/copy.cc index b78053d36..18d70e153 100644 --- a/methods/copy.cc +++ b/methods/copy.cc @@ -118,6 +118,8 @@ int main() { setlocale(LC_ALL, ""); + DropPrivs(); + CopyMethod Mth; return Mth.Run(); } diff --git a/methods/ftp.cc b/methods/ftp.cc index 66787a7be..9d58aa3b9 100644 --- a/methods/ftp.cc +++ b/methods/ftp.cc @@ -1107,6 +1107,9 @@ int main(int, const char *argv[]) { setlocale(LC_ALL, ""); + // no more active ftp, sorry + DropPrivs(); + /* See if we should be come the http client - we do this for http proxy urls */ if (getenv("ftp_proxy") != 0) diff --git a/methods/gpgv.cc b/methods/gpgv.cc index ae521a2ed..159417883 100644 --- a/methods/gpgv.cc +++ b/methods/gpgv.cc @@ -5,6 +5,7 @@ #include #include #include +#include #include #include @@ -262,6 +263,8 @@ int main() { setlocale(LC_ALL, ""); + DropPrivs(); + GPGVMethod Mth; return Mth.Run(); diff --git a/methods/gzip.cc b/methods/gzip.cc index df3f8828f..518e58f82 100644 --- a/methods/gzip.cc +++ b/methods/gzip.cc @@ -135,6 +135,8 @@ int main(int, char *argv[]) { setlocale(LC_ALL, ""); + DropPrivs(); + Prog = strrchr(argv[0],'/'); ++Prog; diff --git a/methods/http_main.cc b/methods/http_main.cc index 3b346a514..788582632 100644 --- a/methods/http_main.cc +++ b/methods/http_main.cc @@ -1,5 +1,5 @@ #include - +#include #include #include "http.h" @@ -12,6 +12,8 @@ int main() // closes the connection (this is dealt with via ServerDie()) signal(SIGPIPE, SIG_IGN); + DropPrivs(); + HttpMethod Mth; return Mth.Loop(); } diff --git a/methods/https.cc b/methods/https.cc index 0499af0c5..a40f37710 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -443,6 +443,8 @@ int main() { setlocale(LC_ALL, ""); + DropPrivs(); + HttpsMethod Mth; curl_global_init(CURL_GLOBAL_SSL) ; -- cgit v1.2.3 From 7b18d5592fd5e0bb173e193d1e6693a66065f971 Mon Sep 17 00:00:00 2001 From: Julian Andres Klode Date: Wed, 24 Sep 2014 21:49:19 +0200 Subject: methods: Fail if we cannot drop privileges --- methods/copy.cc | 4 ++-- methods/ftp.cc | 6 +++--- methods/gpgv.cc | 4 ++-- methods/gzip.cc | 5 +++-- methods/http_main.cc | 4 ++-- methods/https.cc | 4 ++-- 6 files changed, 14 insertions(+), 13 deletions(-) (limited to 'methods') diff --git a/methods/copy.cc b/methods/copy.cc index 18d70e153..3883c822b 100644 --- a/methods/copy.cc +++ b/methods/copy.cc @@ -118,8 +118,8 @@ int main() { setlocale(LC_ALL, ""); - DropPrivs(); - CopyMethod Mth; + + Mth.DropPrivsOrDie(); return Mth.Run(); } diff --git a/methods/ftp.cc b/methods/ftp.cc index 9d58aa3b9..a658b5657 100644 --- a/methods/ftp.cc +++ b/methods/ftp.cc @@ -1107,9 +1107,6 @@ int main(int, const char *argv[]) { setlocale(LC_ALL, ""); - // no more active ftp, sorry - DropPrivs(); - /* See if we should be come the http client - we do this for http proxy urls */ if (getenv("ftp_proxy") != 0) @@ -1134,6 +1131,9 @@ int main(int, const char *argv[]) } FtpMethod Mth; + + // no more active ftp, sorry + Mth.DropPrivsOrDie(); return Mth.Run(); } diff --git a/methods/gpgv.cc b/methods/gpgv.cc index 159417883..4071cbac6 100644 --- a/methods/gpgv.cc +++ b/methods/gpgv.cc @@ -262,10 +262,10 @@ bool GPGVMethod::Fetch(FetchItem *Itm) int main() { setlocale(LC_ALL, ""); - - DropPrivs(); GPGVMethod Mth; + Mth.DropPrivsOrDie(); + return Mth.Run(); } diff --git a/methods/gzip.cc b/methods/gzip.cc index 518e58f82..7ffcda60f 100644 --- a/methods/gzip.cc +++ b/methods/gzip.cc @@ -135,11 +135,12 @@ int main(int, char *argv[]) { setlocale(LC_ALL, ""); - DropPrivs(); - Prog = strrchr(argv[0],'/'); ++Prog; GzipMethod Mth; + + Mth.DropPrivsOrDie(); + return Mth.Run(); } diff --git a/methods/http_main.cc b/methods/http_main.cc index 788582632..d7724701a 100644 --- a/methods/http_main.cc +++ b/methods/http_main.cc @@ -12,8 +12,8 @@ int main() // closes the connection (this is dealt with via ServerDie()) signal(SIGPIPE, SIG_IGN); - DropPrivs(); - HttpMethod Mth; + + Mth.DropPrivsOrDie(); return Mth.Loop(); } diff --git a/methods/https.cc b/methods/https.cc index a40f37710..a74d2a38b 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -443,11 +443,11 @@ int main() { setlocale(LC_ALL, ""); - DropPrivs(); - HttpsMethod Mth; curl_global_init(CURL_GLOBAL_SSL) ; + Mth.DropPrivsOrDie(); + return Mth.Run(); } -- cgit v1.2.3 From 47d278dc7184606f751d015689e0c49eccde4547 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 24 Sep 2014 20:14:55 +0200 Subject: releasing package apt version 1.1~exp3 --- methods/http_main.cc | 1 + 1 file changed, 1 insertion(+) (limited to 'methods') diff --git a/methods/http_main.cc b/methods/http_main.cc index d7724701a..f21a5709c 100644 --- a/methods/http_main.cc +++ b/methods/http_main.cc @@ -1,5 +1,6 @@ #include #include +#include #include #include "http.h" -- cgit v1.2.3 From d36b27305dae21f9b3b6de056853ecd8bbd157e6 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 26 Sep 2014 17:50:18 +0200 Subject: Disable Mth.DropPrivsOrDie() in copy.cc for now Dch-Ignore: true --- methods/copy.cc | 1 - 1 file changed, 1 deletion(-) (limited to 'methods') diff --git a/methods/copy.cc b/methods/copy.cc index 3883c822b..b65bc4dd5 100644 --- a/methods/copy.cc +++ b/methods/copy.cc @@ -120,6 +120,5 @@ int main() CopyMethod Mth; - Mth.DropPrivsOrDie(); return Mth.Run(); } -- cgit v1.2.3 From 25613a61f6f3b9e54d5229af7e2278d0fa54bdd9 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 26 Sep 2014 22:16:26 +0200 Subject: fix: Member variable 'X' is not initialized in the constructor. Reported-By: cppcheck Git-Dch: Ignore --- methods/ftp.cc | 5 +++-- methods/https.h | 5 ++--- 2 files changed, 5 insertions(+), 5 deletions(-) (limited to 'methods') diff --git a/methods/ftp.cc b/methods/ftp.cc index a658b5657..ac76295f0 100644 --- a/methods/ftp.cc +++ b/methods/ftp.cc @@ -75,9 +75,10 @@ time_t FtpMethod::FailTime = 0; // FTPConn::FTPConn - Constructor /*{{{*/ // --------------------------------------------------------------------- /* */ -FTPConn::FTPConn(URI Srv) : Len(0), ServerFd(-1), DataFd(-1), +FTPConn::FTPConn(URI Srv) : Len(0), ServerFd(-1), DataFd(-1), DataListenFd(-1), ServerName(Srv), - ForceExtended(false), TryPassive(true) + ForceExtended(false), TryPassive(true), + PeerAddrLen(0), ServerAddrLen(0) { Debug = _config->FindB("Debug::Acquire::Ftp",false); PasvAddr = 0; diff --git a/methods/https.h b/methods/https.h index faac8a3cd..45d1f7f63 100644 --- a/methods/https.h +++ b/methods/https.h @@ -69,10 +69,9 @@ class HttpsMethod : public pkgAcqMethod public: FileFd *File; - - HttpsMethod() : pkgAcqMethod("1.2",Pipeline | SendConfig), File(NULL) + + HttpsMethod() : pkgAcqMethod("1.2",Pipeline | SendConfig), Server(NULL), File(NULL) { - File = 0; curl = curl_easy_init(); }; -- cgit v1.2.3 From b39bb552f8de65cea13dc5f1fae6fbeb198605c6 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 26 Jan 2014 17:23:50 +0100 Subject: correct the error messages to refer to apt-key instead of gpgv Git-Dch: Ignore --- methods/gpgv.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'methods') diff --git a/methods/gpgv.cc b/methods/gpgv.cc index 4071cbac6..02fb8c356 100644 --- a/methods/gpgv.cc +++ b/methods/gpgv.cc @@ -75,7 +75,7 @@ string GPGVMethod::VerifyGetSigners(const char *file, const char *outfile, FILE *pipein = fdopen(fd[0], "r"); - // Loop over the output of gpgv, and check the signatures. + // Loop over the output of apt-key (which really is gnupg), and check the signatures. size_t buffersize = 64; char *buffer = (char *) malloc(buffersize); size_t bufferoff = 0; @@ -160,7 +160,7 @@ string GPGVMethod::VerifyGetSigners(const char *file, const char *outfile, waitpid(pid, &status, 0); if (Debug == true) { - std::clog << "gpgv exited\n"; + std::clog << "apt-key exited\n"; } if (WEXITSTATUS(status) == 0) @@ -172,7 +172,7 @@ string GPGVMethod::VerifyGetSigners(const char *file, const char *outfile, else if (WEXITSTATUS(status) == 1) return _("At least one invalid signature was encountered."); else if (WEXITSTATUS(status) == 111) - return _("Could not execute 'gpgv' to verify signature (is gpgv installed?)"); + return _("Could not execute 'apt-key' to verify signature (is gnupg installed?)"); else if (WEXITSTATUS(status) == 112) { // acquire system checks for "NODATA" to generate GPG errors (the others are only warnings) @@ -182,7 +182,7 @@ string GPGVMethod::VerifyGetSigners(const char *file, const char *outfile, return errmsg; } else - return _("Unknown error executing gpgv"); + return _("Unknown error executing apt-key"); } bool GPGVMethod::Fetch(FetchItem *Itm) @@ -200,7 +200,7 @@ bool GPGVMethod::Fetch(FetchItem *Itm) Res.Filename = Itm->DestFile; URIStart(Res); - // Run gpgv on file, extract contents and get the key ID of the signer + // Run apt-key on file, extract contents and get the key ID of the signer string msg = VerifyGetSigners(Path.c_str(), Itm->DestFile.c_str(), GoodSigners, BadSigners, WorthlessSigners, NoPubKeySigners); @@ -252,7 +252,7 @@ bool GPGVMethod::Fetch(FetchItem *Itm) if (_config->FindB("Debug::Acquire::gpgv", false)) { - std::clog << "gpgv succeeded\n"; + std::clog << "apt-key succeeded\n"; } return true; -- cgit v1.2.3 From 84361defb573c52d9a5368069254d1e18105aa62 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 27 Sep 2014 01:00:14 +0200 Subject: fix: %i in format string (no. 1) requires 'int' but the argument type is 'unsigned int' Git-Dch: Ignore Reported-By: cppcheck --- methods/server.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'methods') diff --git a/methods/server.cc b/methods/server.cc index ff67eb09b..4a961f454 100644 --- a/methods/server.cc +++ b/methods/server.cc @@ -324,10 +324,10 @@ ServerMethod::DealWithHeaders(FetchResult &Res) failure */ if (Server->Result < 200 || Server->Result >= 300) { - char err[255]; - snprintf(err,sizeof(err)-1,"HttpError%i",Server->Result); + std::string err; + strprintf(err, "HttpError%u", Server->Result); SetFailReason(err); - _error->Error("%u %s",Server->Result,Server->Code); + _error->Error("%u %s", Server->Result, Server->Code); if (Server->HaveContent == true) return ERROR_WITH_CONTENT_PAGE; return ERROR_UNRECOVERABLE; -- cgit v1.2.3 From 62acbba8d1e8c9f395ccd1068e89bf14a93d4aa8 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 7 Oct 2014 08:16:51 +0200 Subject: methods/https.cc: use File->Tell() here too --- methods/https.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'methods') diff --git a/methods/https.cc b/methods/https.cc index eec858417..f8e84a2ff 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -82,8 +82,7 @@ HttpsMethod::write_data(void *buffer, size_t size, size_t nmemb, void *userp) if(me->File->Write(buffer, size*nmemb) != true) return false; - me->TotalWritten += size*nmemb; - if(me->TotalWritten > me->Queue->ExpectedSize) + if(me->Queue->ExpectedSize > 0 && me->File->Tell() > me->Queue->ExpectedSize) return _error->Error("Writing more data than expected (%llu > %llu)", me->TotalWritten, me->Queue->ExpectedSize); -- cgit v1.2.3 From 5b33fab8c9a8c26cbc8ce3bf5e1573279d7884b3 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 7 Oct 2014 08:43:46 +0200 Subject: add ftp expected size check --- methods/ftp.cc | 10 +++++++--- methods/ftp.h | 2 +- methods/https.cc | 1 - 3 files changed, 8 insertions(+), 5 deletions(-) (limited to 'methods') diff --git a/methods/ftp.cc b/methods/ftp.cc index ac76295f0..75ace1c5a 100644 --- a/methods/ftp.cc +++ b/methods/ftp.cc @@ -849,7 +849,7 @@ bool FTPConn::Finalize() /* This opens a data connection, sends REST and RETR and then transfers the file over. */ bool FTPConn::Get(const char *Path,FileFd &To,unsigned long long Resume, - Hashes &Hash,bool &Missing) + Hashes &Hash,bool &Missing, unsigned long long ExpectedSize) { Missing = false; if (CreateDataFd() == false) @@ -922,7 +922,11 @@ bool FTPConn::Get(const char *Path,FileFd &To,unsigned long long Resume, { Close(); return false; - } + } + + if (ExpectedSize > 0 && To.Tell() > ExpectedSize) + return _error->Error("Writing more data than expected (%llu > %llu)", + To.Tell(), ExpectedSize); } // All done @@ -1063,7 +1067,7 @@ bool FtpMethod::Fetch(FetchItem *Itm) FailFd = Fd.Fd(); bool Missing; - if (Server->Get(File,Fd,Res.ResumePoint,Hash,Missing) == false) + if (Server->Get(File,Fd,Res.ResumePoint,Hash,Missing,Itm->ExpectedSize) == false) { Fd.Close(); diff --git a/methods/ftp.h b/methods/ftp.h index dd92f0086..416a91980 100644 --- a/methods/ftp.h +++ b/methods/ftp.h @@ -62,7 +62,7 @@ class FTPConn bool Size(const char *Path,unsigned long long &Size); bool ModTime(const char *Path, time_t &Time); bool Get(const char *Path,FileFd &To,unsigned long long Resume, - Hashes &MD5,bool &Missing); + Hashes &MD5,bool &Missing, unsigned long long ExpectedSize); FTPConn(URI Srv); ~FTPConn(); diff --git a/methods/https.cc b/methods/https.cc index f8e84a2ff..a4794e705 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -86,7 +86,6 @@ HttpsMethod::write_data(void *buffer, size_t size, size_t nmemb, void *userp) return _error->Error("Writing more data than expected (%llu > %llu)", me->TotalWritten, me->Queue->ExpectedSize); - return size*nmemb; } -- cgit v1.2.3 From c48eea97b93920062ea26001081d4fdf7eb967e3 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 7 Oct 2014 17:47:30 +0200 Subject: make expected-size a maximum-size check as this is what we want at this point --- methods/ftp.cc | 8 ++++---- methods/ftp.h | 2 +- methods/http.cc | 4 ++-- methods/https.cc | 4 ++-- methods/server.cc | 4 ++-- methods/server.h | 4 ++-- 6 files changed, 13 insertions(+), 13 deletions(-) (limited to 'methods') diff --git a/methods/ftp.cc b/methods/ftp.cc index 75ace1c5a..bc84dda7d 100644 --- a/methods/ftp.cc +++ b/methods/ftp.cc @@ -849,7 +849,7 @@ bool FTPConn::Finalize() /* This opens a data connection, sends REST and RETR and then transfers the file over. */ bool FTPConn::Get(const char *Path,FileFd &To,unsigned long long Resume, - Hashes &Hash,bool &Missing, unsigned long long ExpectedSize) + Hashes &Hash,bool &Missing, unsigned long long MaximumSize) { Missing = false; if (CreateDataFd() == false) @@ -924,9 +924,9 @@ bool FTPConn::Get(const char *Path,FileFd &To,unsigned long long Resume, return false; } - if (ExpectedSize > 0 && To.Tell() > ExpectedSize) + if (MaximumSize > 0 && To.Tell() > MaximumSize) return _error->Error("Writing more data than expected (%llu > %llu)", - To.Tell(), ExpectedSize); + To.Tell(), MaximumSize); } // All done @@ -1067,7 +1067,7 @@ bool FtpMethod::Fetch(FetchItem *Itm) FailFd = Fd.Fd(); bool Missing; - if (Server->Get(File,Fd,Res.ResumePoint,Hash,Missing,Itm->ExpectedSize) == false) + if (Server->Get(File,Fd,Res.ResumePoint,Hash,Missing,Itm->MaximumSize) == false) { Fd.Close(); diff --git a/methods/ftp.h b/methods/ftp.h index 416a91980..a31ebc999 100644 --- a/methods/ftp.h +++ b/methods/ftp.h @@ -62,7 +62,7 @@ class FTPConn bool Size(const char *Path,unsigned long long &Size); bool ModTime(const char *Path, time_t &Time); bool Get(const char *Path,FileFd &To,unsigned long long Resume, - Hashes &MD5,bool &Missing, unsigned long long ExpectedSize); + Hashes &MD5,bool &Missing, unsigned long long MaximumSize); FTPConn(URI Srv); ~FTPConn(); diff --git a/methods/http.cc b/methods/http.cc index b076e59cc..f8faa0cf8 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -655,10 +655,10 @@ bool HttpServerState::Go(bool ToFile, FileFd * const File) return _error->Errno("write",_("Error writing to output file")); } - if (ExpectedSize > 0 && File && File->Tell() > ExpectedSize) + if (MaximumSize > 0 && File && File->Tell() > MaximumSize) { return _error->Error("Writing more data than expected (%llu > %llu)", - File->Tell(), ExpectedSize); + File->Tell(), MaximumSize); } // Handle commands from APT diff --git a/methods/https.cc b/methods/https.cc index a4794e705..787e4a507 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -82,9 +82,9 @@ HttpsMethod::write_data(void *buffer, size_t size, size_t nmemb, void *userp) if(me->File->Write(buffer, size*nmemb) != true) return false; - if(me->Queue->ExpectedSize > 0 && me->File->Tell() > me->Queue->ExpectedSize) + if(me->Queue->MaximumSize > 0 && me->File->Tell() > me->Queue->MaximumSize) return _error->Error("Writing more data than expected (%llu > %llu)", - me->TotalWritten, me->Queue->ExpectedSize); + me->TotalWritten, me->Queue->MaximumSize); return size*nmemb; } diff --git a/methods/server.cc b/methods/server.cc index 223737901..82f9b4750 100644 --- a/methods/server.cc +++ b/methods/server.cc @@ -534,8 +534,8 @@ int ServerMethod::Loop() bool Result = true; // ensure we don't fetch too much - if (Queue->ExpectedSize > 0) - Server->ExpectedSize = Queue->ExpectedSize; + if (Queue->MaximumSize > 0) + Server->MaximumSize = Queue->MaximumSize; if (Server->HaveContent) Result = Server->RunData(File); diff --git a/methods/server.h b/methods/server.h index 0134a9538..3093e00c9 100644 --- a/methods/server.h +++ b/methods/server.h @@ -49,7 +49,7 @@ struct ServerState URI Proxy; unsigned long TimeOut; - unsigned long long ExpectedSize; + unsigned long long MaximumSize; protected: ServerMethod *Owner; @@ -75,7 +75,7 @@ struct ServerState bool Comp(URI Other) const {return Other.Host == ServerName.Host && Other.Port == ServerName.Port;}; virtual void Reset() {Major = 0; Minor = 0; Result = 0; Code[0] = '\0'; Size = 0; StartPos = 0; Encoding = Closes; time(&Date); HaveContent = false; - State = Header; Persistent = false; Pipeline = true; ExpectedSize = 0;}; + State = Header; Persistent = false; Pipeline = true; MaximumSize = 0;}; virtual bool WriteResponse(std::string const &Data) = 0; /** \brief Transfer the data from the socket */ -- cgit v1.2.3 From ee27950632c149bb14c9c490e92147579ba4fc2a Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 7 Oct 2014 22:36:09 +0200 Subject: Send "Fail-Reason: MaximumSizeExceeded" from the method Communicate the fail reason from the methods to the parent and Rename() failed files. --- methods/ftp.cc | 8 ++++++-- methods/ftp.h | 3 ++- methods/http.cc | 1 + methods/https.cc | 4 +++- 4 files changed, 12 insertions(+), 4 deletions(-) (limited to 'methods') diff --git a/methods/ftp.cc b/methods/ftp.cc index bc84dda7d..5b739ea06 100644 --- a/methods/ftp.cc +++ b/methods/ftp.cc @@ -849,7 +849,8 @@ bool FTPConn::Finalize() /* This opens a data connection, sends REST and RETR and then transfers the file over. */ bool FTPConn::Get(const char *Path,FileFd &To,unsigned long long Resume, - Hashes &Hash,bool &Missing, unsigned long long MaximumSize) + Hashes &Hash,bool &Missing, unsigned long long MaximumSize, + pkgAcqMethod *Owner) { Missing = false; if (CreateDataFd() == false) @@ -925,8 +926,11 @@ bool FTPConn::Get(const char *Path,FileFd &To,unsigned long long Resume, } if (MaximumSize > 0 && To.Tell() > MaximumSize) + { + Owner->SetFailReason("MaximumSizeExceeded"); return _error->Error("Writing more data than expected (%llu > %llu)", To.Tell(), MaximumSize); + } } // All done @@ -1067,7 +1071,7 @@ bool FtpMethod::Fetch(FetchItem *Itm) FailFd = Fd.Fd(); bool Missing; - if (Server->Get(File,Fd,Res.ResumePoint,Hash,Missing,Itm->MaximumSize) == false) + if (Server->Get(File,Fd,Res.ResumePoint,Hash,Missing,Itm->MaximumSize,this) == false) { Fd.Close(); diff --git a/methods/ftp.h b/methods/ftp.h index a31ebc999..2efd28ec6 100644 --- a/methods/ftp.h +++ b/methods/ftp.h @@ -62,7 +62,8 @@ class FTPConn bool Size(const char *Path,unsigned long long &Size); bool ModTime(const char *Path, time_t &Time); bool Get(const char *Path,FileFd &To,unsigned long long Resume, - Hashes &MD5,bool &Missing, unsigned long long MaximumSize); + Hashes &MD5,bool &Missing, unsigned long long MaximumSize, + pkgAcqMethod *Owner); FTPConn(URI Srv); ~FTPConn(); diff --git a/methods/http.cc b/methods/http.cc index f8faa0cf8..c00b439b7 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -657,6 +657,7 @@ bool HttpServerState::Go(bool ToFile, FileFd * const File) if (MaximumSize > 0 && File && File->Tell() > MaximumSize) { + Owner->SetFailReason("MaximumSizeExceeded"); return _error->Error("Writing more data than expected (%llu > %llu)", File->Tell(), MaximumSize); } diff --git a/methods/https.cc b/methods/https.cc index 787e4a507..16d564b34 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -83,9 +83,11 @@ HttpsMethod::write_data(void *buffer, size_t size, size_t nmemb, void *userp) return false; if(me->Queue->MaximumSize > 0 && me->File->Tell() > me->Queue->MaximumSize) + { + me->SetFailReason("MaximumSizeExceeded"); return _error->Error("Writing more data than expected (%llu > %llu)", me->TotalWritten, me->Queue->MaximumSize); - + } return size*nmemb; } -- cgit v1.2.3 From f2b47ba290f3a178c584da83f007cf0f720baabb Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 8 Oct 2014 08:32:42 +0200 Subject: Fix http pipeline messup detection The Maximum-Size protection breaks the http pipeline reorder code because it relies on that the object got fetched entirely so that it can compare the hash of the downloaded data. So instead of stopping when the Maximum-Size of the expected item is reached we only stop when the maximum size of the biggest item in the queue is reached. This way the pipeline reoder code keeps working. --- methods/server.cc | 16 ++++++++++++++-- methods/server.h | 4 ++++ 2 files changed, 18 insertions(+), 2 deletions(-) (limited to 'methods') diff --git a/methods/server.cc b/methods/server.cc index 82f9b4750..cef809738 100644 --- a/methods/server.cc +++ b/methods/server.cc @@ -534,8 +534,10 @@ int ServerMethod::Loop() bool Result = true; // ensure we don't fetch too much - if (Queue->MaximumSize > 0) - Server->MaximumSize = Queue->MaximumSize; + // we could do "Server->MaximumSize = Queue->MaximumSize" here + // but that would break the clever pipeline messup detection + // so instead we use the size of the biggest item in the queue + Server->MaximumSize = FindMaximumObjectSizeInQueue(); if (Server->HaveContent) Result = Server->RunData(File); @@ -706,5 +708,15 @@ int ServerMethod::Loop() } return 0; +} + /*}}}*/ + /*{{{*/ +unsigned long long +ServerMethod::FindMaximumObjectSizeInQueue() const +{ + unsigned long long MaxSizeInQueue = 0; + for (FetchItem *I = Queue->Next; I != 0 && I != QueueBack; I = I->Next) + MaxSizeInQueue = std::max(MaxSizeInQueue, I->MaximumSize); + return MaxSizeInQueue; } /*}}}*/ diff --git a/methods/server.h b/methods/server.h index 3093e00c9..7d5198478 100644 --- a/methods/server.h +++ b/methods/server.h @@ -106,6 +106,10 @@ class ServerMethod : public pkgAcqMethod unsigned long PipelineDepth; bool AllowRedirect; + // Find the biggest item in the fetch queue for the checking of the maximum + // size + unsigned long long FindMaximumObjectSizeInQueue() const APT_PURE; + public: bool Debug; -- cgit v1.2.3 From 8bb043fcd1def35cf516d0961a2b5e6ee53f0fd7 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 8 Oct 2014 09:45:11 +0200 Subject: Fix ServerMethod::FindMaximumObjectSizeInQueue() Git-Dch: ignore --- methods/server.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'methods') diff --git a/methods/server.cc b/methods/server.cc index cef809738..c4689ff12 100644 --- a/methods/server.cc +++ b/methods/server.cc @@ -715,7 +715,7 @@ unsigned long long ServerMethod::FindMaximumObjectSizeInQueue() const { unsigned long long MaxSizeInQueue = 0; - for (FetchItem *I = Queue->Next; I != 0 && I != QueueBack; I = I->Next) + for (FetchItem *I = Queue; I != 0 && I != QueueBack; I = I->Next) MaxSizeInQueue = std::max(MaxSizeInQueue, I->MaximumSize); return MaxSizeInQueue; } -- cgit v1.2.3 From 180b693262d71381d650d10c3f95a5a70553f40f Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 8 Oct 2014 11:35:48 +0200 Subject: methods/rsh.cc: replace strcat with std::string Instead of using strcat use a C++ std::string to avoid overflowing this buffer. Thanks to David Garfield Closes: #76442 --- methods/rsh.cc | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) (limited to 'methods') diff --git a/methods/rsh.cc b/methods/rsh.cc index bd46d2515..0e949160b 100644 --- a/methods/rsh.cc +++ b/methods/rsh.cc @@ -218,17 +218,20 @@ bool RSHConn::WriteMsg(std::string &Text,bool Sync,const char *Fmt,...) va_list args; va_start(args,Fmt); - // sprintf the description - char S[512]; - vsnprintf(S,sizeof(S) - 4,Fmt,args); + // sprintf into a buffer + char Tmp[1024]; + vsnprintf(Tmp,sizeof(Tmp),Fmt,args); va_end(args); + // concat to create the real msg + std::string Msg; if (Sync == true) - strcat(S," 2> /dev/null || echo\n"); + Msg = std::string(Tmp) + " 2> /dev/null || echo\n"; else - strcat(S," 2> /dev/null\n"); + Msg = std::string(Tmp) + " 2> /dev/null\n"; // Send it off + const char *S = Msg.c_str(); unsigned long Len = strlen(S); unsigned long Start = 0; while (Len != 0) -- cgit v1.2.3 From 9983999d294887046abf386adc31190700d89b61 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 13 Oct 2014 10:57:30 +0200 Subject: Fix backward compatiblity of the new pkgAcquireMethod::DropPrivsOrDie() Do not drop privileges in the methods when using a older version of libapt that does not support the chown magic in partial/ yet. To do this DropPrivileges() now will ignore a empty Apt::Sandbox::User. Cleanup all hardcoded _apt along the way. --- methods/ftp.cc | 7 ++++--- methods/gpgv.cc | 14 +++++++++++--- methods/gzip.cc | 12 ++++++++++-- methods/http.cc | 2 ++ methods/http_main.cc | 1 - methods/https.cc | 12 ++++++++++-- methods/https.h | 2 ++ 7 files changed, 39 insertions(+), 11 deletions(-) (limited to 'methods') diff --git a/methods/ftp.cc b/methods/ftp.cc index 5b739ea06..0504e5872 100644 --- a/methods/ftp.cc +++ b/methods/ftp.cc @@ -988,6 +988,10 @@ bool FtpMethod::Configuration(string Message) return false; TimeOut = _config->FindI("Acquire::Ftp::Timeout",TimeOut); + + // no more active ftp, sorry + DropPrivsOrDie(); + return true; } /*}}}*/ @@ -1141,8 +1145,5 @@ int main(int, const char *argv[]) FtpMethod Mth; - // no more active ftp, sorry - Mth.DropPrivsOrDie(); - return Mth.Run(); } diff --git a/methods/gpgv.cc b/methods/gpgv.cc index 7e8500c51..488c16826 100644 --- a/methods/gpgv.cc +++ b/methods/gpgv.cc @@ -44,12 +44,22 @@ class GPGVMethod : public pkgAcqMethod protected: virtual bool Fetch(FetchItem *Itm); - + virtual bool Configuration(string Message); public: GPGVMethod() : pkgAcqMethod("1.0",SingleInstance | SendConfig) {}; }; +bool GPGVMethod::Configuration(string Message) +{ + if (pkgAcqMethod::Configuration(Message) == false) + return false; + + DropPrivsOrDie(); + + return true; +} + string GPGVMethod::VerifyGetSigners(const char *file, const char *outfile, vector &GoodSigners, vector &BadSigners, @@ -265,7 +275,5 @@ int main() GPGVMethod Mth; - Mth.DropPrivsOrDie(); - return Mth.Run(); } diff --git a/methods/gzip.cc b/methods/gzip.cc index 7ffcda60f..387c05f2e 100644 --- a/methods/gzip.cc +++ b/methods/gzip.cc @@ -33,12 +33,22 @@ const char *Prog; class GzipMethod : public pkgAcqMethod { virtual bool Fetch(FetchItem *Itm); + virtual bool Configuration(std::string Message); public: GzipMethod() : pkgAcqMethod("1.1",SingleInstance | SendConfig) {}; }; +bool GzipMethod::Configuration(std::string Message) +{ + if (pkgAcqMethod::Configuration(Message) == false) + return false; + + DropPrivsOrDie(); + + return true; +} // GzipMethod::Fetch - Decompress the passed URI /*{{{*/ // --------------------------------------------------------------------- @@ -140,7 +150,5 @@ int main(int, char *argv[]) GzipMethod Mth; - Mth.DropPrivsOrDie(); - return Mth.Run(); } diff --git a/methods/http.cc b/methods/http.cc index c00b439b7..a5de13511 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -770,6 +770,8 @@ bool HttpMethod::Configuration(string Message) if (ServerMethod::Configuration(Message) == false) return false; + DropPrivsOrDie(); + AllowRedirect = _config->FindB("Acquire::http::AllowRedirect",true); PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth", PipelineDepth); diff --git a/methods/http_main.cc b/methods/http_main.cc index f21a5709c..cd52c42e8 100644 --- a/methods/http_main.cc +++ b/methods/http_main.cc @@ -15,6 +15,5 @@ int main() HttpMethod Mth; - Mth.DropPrivsOrDie(); return Mth.Loop(); } diff --git a/methods/https.cc b/methods/https.cc index 16d564b34..366148e19 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -37,6 +37,16 @@ /*}}}*/ using namespace std; +bool HttpsMethod::Configuration(std::string Message) +{ + if (pkgAcqMethod::Configuration(Message) == false) + return false; + + DropPrivsOrDie(); + + return true; +} + size_t HttpsMethod::parse_header(void *buffer, size_t size, size_t nmemb, void *userp) { @@ -452,8 +462,6 @@ int main() HttpsMethod Mth; curl_global_init(CURL_GLOBAL_SSL) ; - Mth.DropPrivsOrDie(); - return Mth.Run(); } diff --git a/methods/https.h b/methods/https.h index 0387cb9b5..9df18e83a 100644 --- a/methods/https.h +++ b/methods/https.h @@ -58,6 +58,8 @@ class HttpsMethod : public pkgAcqMethod static const int DL_MIN_SPEED = 10; virtual bool Fetch(FetchItem *); + virtual bool Configuration(std::string Message); + static size_t parse_header(void *buffer, size_t size, size_t nmemb, void *userp); static size_t write_data(void *buffer, size_t size, size_t nmemb, void *userp); static int progress_callback(void *clientp, double dltotal, double dlnow, -- cgit v1.2.3 From 9dd940ed722e8235c615e79b7eb688eb427e9a23 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 5 Nov 2014 18:33:07 +0100 Subject: Assert statement calls a function which may have desired side effects: 'pos_is_okay' It does not have any desired sideeffect, so we just mark it as const to properly advertise this fact to developer, compiler and linter alike. Reported-By: cppcheck Git-Dch: Ignore --- methods/rred.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'methods') diff --git a/methods/rred.cc b/methods/rred.cc index cabb3c456..774b58a40 100644 --- a/methods/rred.cc +++ b/methods/rred.cc @@ -150,11 +150,11 @@ class FileChanges { std::list::iterator where; size_t pos; // line number is as far left of iterator as possible - bool pos_is_okay(void) + bool pos_is_okay(void) const { #ifdef POSDEBUG size_t cpos = 0; - std::list::iterator x; + std::list::const_iterator x; for (x = changes.begin(); x != where; ++x) { assert(x != changes.end()); cpos += x->offset + x->add_cnt; -- cgit v1.2.3 From bf6ac7ca615922c23d1f3cf1963efc5be9c23e32 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 9 Nov 2014 15:57:43 +0100 Subject: use getline() instead of rolling our own We use it in other places already as well even though it is farly new addition to the POSIX family with 2008, but rolling our own here is really something which should be avoided in such a important method. Git-Dch: Ignore --- methods/gpgv.cc | 31 +++++-------------------------- 1 file changed, 5 insertions(+), 26 deletions(-) (limited to 'methods') diff --git a/methods/gpgv.cc b/methods/gpgv.cc index 488c16826..41f138be6 100644 --- a/methods/gpgv.cc +++ b/methods/gpgv.cc @@ -86,33 +86,12 @@ string GPGVMethod::VerifyGetSigners(const char *file, const char *outfile, FILE *pipein = fdopen(fd[0], "r"); // Loop over the output of apt-key (which really is gnupg), and check the signatures. - size_t buffersize = 64; - char *buffer = (char *) malloc(buffersize); - size_t bufferoff = 0; + size_t buffersize = 0; + char *buffer = NULL; while (1) { - int c; - - // Read a line. Sigh. - while ((c = getc(pipein)) != EOF && c != '\n') - { - if (bufferoff == buffersize) - { - char* newBuffer = (char *) realloc(buffer, buffersize *= 2); - if (newBuffer == NULL) - { - free(buffer); - return "Couldn't allocate a buffer big enough for reading"; - } - buffer = newBuffer; - } - *(buffer+bufferoff) = c; - bufferoff++; - } - if (bufferoff == 0 && c == EOF) - break; - *(buffer+bufferoff) = '\0'; - bufferoff = 0; + if (getline(&buffer, &buffersize, pipein) == -1) + break; if (Debug == true) std::clog << "Read: " << buffer << std::endl; @@ -126,7 +105,7 @@ string GPGVMethod::VerifyGetSigners(const char *file, const char *outfile, std::clog << "Got BADSIG! " << std::endl; BadSigners.push_back(string(buffer+sizeof(GNUPGPREFIX))); } - + if (strncmp(buffer, GNUPGNOPUBKEY, sizeof(GNUPGNOPUBKEY)-1) == 0) { if (Debug == true) -- cgit v1.2.3 From ed793a19ec00b83254029509bc516e3ba911c75a Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 29 Nov 2014 17:59:52 +0100 Subject: dispose http(s) 416 error page as non-content MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Real webservers (like apache) actually send an error page with a 416 response, but our client didn't expect it leaving the page on the socket to be parsed as response for the next request (http) or as file content (https), which isn't what we want at all… Symptom is a "Bad header line" as html usually doesn't parse that well to an http-header. This manifests itself e.g. if we have a complete file (or larger) in partial/ which isn't discarded by If-Range as the server doesn't support it (or it is just newer, think: mirror rotation). It is a sort-of regression of 78c72d0ce22e00b194251445aae306df357d5c1a, which removed the filesize - 1 trick, but this had its own problems… To properly test this our webserver gains the ability to reply with transfer-encoding: chunked as most real webservers will use it to send the dynamically generated error pages. Closes: 768797 --- methods/http.cc | 2 ++ methods/https.cc | 15 ++++++++++++--- methods/server.cc | 26 +++++++++++++++----------- methods/server.h | 5 +++-- 4 files changed, 32 insertions(+), 16 deletions(-) (limited to 'methods') diff --git a/methods/http.cc b/methods/http.cc index a5de13511..ad1347d36 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -444,6 +444,8 @@ bool HttpServerState::RunData(FileFd * const File) loss of the connection means we are done */ if (Encoding == Closes) In.Limit(-1); + else if (JunkSize != 0) + In.Limit(JunkSize); else In.Limit(Size - StartPos); diff --git a/methods/https.cc b/methods/https.cc index 366148e19..23b3a10d4 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -69,6 +69,9 @@ HttpsMethod::parse_header(void *buffer, size_t size, size_t nmemb, void *userp) { me->Server->Result = 200; me->Server->StartPos = me->Server->Size; + // the actual size is not important for https as curl will deal with it + // by itself and e.g. doesn't bother us with transport-encoding… + me->Server->JunkSize = std::numeric_limits::max(); } else me->Server->StartPos = 0; @@ -86,19 +89,25 @@ size_t HttpsMethod::write_data(void *buffer, size_t size, size_t nmemb, void *userp) { HttpsMethod *me = (HttpsMethod *)userp; + size_t buffer_size = size * nmemb; + // we don't need to count the junk here, just drop anything we get as + // we don't always know how long it would be, e.g. in chunked encoding. + if (me->Server->JunkSize != 0) + return buffer_size; if (me->Res.Size == 0) me->URIStart(me->Res); - if(me->File->Write(buffer, size*nmemb) != true) + if(me->File->Write(buffer, buffer_size) != true) return false; if(me->Queue->MaximumSize > 0 && me->File->Tell() > me->Queue->MaximumSize) { me->SetFailReason("MaximumSizeExceeded"); - return _error->Error("Writing more data than expected (%llu > %llu)", + _error->Error("Writing more data than expected (%llu > %llu)", me->TotalWritten, me->Queue->MaximumSize); + return 0; } - return size*nmemb; + return buffer_size; } int diff --git a/methods/server.cc b/methods/server.cc index c4689ff12..9b3d39cf2 100644 --- a/methods/server.cc +++ b/methods/server.cc @@ -55,6 +55,7 @@ ServerState::RunHeadersResult ServerState::RunHeaders(FileFd * const File, Minor = 0; Result = 0; Size = 0; + JunkSize = 0; StartPos = 0; Encoding = Closes; HaveContent = false; @@ -163,14 +164,14 @@ bool ServerState::HeaderLine(string Line) Encoding = Stream; HaveContent = true; - // The length is already set from the Content-Range header - if (StartPos != 0) - return true; + unsigned long long * SizePtr = &Size; + if (Result == 416) + SizePtr = &JunkSize; - Size = strtoull(Val.c_str(), NULL, 10); - if (Size >= std::numeric_limits::max()) + *SizePtr = strtoull(Val.c_str(), NULL, 10); + if (*SizePtr >= std::numeric_limits::max()) return _error->Errno("HeaderLine", _("The HTTP server sent an invalid Content-Length header")); - else if (Size == 0) + else if (*SizePtr == 0) HaveContent = false; return true; } @@ -187,10 +188,7 @@ bool ServerState::HeaderLine(string Line) // §14.16 says 'byte-range-resp-spec' should be a '*' in case of 416 if (Result == 416 && sscanf(Val.c_str(), "bytes */%llu",&Size) == 1) - { - StartPos = 1; // ignore Content-Length, it would override Size - HaveContent = false; - } + ; // we got the expected filesize which is all we wanted else if (sscanf(Val.c_str(),"bytes %llu-%*u/%llu",&StartPos,&Size) != 2) return _error->Error(_("The HTTP server sent an invalid Content-Range header")); if ((unsigned long long)StartPos > Size) @@ -308,9 +306,15 @@ ServerMethod::DealWithHeaders(FetchResult &Res) if ((unsigned long long)SBuf.st_size == Server->Size) { // the file is completely downloaded, but was not moved + if (Server->HaveContent == true) + { + // Send to error page to dev/null + FileFd DevNull("/dev/null",FileFd::WriteExists); + Server->RunData(&DevNull); + } + Server->HaveContent = false; Server->StartPos = Server->Size; Server->Result = 200; - Server->HaveContent = false; } else if (unlink(Queue->DestFile.c_str()) == 0) { diff --git a/methods/server.h b/methods/server.h index 7d5198478..b974ec89a 100644 --- a/methods/server.h +++ b/methods/server.h @@ -34,7 +34,8 @@ struct ServerState char Code[360]; // These are some statistics from the last parsed header lines - unsigned long long Size; + unsigned long long Size; // size of the usable content (aka: the file) + unsigned long long JunkSize; // size of junk content (aka: server error pages) unsigned long long StartPos; time_t Date; bool HaveContent; @@ -73,7 +74,7 @@ struct ServerState RunHeadersResult RunHeaders(FileFd * const File, const std::string &Uri); bool Comp(URI Other) const {return Other.Host == ServerName.Host && Other.Port == ServerName.Port;}; - virtual void Reset() {Major = 0; Minor = 0; Result = 0; Code[0] = '\0'; Size = 0; + virtual void Reset() {Major = 0; Minor = 0; Result = 0; Code[0] = '\0'; Size = 0; JunkSize = 0; StartPos = 0; Encoding = Closes; time(&Date); HaveContent = false; State = Header; Persistent = false; Pipeline = true; MaximumSize = 0;}; virtual bool WriteResponse(std::string const &Data) = 0; -- cgit v1.2.3 From 92e8c1ff287ab829de825e00cdf94744e699ff97 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 29 Nov 2014 17:59:52 +0100 Subject: dispose http(s) 416 error page as non-content MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Real webservers (like apache) actually send an error page with a 416 response, but our client didn't expect it leaving the page on the socket to be parsed as response for the next request (http) or as file content (https), which isn't what we want at all… Symptom is a "Bad header line" as html usually doesn't parse that well to an http-header. This manifests itself e.g. if we have a complete file (or larger) in partial/ which isn't discarded by If-Range as the server doesn't support it (or it is just newer, think: mirror rotation). It is a sort-of regression of 78c72d0ce22e00b194251445aae306df357d5c1a, which removed the filesize - 1 trick, but this had its own problems… To properly test this our webserver gains the ability to reply with transfer-encoding: chunked as most real webservers will use it to send the dynamically generated error pages. (The tests and their binary helpers had to be slightly modified to apply, but the patch to fix the issue itself is unchanged.) Closes: 768797 --- methods/http.cc | 2 ++ methods/https.cc | 12 ++++++++++-- methods/server.cc | 26 +++++++++++++++----------- methods/server.h | 5 +++-- 4 files changed, 30 insertions(+), 15 deletions(-) (limited to 'methods') diff --git a/methods/http.cc b/methods/http.cc index f2a4a4db6..1b996db98 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -440,6 +440,8 @@ bool HttpServerState::RunData(FileFd * const File) loss of the connection means we are done */ if (Encoding == Closes) In.Limit(-1); + else if (JunkSize != 0) + In.Limit(JunkSize); else In.Limit(Size - StartPos); diff --git a/methods/https.cc b/methods/https.cc index 0499af0c5..65a744e2a 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -59,6 +59,9 @@ HttpsMethod::parse_header(void *buffer, size_t size, size_t nmemb, void *userp) { me->Server->Result = 200; me->Server->StartPos = me->Server->Size; + // the actual size is not important for https as curl will deal with it + // by itself and e.g. doesn't bother us with transport-encoding… + me->Server->JunkSize = std::numeric_limits::max(); } else me->Server->StartPos = 0; @@ -76,13 +79,18 @@ size_t HttpsMethod::write_data(void *buffer, size_t size, size_t nmemb, void *userp) { HttpsMethod *me = (HttpsMethod *)userp; + size_t buffer_size = size * nmemb; + // we don't need to count the junk here, just drop anything we get as + // we don't always know how long it would be, e.g. in chunked encoding. + if (me->Server->JunkSize != 0) + return buffer_size; if (me->Res.Size == 0) me->URIStart(me->Res); - if(me->File->Write(buffer, size*nmemb) != true) + if(me->File->Write(buffer, buffer_size) != true) return false; - return size*nmemb; + return buffer_size; } int diff --git a/methods/server.cc b/methods/server.cc index 92d94e638..cb0341d5f 100644 --- a/methods/server.cc +++ b/methods/server.cc @@ -55,6 +55,7 @@ ServerState::RunHeadersResult ServerState::RunHeaders(FileFd * const File, Minor = 0; Result = 0; Size = 0; + JunkSize = 0; StartPos = 0; Encoding = Closes; HaveContent = false; @@ -163,14 +164,14 @@ bool ServerState::HeaderLine(string Line) Encoding = Stream; HaveContent = true; - // The length is already set from the Content-Range header - if (StartPos != 0) - return true; + unsigned long long * SizePtr = &Size; + if (Result == 416) + SizePtr = &JunkSize; - Size = strtoull(Val.c_str(), NULL, 10); - if (Size >= std::numeric_limits::max()) + *SizePtr = strtoull(Val.c_str(), NULL, 10); + if (*SizePtr >= std::numeric_limits::max()) return _error->Errno("HeaderLine", _("The HTTP server sent an invalid Content-Length header")); - else if (Size == 0) + else if (*SizePtr == 0) HaveContent = false; return true; } @@ -187,10 +188,7 @@ bool ServerState::HeaderLine(string Line) // §14.16 says 'byte-range-resp-spec' should be a '*' in case of 416 if (Result == 416 && sscanf(Val.c_str(), "bytes */%llu",&Size) == 1) - { - StartPos = 1; // ignore Content-Length, it would override Size - HaveContent = false; - } + ; // we got the expected filesize which is all we wanted else if (sscanf(Val.c_str(),"bytes %llu-%*u/%llu",&StartPos,&Size) != 2) return _error->Error(_("The HTTP server sent an invalid Content-Range header")); if ((unsigned long long)StartPos > Size) @@ -308,9 +306,15 @@ ServerMethod::DealWithHeaders(FetchResult &Res) if ((unsigned long long)SBuf.st_size == Server->Size) { // the file is completely downloaded, but was not moved + if (Server->HaveContent == true) + { + // Send to error page to dev/null + FileFd DevNull("/dev/null",FileFd::WriteExists); + Server->RunData(&DevNull); + } + Server->HaveContent = false; Server->StartPos = Server->Size; Server->Result = 200; - Server->HaveContent = false; } else if (unlink(Queue->DestFile.c_str()) == 0) { diff --git a/methods/server.h b/methods/server.h index f5e68d902..1b81e3549 100644 --- a/methods/server.h +++ b/methods/server.h @@ -34,7 +34,8 @@ struct ServerState char Code[360]; // These are some statistics from the last parsed header lines - unsigned long long Size; + unsigned long long Size; // size of the usable content (aka: the file) + unsigned long long JunkSize; // size of junk content (aka: server error pages) unsigned long long StartPos; time_t Date; bool HaveContent; @@ -71,7 +72,7 @@ struct ServerState RunHeadersResult RunHeaders(FileFd * const File, const std::string &Uri); bool Comp(URI Other) const {return Other.Host == ServerName.Host && Other.Port == ServerName.Port;}; - virtual void Reset() {Major = 0; Minor = 0; Result = 0; Code[0] = '\0'; Size = 0; + virtual void Reset() {Major = 0; Minor = 0; Result = 0; Code[0] = '\0'; Size = 0; JunkSize = 0; StartPos = 0; Encoding = Closes; time(&Date); HaveContent = false; State = Header; Persistent = false; Pipeline = true;}; virtual bool WriteResponse(std::string const &Data) = 0; -- cgit v1.2.3 From 9127d7aecf01f2999a2589e4b0503288518b2927 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 5 Jan 2015 10:27:53 +0100 Subject: Fix missing URIStart() for https downloads Add a explicit ReceivedData to HttpsMethod that indicates when we got data from the connection so that we can send URISTart() to the parent. This is needed because URIStart got moved in f9b4f12d from the progress_callback to write_data() and it only checks for Res.Size. In the old code if progress_callback is called by libcurl (and sets Res.Size) before write_data is called then URIStart() is never send. Making this a explicit ReceivedData variable fixes this issue. --- methods/https.cc | 9 +++++++-- methods/https.h | 1 + 2 files changed, 8 insertions(+), 2 deletions(-) (limited to 'methods') diff --git a/methods/https.cc b/methods/https.cc index 65a744e2a..3a5981b58 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -85,8 +85,12 @@ HttpsMethod::write_data(void *buffer, size_t size, size_t nmemb, void *userp) if (me->Server->JunkSize != 0) return buffer_size; - if (me->Res.Size == 0) + if (me->ReceivedData == false) + { me->URIStart(me->Res); + me->ReceivedData = true; + } + if(me->File->Write(buffer, buffer_size) != true) return false; @@ -95,7 +99,7 @@ HttpsMethod::write_data(void *buffer, size_t size, size_t nmemb, void *userp) int HttpsMethod::progress_callback(void *clientp, double dltotal, double /*dlnow*/, - double /*ultotal*/, double /*ulnow*/) + double /*ultotal*/, double /*ulnow*/) { HttpsMethod *me = (HttpsMethod *)clientp; if(dltotal > 0 && me->Res.Size == 0) { @@ -179,6 +183,7 @@ bool HttpsMethod::Fetch(FetchItem *Itm) char curl_errorstr[CURL_ERROR_SIZE]; URI Uri = Itm->Uri; string remotehost = Uri.Host; + ReceivedData = false; // TODO: // - http::Pipeline-Depth diff --git a/methods/https.h b/methods/https.h index faac8a3cd..411b71440 100644 --- a/methods/https.h +++ b/methods/https.h @@ -66,6 +66,7 @@ class HttpsMethod : public pkgAcqMethod CURL *curl; FetchResult Res; HttpsServerState *Server; + bool ReceivedData; public: FileFd *File; -- cgit v1.2.3 From 0c2dc43d4fe1d026650b5e2920a021557f9534a6 Mon Sep 17 00:00:00 2001 From: Tomasz Buchert Date: Mon, 16 Feb 2015 00:57:29 +0100 Subject: Fix crash in the apt-transport-https when Owner is NULL Do not crash in ServerState::HeaderLine if there is no Owner. Closes: #778375 --- methods/server.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'methods') diff --git a/methods/server.cc b/methods/server.cc index cb0341d5f..e321e0230 100644 --- a/methods/server.cc +++ b/methods/server.cc @@ -129,7 +129,7 @@ bool ServerState::HeaderLine(string Line) if (elements == 3) { Code[0] = '\0'; - if (Owner->Debug == true) + if (Owner != NULL && Owner->Debug == true) clog << "HTTP server doesn't give Reason-Phrase for " << Result << std::endl; } else if (elements != 4) -- cgit v1.2.3 From 905fba60a046646a26a56b4c5d4a5dc7d5906f0d Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Mon, 9 Mar 2015 01:54:46 +0100 Subject: derive more of https from http method Bug #778375 uncovered that https wasn't properly integrated in the class family tree of http as it was supposed to be leading to a NULL pointer dereference. Fixing this 'properly' was deemed to much diff for practically no gain that late in the release, so commit 0c2dc43d4fe1d026650b5e2920a021557f9534a6 just fixed the synptom, while this commit here is fixing the cause plus adding a test. --- methods/http.cc | 2 -- methods/https.cc | 37 ++++++++++++++++++++++--------------- methods/https.h | 16 ++++++++++------ methods/server.cc | 7 ++++++- methods/server.h | 3 ++- 5 files changed, 40 insertions(+), 25 deletions(-) (limited to 'methods') diff --git a/methods/http.cc b/methods/http.cc index ad1347d36..021b284d0 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -772,8 +772,6 @@ bool HttpMethod::Configuration(string Message) if (ServerMethod::Configuration(Message) == false) return false; - DropPrivsOrDie(); - AllowRedirect = _config->FindB("Acquire::http::AllowRedirect",true); PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth", PipelineDepth); diff --git a/methods/https.cc b/methods/https.cc index 37a8ff5fd..32de42e4b 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -37,16 +37,6 @@ /*}}}*/ using namespace std; -bool HttpsMethod::Configuration(std::string Message) -{ - if (pkgAcqMethod::Configuration(Message) == false) - return false; - - DropPrivsOrDie(); - - return true; -} - size_t HttpsMethod::parse_header(void *buffer, size_t size, size_t nmemb, void *userp) { @@ -131,7 +121,7 @@ HttpsMethod::progress_callback(void *clientp, double dltotal, double /*dlnow*/, } // HttpsServerState::HttpsServerState - Constructor /*{{{*/ -HttpsServerState::HttpsServerState(URI Srv,HttpsMethod * /*Owner*/) : ServerState(Srv, NULL) +HttpsServerState::HttpsServerState(URI Srv,HttpsMethod * Owner) : ServerState(Srv, Owner) { TimeOut = _config->FindI("Acquire::https::Timeout",TimeOut); ReceivedData = false; @@ -335,13 +325,11 @@ bool HttpsMethod::Fetch(FetchItem *Itm) curl_easy_setopt(curl, CURLOPT_LOW_SPEED_TIME, timeout); // set redirect options and default to 10 redirects - bool const AllowRedirect = _config->FindB("Acquire::https::AllowRedirect", - _config->FindB("Acquire::http::AllowRedirect",true)); curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, AllowRedirect); curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 10); // debug - if(_config->FindB("Debug::Acquire::https", false)) + if (Debug == true) curl_easy_setopt(curl, CURLOPT_VERBOSE, true); // error handling @@ -378,7 +366,7 @@ bool HttpsMethod::Fetch(FetchItem *Itm) // go for it - if the file exists, append on it File = new FileFd(Itm->DestFile, FileFd::WriteAny); - Server = new HttpsServerState(Itm->Uri, this); + Server = CreateServerState(Itm->Uri); // keep apt updated Res.Filename = Itm->DestFile; @@ -473,6 +461,25 @@ bool HttpsMethod::Fetch(FetchItem *Itm) return true; } + /*}}}*/ +// HttpsMethod::Configuration - Handle a configuration message /*{{{*/ +bool HttpsMethod::Configuration(string Message) +{ + if (ServerMethod::Configuration(Message) == false) + return false; + + AllowRedirect = _config->FindB("Acquire::https::AllowRedirect", + _config->FindB("Acquire::http::AllowRedirect", true)); + Debug = _config->FindB("Debug::Acquire::https",false); + + return true; +} + /*}}}*/ +ServerState * HttpsMethod::CreateServerState(URI uri) /*{{{*/ +{ + return new HttpsServerState(uri, this); +} + /*}}}*/ int main() { diff --git a/methods/https.h b/methods/https.h index 6917a6ff6..433a84680 100644 --- a/methods/https.h +++ b/methods/https.h @@ -50,17 +50,14 @@ class HttpsServerState : public ServerState HttpsServerState(URI Srv, HttpsMethod *Owner); virtual ~HttpsServerState() {Close();}; - - bool ReceivedData; }; -class HttpsMethod : public pkgAcqMethod +class HttpsMethod : public ServerMethod { // minimum speed in bytes/se that triggers download timeout handling static const int DL_MIN_SPEED = 10; virtual bool Fetch(FetchItem *); - virtual bool Configuration(std::string Message); static size_t parse_header(void *buffer, size_t size, size_t nmemb, void *userp); static size_t write_data(void *buffer, size_t size, size_t nmemb, void *userp); @@ -69,12 +66,19 @@ class HttpsMethod : public pkgAcqMethod void SetupProxy(); CURL *curl; FetchResult Res; - HttpsServerState *Server; + ServerState *Server; + + // Used by ServerMethods unused by https + virtual void SendReq(FetchItem *) { exit(42); } + virtual void RotateDNS() { exit(42); } public: FileFd *File; - HttpsMethod() : pkgAcqMethod("1.2",Pipeline | SendConfig), Server(NULL), File(NULL) + virtual bool Configuration(std::string Message); + virtual ServerState * CreateServerState(URI uri); + + HttpsMethod() : ServerMethod("1.2",Pipeline | SendConfig), File(NULL) { curl = curl_easy_init(); }; diff --git a/methods/server.cc b/methods/server.cc index c17f27f73..91ec824d1 100644 --- a/methods/server.cc +++ b/methods/server.cc @@ -238,7 +238,12 @@ ServerState::ServerState(URI Srv, ServerMethod *Owner) : ServerName(Srv), TimeOu bool ServerMethod::Configuration(string Message) /*{{{*/ { - return pkgAcqMethod::Configuration(Message); + if (pkgAcqMethod::Configuration(Message) == false) + return false; + + DropPrivsOrDie(); + + return true; } /*}}}*/ diff --git a/methods/server.h b/methods/server.h index b974ec89a..3b232dcac 100644 --- a/methods/server.h +++ b/methods/server.h @@ -37,6 +37,7 @@ struct ServerState unsigned long long Size; // size of the usable content (aka: the file) unsigned long long JunkSize; // size of junk content (aka: server error pages) unsigned long long StartPos; + bool ReceivedData; time_t Date; bool HaveContent; enum {Chunked,Stream,Closes} Encoding; @@ -75,7 +76,7 @@ struct ServerState bool Comp(URI Other) const {return Other.Host == ServerName.Host && Other.Port == ServerName.Port;}; virtual void Reset() {Major = 0; Minor = 0; Result = 0; Code[0] = '\0'; Size = 0; JunkSize = 0; - StartPos = 0; Encoding = Closes; time(&Date); HaveContent = false; + StartPos = 0; ReceivedData = false; Encoding = Closes; time(&Date); HaveContent = false; State = Header; Persistent = false; Pipeline = true; MaximumSize = 0;}; virtual bool WriteResponse(std::string const &Data) = 0; -- cgit v1.2.3 From 1296bc7c466181a7978c313c40a041b34ce3eaeb Mon Sep 17 00:00:00 2001 From: Robert Edmonds Date: Sun, 22 Mar 2015 00:12:45 -0400 Subject: HttpsMethod::Fetch(): Zero the FetchResult object when leaving due to 404 --- methods/https.cc | 2 ++ 1 file changed, 2 insertions(+) (limited to 'methods') diff --git a/methods/https.cc b/methods/https.cc index 3a5981b58..f2b00dd64 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -402,6 +402,8 @@ bool HttpsMethod::Fetch(FetchItem *Itm) _error->Error("%s", err); // unlink, no need keep 401/404 page content in partial/ unlink(File->Name().c_str()); + Res.Size = 0; + Res.LastModified = 0; return false; } -- cgit v1.2.3 From a2679f55b5b14092db88fab3799f06e6b68e439e Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 7 Apr 2015 14:34:04 +0200 Subject: properly handle expected filesize in https The worker expects that the methods tell him when they start or finish downloading a file. Various information pieces are passed along in this report including the (expected) filesize. https is using a "global" struct for reporting which made it 'reuse' incorrect values in some cases like a non-existent InRelease fallbacking to Release{,.gpg} resulting in an incorrect size-mismatch warning scaring and desensitizing users as well as being subject to a race between the write_data and progress callbacks generating incorrect progress reporting and potentially the same error message. Other branches as well as the bugreports contain 'better' fixes making the struct local and other sensible changes, but are larger as a result, so in this version we opted for short diff with minimal effect above else instead. Closes: 777565, 781509 Thanks: Robert Edmonds and Anders Kaseorg for initial patchs --- methods/https.cc | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) (limited to 'methods') diff --git a/methods/https.cc b/methods/https.cc index 3a5981b58..cb11159bc 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -68,6 +68,8 @@ HttpsMethod::parse_header(void *buffer, size_t size, size_t nmemb, void *userp) me->File->Truncate(me->Server->StartPos); me->File->Seek(me->Server->StartPos); + + me->Res.Size = me->Server->Size; } else if (me->Server->HeaderLine(line) == false) return 0; @@ -97,17 +99,6 @@ HttpsMethod::write_data(void *buffer, size_t size, size_t nmemb, void *userp) return buffer_size; } -int -HttpsMethod::progress_callback(void *clientp, double dltotal, double /*dlnow*/, - double /*ultotal*/, double /*ulnow*/) -{ - HttpsMethod *me = (HttpsMethod *)clientp; - if(dltotal > 0 && me->Res.Size == 0) { - me->Res.Size = (unsigned long long)dltotal; - } - return 0; -} - // HttpsServerState::HttpsServerState - Constructor /*{{{*/ HttpsServerState::HttpsServerState(URI Srv,HttpsMethod * /*Owner*/) : ServerState(Srv, NULL) { @@ -201,10 +192,8 @@ bool HttpsMethod::Fetch(FetchItem *Itm) curl_easy_setopt(curl, CURLOPT_WRITEHEADER, this); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data); curl_easy_setopt(curl, CURLOPT_WRITEDATA, this); - curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, progress_callback); - curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, this); // options - curl_easy_setopt(curl, CURLOPT_NOPROGRESS, false); + curl_easy_setopt(curl, CURLOPT_NOPROGRESS, true); curl_easy_setopt(curl, CURLOPT_FILETIME, true); // only allow curl to handle https, not the other stuff it supports curl_easy_setopt(curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTPS); @@ -357,6 +346,7 @@ bool HttpsMethod::Fetch(FetchItem *Itm) // go for it - if the file exists, append on it File = new FileFd(Itm->DestFile, FileFd::WriteAny); Server = new HttpsServerState(Itm->Uri, this); + Res = FetchResult(); // keep apt updated Res.Filename = Itm->DestFile; -- cgit v1.2.3 From b8eba208daebe3e3f235983e44da9c398d6f7a57 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 10 Mar 2015 14:11:54 +0100 Subject: reimplement the last uses of sprintf Working with strings c-style is complicated and error-prune, so by converting to c++ style we gain some simplicity and avoid buffer overflows by later extensions. Git-Dch: Ignore --- methods/ftp.cc | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) (limited to 'methods') diff --git a/methods/ftp.cc b/methods/ftp.cc index 0504e5872..7764acf6a 100644 --- a/methods/ftp.cc +++ b/methods/ftp.cc @@ -259,19 +259,21 @@ bool FTPConn::Login() { if (Opts->Value.empty() == true) continue; - + // Substitute the variables into the command - char SitePort[20]; - if (ServerName.Port != 0) - sprintf(SitePort,"%u",ServerName.Port); - else - strcpy(SitePort,"21"); string Tmp = Opts->Value; Tmp = SubstVar(Tmp,"$(PROXY_USER)",Proxy.User); Tmp = SubstVar(Tmp,"$(PROXY_PASS)",Proxy.Password); Tmp = SubstVar(Tmp,"$(SITE_USER)",User); Tmp = SubstVar(Tmp,"$(SITE_PASS)",Pass); - Tmp = SubstVar(Tmp,"$(SITE_PORT)",SitePort); + if (ServerName.Port != 0) + { + std::string SitePort; + strprintf(SitePort, "%u", ServerName.Port); + Tmp = SubstVar(Tmp,"$(SITE_PORT)", SitePort); + } + else + Tmp = SubstVar(Tmp,"$(SITE_PORT)", "21"); Tmp = SubstVar(Tmp,"$(SITE)",ServerName.Host); // Send the command -- cgit v1.2.3 From 3a53f6a1510d332e24c3330a69b987f2341d1a94 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 13 Apr 2015 12:57:22 -0400 Subject: Revert "HttpsMethod::Fetch(): Zero the FetchResult object when leaving due to 404" This reverts commit 1296bc7c466181a7978c313c40a041b34ce3eaeb. --- methods/https.cc | 2 -- 1 file changed, 2 deletions(-) (limited to 'methods') diff --git a/methods/https.cc b/methods/https.cc index f2b00dd64..3a5981b58 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -402,8 +402,6 @@ bool HttpsMethod::Fetch(FetchItem *Itm) _error->Error("%s", err); // unlink, no need keep 401/404 page content in partial/ unlink(File->Name().c_str()); - Res.Size = 0; - Res.LastModified = 0; return false; } -- cgit v1.2.3 From bb948ef562862e5cc9fcfb3d7b5e41c70382adeb Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 27 Mar 2015 11:14:44 +0100 Subject: do not unlink https file on general error It might be quite interesting which file (content) made curl freak out and other methods keep the file around as well. Git-Dch: Ignore --- methods/https.cc | 1 - 1 file changed, 1 deletion(-) (limited to 'methods') diff --git a/methods/https.cc b/methods/https.cc index 32de42e4b..c69e84d3a 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -386,7 +386,6 @@ bool HttpsMethod::Fetch(FetchItem *Itm) if (success != 0) { _error->Error("%s", curl_errorstr); - unlink(File->Name().c_str()); return false; } -- cgit v1.2.3 From 27925d82dd0cbae74d48040363fe6f6c2bae5215 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 27 Mar 2015 15:53:43 +0100 Subject: improve https method queue progress reporting The worker expects that the methods tell him when they start or finish downloading a file. Various information pieces are passed along in this report including the (expected) filesize. https was using a "global" struct for reporting which made it 'reuse' incorrect values in some cases like a non-existent InRelease fallbacking to Release{,.gpg} resulting in a size-mismatch warning. Reducing the scope and redesigning the setting of the values we can fix this and related issues. Closes: 777565, 781509 Thanks: Robert Edmonds and Anders Kaseorg for initial patchs --- methods/https.cc | 82 +++++++++++++++++++++++++++----------------------------- methods/https.h | 2 +- methods/server.h | 3 +-- 3 files changed, 41 insertions(+), 46 deletions(-) (limited to 'methods') diff --git a/methods/https.cc b/methods/https.cc index c69e84d3a..70f6a1046 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -37,11 +37,17 @@ /*}}}*/ using namespace std; +struct APT_HIDDEN CURLUserPointer { + HttpsMethod * const https; + HttpsMethod::FetchResult * const Res; + CURLUserPointer(HttpsMethod * const https, HttpsMethod::FetchResult * const Res) : https(https), Res(Res) {} +}; + size_t HttpsMethod::parse_header(void *buffer, size_t size, size_t nmemb, void *userp) { size_t len = size * nmemb; - HttpsMethod *me = (HttpsMethod *)userp; + CURLUserPointer *me = (CURLUserPointer *)userp; std::string line((char*) buffer, len); for (--len; len > 0; --len) if (isspace(line[len]) == 0) @@ -53,23 +59,33 @@ HttpsMethod::parse_header(void *buffer, size_t size, size_t nmemb, void *userp) if (line.empty() == true) { - if (me->Server->Result != 416 && me->Server->StartPos != 0) + if (me->https->Server->Result != 416 && me->https->Server->StartPos != 0) ; - else if (me->Server->Result == 416 && me->Server->Size == me->File->FileSize()) + else if (me->https->Server->Result == 416 && me->https->Server->Size == me->https->File->FileSize()) { - me->Server->Result = 200; - me->Server->StartPos = me->Server->Size; + me->https->Server->Result = 200; + me->https->Server->StartPos = me->https->Server->Size; // the actual size is not important for https as curl will deal with it // by itself and e.g. doesn't bother us with transport-encoding… - me->Server->JunkSize = std::numeric_limits::max(); + me->https->Server->JunkSize = std::numeric_limits::max(); } else - me->Server->StartPos = 0; + me->https->Server->StartPos = 0; + + me->https->File->Truncate(me->https->Server->StartPos); + me->https->File->Seek(me->https->Server->StartPos); + + me->Res->LastModified = me->https->Server->Date; + me->Res->Size = me->https->Server->Size; + me->Res->ResumePoint = me->https->Server->StartPos; - me->File->Truncate(me->Server->StartPos); - me->File->Seek(me->Server->StartPos); + // we expect valid data, so tell our caller we get the file now + if (me->https->Server->Result >= 200 && me->https->Server->Result < 300 && + me->https->Server->JunkSize == 0 && + me->Res->Size != 0 && me->Res->Size > me->Res->ResumePoint) + me->https->URIStart(*me->Res); } - else if (me->Server->HeaderLine(line) == false) + else if (me->https->Server->HeaderLine(line) == false) return 0; return size*nmemb; @@ -85,12 +101,6 @@ HttpsMethod::write_data(void *buffer, size_t size, size_t nmemb, void *userp) if (me->Server->JunkSize != 0) return buffer_size; - if (me->Server->ReceivedData == false) - { - me->URIStart(me->Res); - me->Server->ReceivedData = true; - } - if(me->File->Write(buffer, buffer_size) != true) return 0; @@ -109,27 +119,15 @@ HttpsMethod::write_data(void *buffer, size_t size, size_t nmemb, void *userp) return buffer_size; } -int -HttpsMethod::progress_callback(void *clientp, double dltotal, double /*dlnow*/, - double /*ultotal*/, double /*ulnow*/) -{ - HttpsMethod *me = (HttpsMethod *)clientp; - if(dltotal > 0 && me->Res.Size == 0) { - me->Res.Size = (unsigned long long)dltotal; - } - return 0; -} - // HttpsServerState::HttpsServerState - Constructor /*{{{*/ HttpsServerState::HttpsServerState(URI Srv,HttpsMethod * Owner) : ServerState(Srv, Owner) { TimeOut = _config->FindI("Acquire::https::Timeout",TimeOut); - ReceivedData = false; Reset(); } /*}}}*/ -void HttpsMethod::SetupProxy() /*{{{*/ +void HttpsMethod::SetupProxy() /*{{{*/ { URI ServerName = Queue->Uri; @@ -207,16 +205,16 @@ bool HttpsMethod::Fetch(FetchItem *Itm) maybe_add_auth (Uri, _config->FindFile("Dir::Etc::netrc")); + FetchResult Res; + CURLUserPointer userp(this, &Res); // callbacks curl_easy_setopt(curl, CURLOPT_URL, static_cast(Uri).c_str()); curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, parse_header); - curl_easy_setopt(curl, CURLOPT_WRITEHEADER, this); + curl_easy_setopt(curl, CURLOPT_WRITEHEADER, &userp); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data); curl_easy_setopt(curl, CURLOPT_WRITEDATA, this); - curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, progress_callback); - curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, this); // options - curl_easy_setopt(curl, CURLOPT_NOPROGRESS, false); + curl_easy_setopt(curl, CURLOPT_NOPROGRESS, true); curl_easy_setopt(curl, CURLOPT_FILETIME, true); // only allow curl to handle https, not the other stuff it supports curl_easy_setopt(curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTPS); @@ -414,24 +412,23 @@ bool HttpsMethod::Fetch(FetchItem *Itm) return false; } - struct stat resultStat; - if (unlikely(stat(File->Name().c_str(), &resultStat) != 0)) - { - _error->Errno("stat", "Unable to access file %s", File->Name().c_str()); - return false; - } - Res.Size = resultStat.st_size; - // invalid range-request if (Server->Result == 416) { unlink(File->Name().c_str()); - Res.Size = 0; delete File; Redirect(Itm->Uri); return true; } + struct stat resultStat; + if (unlikely(stat(File->Name().c_str(), &resultStat) != 0)) + { + _error->Errno("stat", "Unable to access file %s", File->Name().c_str()); + return false; + } + Res.Size = resultStat.st_size; + // Timestamp curl_easy_getinfo(curl, CURLINFO_FILETIME, &Res.LastModified); if (Res.LastModified != -1) @@ -455,7 +452,6 @@ bool HttpsMethod::Fetch(FetchItem *Itm) URIDone(Res); // cleanup - Res.Size = 0; delete File; return true; diff --git a/methods/https.h b/methods/https.h index 433a84680..4cc48fc34 100644 --- a/methods/https.h +++ b/methods/https.h @@ -65,7 +65,6 @@ class HttpsMethod : public ServerMethod double ultotal, double ulnow); void SetupProxy(); CURL *curl; - FetchResult Res; ServerState *Server; // Used by ServerMethods unused by https @@ -77,6 +76,7 @@ class HttpsMethod : public ServerMethod virtual bool Configuration(std::string Message); virtual ServerState * CreateServerState(URI uri); + using pkgAcqMethod::FetchResult; HttpsMethod() : ServerMethod("1.2",Pipeline | SendConfig), File(NULL) { diff --git a/methods/server.h b/methods/server.h index 3b232dcac..b974ec89a 100644 --- a/methods/server.h +++ b/methods/server.h @@ -37,7 +37,6 @@ struct ServerState unsigned long long Size; // size of the usable content (aka: the file) unsigned long long JunkSize; // size of junk content (aka: server error pages) unsigned long long StartPos; - bool ReceivedData; time_t Date; bool HaveContent; enum {Chunked,Stream,Closes} Encoding; @@ -76,7 +75,7 @@ struct ServerState bool Comp(URI Other) const {return Other.Host == ServerName.Host && Other.Port == ServerName.Port;}; virtual void Reset() {Major = 0; Minor = 0; Result = 0; Code[0] = '\0'; Size = 0; JunkSize = 0; - StartPos = 0; ReceivedData = false; Encoding = Closes; time(&Date); HaveContent = false; + StartPos = 0; Encoding = Closes; time(&Date); HaveContent = false; State = Header; Persistent = false; Pipeline = true; MaximumSize = 0;}; virtual bool WriteResponse(std::string const &Data) = 0; -- cgit v1.2.3 From 936d5613e4b6145798c5a1d0c484158115576fa8 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 27 Mar 2015 18:56:44 +0100 Subject: remove duplicated check for same file copy Git-Dch: Ignore --- methods/copy.cc | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) (limited to 'methods') diff --git a/methods/copy.cc b/methods/copy.cc index a23c0316c..f3cd84215 100644 --- a/methods/copy.cc +++ b/methods/copy.cc @@ -65,17 +65,9 @@ bool CopyMethod::Fetch(FetchItem *Itm) Res.Size = Buf.st_size; Res.Filename = Itm->DestFile; Res.LastModified = Buf.st_mtime; - Res.IMSHit = false; + Res.IMSHit = false; URIStart(Res); - // when the files are identical, just compute the hashes - if(File == Itm->DestFile) - { - CalculateHashes(Res); - URIDone(Res); - return true; - } - // just calc the hashes if the source and destination are identical if (File == Itm->DestFile) { -- cgit v1.2.3 From a09f6eb8fc67cd2d836019f448f18580396185e5 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 27 Mar 2015 19:59:44 +0100 Subject: send Alt-* info for uncompressed based on any compressions file sends information about the uncompressed file if it can find it as well as for the compressed file. This was done only for gzip so far, but we support more compression types. That this information isn't used a lot is a different story. Git-Dch: Ignore --- methods/file.cc | 43 ++++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 19 deletions(-) (limited to 'methods') diff --git a/methods/file.cc b/methods/file.cc index 12db62203..5d9d7b951 100644 --- a/methods/file.cc +++ b/methods/file.cc @@ -16,6 +16,7 @@ #include #include +#include #include #include #include @@ -33,7 +34,7 @@ class FileMethod : public pkgAcqMethod public: - FileMethod() : pkgAcqMethod("1.0",SingleInstance | LocalOnly) {}; + FileMethod() : pkgAcqMethod("1.0",SingleInstance | SendConfig | LocalOnly) {}; }; // FileMethod::Fetch - Fetch a file /*{{{*/ @@ -58,27 +59,31 @@ bool FileMethod::Fetch(FetchItem *Itm) if (Itm->LastModified == Buf.st_mtime && Itm->LastModified != 0) Res.IMSHit = true; } - - // See if we can compute a file without a .gz exentsion - std::string::size_type Pos = File.rfind(".gz"); - if (Pos + 3 == File.length()) + + // See if the uncompressed file exists and reuse it + std::vector extensions = APT::Configuration::getCompressorExtensions(); + for (std::vector::const_iterator ext = extensions.begin(); ext != extensions.end(); ++ext) { - File = std::string(File,0,Pos); - if (stat(File.c_str(),&Buf) == 0) + if (APT::String::Endswith(File, *ext) == true) { - FetchResult AltRes; - AltRes.Size = Buf.st_size; - AltRes.Filename = File; - AltRes.LastModified = Buf.st_mtime; - AltRes.IMSHit = false; - if (Itm->LastModified == Buf.st_mtime && Itm->LastModified != 0) - AltRes.IMSHit = true; - - URIDone(Res,&AltRes); - return true; - } + std::string const unfile = File.substr(0, File.length() - ext->length() - 1); + if (stat(unfile.c_str(),&Buf) == 0) + { + FetchResult AltRes; + AltRes.Size = Buf.st_size; + AltRes.Filename = unfile; + AltRes.LastModified = Buf.st_mtime; + AltRes.IMSHit = false; + if (Itm->LastModified == Buf.st_mtime && Itm->LastModified != 0) + AltRes.IMSHit = true; + + URIDone(Res,&AltRes); + return true; + } + // no break here as we could have situations similar to '.gz' vs '.tar.gz' here + } } - + if (Res.Filename.empty() == true) return _error->Error(_("File not found")); -- cgit v1.2.3 From 117038bac90261351518870b3f48136f134d4bfc Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Mon, 30 Mar 2015 19:52:32 +0200 Subject: handle servers closing encoded connections correctly Servers who advertise that they close the connection get the 'Closes' encoding flag, but this conflicts with servers who response with a transfer-encoding (e.g. encoding) as it is saved in the same flag. We have a better flag for the keep-alive (or not) of the connection anyway, so we check this instead of the encoding. This is in practice not much of a problem as real servers we talk to are HTTP1.1 servers (with keep-alive) and there isn't much point in doing chunked encoding if you are going to close anyway, but our simple testserver stumbles over this if pressed and its a bit cleaner, too. Git-Dch: Ignore --- methods/http.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'methods') diff --git a/methods/http.cc b/methods/http.cc index 021b284d0..947002cc6 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -442,7 +442,7 @@ bool HttpServerState::RunData(FileFd * const File) { /* Closes encoding is used when the server did not specify a size, the loss of the connection means we are done */ - if (Encoding == Closes) + if (Persistent == false) In.Limit(-1); else if (JunkSize != 0) In.Limit(JunkSize); @@ -524,7 +524,7 @@ bool HttpServerState::Die(FileFd &File) // See if this is because the server finished the data stream if (In.IsLimit() == false && State != HttpServerState::Header && - Encoding != HttpServerState::Closes) + Persistent == true) { Close(); if (LErrno == 0) @@ -571,7 +571,7 @@ bool HttpServerState::Flush(FileFd * const File) return true; } - if (In.IsLimit() == true || Encoding == ServerState::Closes) + if (In.IsLimit() == true || Persistent == false) return true; } return false; -- cgit v1.2.3 From 9224ce3d4d1ea0428a70e75134998e08aa45b1e6 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Mon, 30 Mar 2015 20:47:13 +0200 Subject: calculate only expected hashes in methods Methods get told which hashes are expected by the acquire system, which means we can use this list to restrict what we calculate in the methods as any extra we are calculating is wasted effort as we can't compare it with anything anyway. Adding support for a new hash algorithm is therefore 'free' now and if a algorithm is no longer provided in a repository for a file, we automatically stop calculating it. In practice this results in a speed-up in Debian as we don't have SHA512 here (so far), so we practically stop calculating it. --- methods/cdrom.cc | 2 +- methods/copy.cc | 10 +++++----- methods/file.cc | 2 +- methods/ftp.cc | 2 +- methods/gzip.cc | 2 +- methods/http.cc | 14 +++++++------- methods/http.h | 2 +- methods/https.cc | 2 +- methods/https.h | 2 +- methods/rred.cc | 2 +- methods/rsh.cc | 2 +- methods/server.cc | 2 +- methods/server.h | 2 +- 13 files changed, 23 insertions(+), 23 deletions(-) (limited to 'methods') diff --git a/methods/cdrom.cc b/methods/cdrom.cc index 74e2ecc6b..10cb29f66 100644 --- a/methods/cdrom.cc +++ b/methods/cdrom.cc @@ -266,7 +266,7 @@ bool CDROMMethod::Fetch(FetchItem *Itm) Res.LastModified = Buf.st_mtime; Res.Size = Buf.st_size; - Hashes Hash; + Hashes Hash(Itm->ExpectedHashes); FileFd Fd(Res.Filename, FileFd::ReadOnly); Hash.AddFD(Fd); Res.TakeHashes(Hash); diff --git a/methods/copy.cc b/methods/copy.cc index f3cd84215..a8e289df5 100644 --- a/methods/copy.cc +++ b/methods/copy.cc @@ -28,16 +28,16 @@ class CopyMethod : public pkgAcqMethod { virtual bool Fetch(FetchItem *Itm); - void CalculateHashes(FetchResult &Res); + void CalculateHashes(FetchItem const * const Itm, FetchResult &Res); public: CopyMethod() : pkgAcqMethod("1.0",SingleInstance | SendConfig) {}; }; -void CopyMethod::CalculateHashes(FetchResult &Res) +void CopyMethod::CalculateHashes(FetchItem const * const Itm, FetchResult &Res) { - Hashes Hash; + Hashes Hash(Itm->ExpectedHashes); FileFd::CompressMode CompressMode = FileFd::None; if (_config->FindB("Acquire::GzipIndexes", false) == true) CompressMode = FileFd::Extension; @@ -71,7 +71,7 @@ bool CopyMethod::Fetch(FetchItem *Itm) // just calc the hashes if the source and destination are identical if (File == Itm->DestFile) { - CalculateHashes(Res); + CalculateHashes(Itm, Res); URIDone(Res); return true; } @@ -104,7 +104,7 @@ bool CopyMethod::Fetch(FetchItem *Itm) if (utimes(Res.Filename.c_str(), times) != 0) return _error->Errno("utimes",_("Failed to set modification time")); - CalculateHashes(Res); + CalculateHashes(Itm, Res); URIDone(Res); return true; diff --git a/methods/file.cc b/methods/file.cc index 5d9d7b951..043ab04b8 100644 --- a/methods/file.cc +++ b/methods/file.cc @@ -87,7 +87,7 @@ bool FileMethod::Fetch(FetchItem *Itm) if (Res.Filename.empty() == true) return _error->Error(_("File not found")); - Hashes Hash; + Hashes Hash(Itm->ExpectedHashes); FileFd Fd(Res.Filename, FileFd::ReadOnly); Hash.AddFD(Fd); Res.TakeHashes(Hash); diff --git a/methods/ftp.cc b/methods/ftp.cc index 7764acf6a..92d8573f1 100644 --- a/methods/ftp.cc +++ b/methods/ftp.cc @@ -1064,7 +1064,7 @@ bool FtpMethod::Fetch(FetchItem *Itm) } // Open the file - Hashes Hash; + Hashes Hash(Itm->ExpectedHashes); { FileFd Fd(Itm->DestFile,FileFd::WriteAny); if (_error->PendingError() == true) diff --git a/methods/gzip.cc b/methods/gzip.cc index 387c05f2e..65519633c 100644 --- a/methods/gzip.cc +++ b/methods/gzip.cc @@ -91,7 +91,7 @@ bool GzipMethod::Fetch(FetchItem *Itm) return false; // Read data from source, generate checksums and write - Hashes Hash; + Hashes Hash(Itm->ExpectedHashes); bool Failed = false; while (1) { diff --git a/methods/http.cc b/methods/http.cc index 947002cc6..e4773b0e2 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -64,8 +64,8 @@ const unsigned int CircleBuf::BW_HZ=10; // CircleBuf::CircleBuf - Circular input buffer /*{{{*/ // --------------------------------------------------------------------- /* */ -CircleBuf::CircleBuf(unsigned long long Size) - : Size(Size), Hash(0), TotalWriten(0) +CircleBuf::CircleBuf(unsigned long long Size) + : Size(Size), Hash(NULL), TotalWriten(0) { Buf = new unsigned char[Size]; Reset(); @@ -84,10 +84,10 @@ void CircleBuf::Reset() TotalWriten = 0; MaxGet = (unsigned long long)-1; OutQueue = string(); - if (Hash != 0) + if (Hash != NULL) { delete Hash; - Hash = new Hashes; + Hash = NULL; } } /*}}}*/ @@ -222,7 +222,7 @@ bool CircleBuf::Write(int Fd) TotalWriten += Res; - if (Hash != 0) + if (Hash != NULL) Hash->Add(Buf + (OutP%Size),Res); OutP += Res; @@ -484,10 +484,10 @@ APT_PURE bool HttpServerState::IsOpen() /*{{{*/ return (ServerFd != -1); } /*}}}*/ -bool HttpServerState::InitHashes(FileFd &File) /*{{{*/ +bool HttpServerState::InitHashes(FileFd &File, HashStringList const &ExpectedHashes)/*{{{*/ { delete In.Hash; - In.Hash = new Hashes; + In.Hash = new Hashes(ExpectedHashes); // Set the expected size and read file for the hashes File.Truncate(StartPos); diff --git a/methods/http.h b/methods/http.h index 40a88a7be..6dc872659 100644 --- a/methods/http.h +++ b/methods/http.h @@ -111,7 +111,7 @@ struct HttpServerState: public ServerState virtual bool Open(); virtual bool IsOpen(); virtual bool Close(); - virtual bool InitHashes(FileFd &File); + virtual bool InitHashes(FileFd &File, HashStringList const &ExpectedHashes); virtual Hashes * GetHashes(); virtual bool Die(FileFd &File); virtual bool Flush(FileFd * const File); diff --git a/methods/https.cc b/methods/https.cc index 70f6a1046..81903b239 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -443,7 +443,7 @@ bool HttpsMethod::Fetch(FetchItem *Itm) Res.LastModified = resultStat.st_mtime; // take hashes - Hashes Hash; + Hashes Hash(Itm->ExpectedHashes); FileFd Fd(Res.Filename, FileFd::ReadOnly); Hash.AddFD(Fd); Res.TakeHashes(Hash); diff --git a/methods/https.h b/methods/https.h index 4cc48fc34..dc0ff3322 100644 --- a/methods/https.h +++ b/methods/https.h @@ -42,7 +42,7 @@ class HttpsServerState : public ServerState virtual bool Open() { return false; } virtual bool IsOpen() { return false; } virtual bool Close() { return false; } - virtual bool InitHashes(FileFd &/*File*/) { return false; } + virtual bool InitHashes(FileFd &/*File*/, HashStringList const &/*ExpectedHashes*/) { return false; } virtual Hashes * GetHashes() { return NULL; } virtual bool Die(FileFd &/*File*/) { return false; } virtual bool Flush(FileFd * const /*File*/) { return false; } diff --git a/methods/rred.cc b/methods/rred.cc index 774b58a40..554ac99b4 100644 --- a/methods/rred.cc +++ b/methods/rred.cc @@ -581,7 +581,7 @@ class RredMethod : public pkgAcqMethod { FILE *inp = fopen(Path.c_str(), "r"); FILE *out = fopen(Itm->DestFile.c_str(), "w"); - Hashes hash; + Hashes hash(Itm->ExpectedHashes); patch.apply_against_file(out, inp, &hash); diff --git a/methods/rsh.cc b/methods/rsh.cc index 0e949160b..52349c61c 100644 --- a/methods/rsh.cc +++ b/methods/rsh.cc @@ -477,7 +477,7 @@ bool RSHMethod::Fetch(FetchItem *Itm) } // Open the file - Hashes Hash; + Hashes Hash(Itm->ExpectedHashes); { FileFd Fd(Itm->DestFile,FileFd::WriteAny); if (_error->PendingError() == true) diff --git a/methods/server.cc b/methods/server.cc index 91ec824d1..e403f1071 100644 --- a/methods/server.cc +++ b/methods/server.cc @@ -357,7 +357,7 @@ ServerMethod::DealWithHeaders(FetchResult &Res) FailFd = File->Fd(); FailTime = Server->Date; - if (Server->InitHashes(*File) == false) + if (Server->InitHashes(*File, Queue->ExpectedHashes) == false) { _error->Errno("read",_("Problem hashing file")); return ERROR_NOT_FROM_SERVER; diff --git a/methods/server.h b/methods/server.h index b974ec89a..45622dd34 100644 --- a/methods/server.h +++ b/methods/server.h @@ -85,7 +85,7 @@ struct ServerState virtual bool Open() = 0; virtual bool IsOpen() = 0; virtual bool Close() = 0; - virtual bool InitHashes(FileFd &File) = 0; + virtual bool InitHashes(FileFd &File, HashStringList const &ExpectedHashes) = 0; virtual Hashes * GetHashes() = 0; virtual bool Die(FileFd &File) = 0; virtual bool Flush(FileFd * const File) = 0; -- cgit v1.2.3 From 34faa8f7ae2526f46cd1f84bb6962ad06d841e5e Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 11 Apr 2015 10:23:52 +0200 Subject: calculate hashes while downloading in https MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We do this in HTTP already to give the CPU some exercise while the disk is heavily spinning (or flashing?) to store the data avoiding the need to reread the entire file again later on to calculate the hashes – which happens outside of the eyes of progress reporting, so you might ended up with a bunch of https workers 'stuck' at 100% while they were busy calculating hashes. This is a bummer for everyone using apt as a connection speedtest as the https method works slower now (not really, it just isn't reporting done too early anymore). --- methods/http.cc | 8 +++----- methods/http.h | 2 +- methods/https.cc | 38 ++++++++++++++++++++++++++------------ methods/https.h | 6 ++++-- methods/server.cc | 8 +++++++- methods/server.h | 3 ++- 6 files changed, 43 insertions(+), 22 deletions(-) (limited to 'methods') diff --git a/methods/http.cc b/methods/http.cc index e4773b0e2..af3d5ccb6 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -484,16 +484,14 @@ APT_PURE bool HttpServerState::IsOpen() /*{{{*/ return (ServerFd != -1); } /*}}}*/ -bool HttpServerState::InitHashes(FileFd &File, HashStringList const &ExpectedHashes)/*{{{*/ +bool HttpServerState::InitHashes(HashStringList const &ExpectedHashes) /*{{{*/ { delete In.Hash; In.Hash = new Hashes(ExpectedHashes); - - // Set the expected size and read file for the hashes - File.Truncate(StartPos); - return In.Hash->AddFD(File, StartPos); + return true; } /*}}}*/ + APT_PURE Hashes * HttpServerState::GetHashes() /*{{{*/ { return In.Hash; diff --git a/methods/http.h b/methods/http.h index 6dc872659..e73871931 100644 --- a/methods/http.h +++ b/methods/http.h @@ -111,7 +111,7 @@ struct HttpServerState: public ServerState virtual bool Open(); virtual bool IsOpen(); virtual bool Close(); - virtual bool InitHashes(FileFd &File, HashStringList const &ExpectedHashes); + virtual bool InitHashes(HashStringList const &ExpectedHashes); virtual Hashes * GetHashes(); virtual bool Die(FileFd &File); virtual bool Flush(FileFd * const File); diff --git a/methods/https.cc b/methods/https.cc index 81903b239..c6b75d9ad 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -72,18 +72,18 @@ HttpsMethod::parse_header(void *buffer, size_t size, size_t nmemb, void *userp) else me->https->Server->StartPos = 0; - me->https->File->Truncate(me->https->Server->StartPos); - me->https->File->Seek(me->https->Server->StartPos); - me->Res->LastModified = me->https->Server->Date; me->Res->Size = me->https->Server->Size; me->Res->ResumePoint = me->https->Server->StartPos; // we expect valid data, so tell our caller we get the file now - if (me->https->Server->Result >= 200 && me->https->Server->Result < 300 && - me->https->Server->JunkSize == 0 && - me->Res->Size != 0 && me->Res->Size > me->Res->ResumePoint) - me->https->URIStart(*me->Res); + if (me->https->Server->Result >= 200 && me->https->Server->Result < 300) + { + if (me->https->Server->JunkSize == 0 && me->Res->Size != 0 && me->Res->Size > me->Res->ResumePoint) + me->https->URIStart(*me->Res); + if (me->https->Server->AddPartialFileToHashes(*(me->https->File)) == false) + return 0; + } } else if (me->https->Server->HeaderLine(line) == false) return 0; @@ -116,16 +116,31 @@ HttpsMethod::write_data(void *buffer, size_t size, size_t nmemb, void *userp) } } + if (me->Server->GetHashes()->Add((unsigned char const * const)buffer, buffer_size) == false) + return 0; + return buffer_size; } // HttpsServerState::HttpsServerState - Constructor /*{{{*/ -HttpsServerState::HttpsServerState(URI Srv,HttpsMethod * Owner) : ServerState(Srv, Owner) +HttpsServerState::HttpsServerState(URI Srv,HttpsMethod * Owner) : ServerState(Srv, Owner), Hash(NULL) { TimeOut = _config->FindI("Acquire::https::Timeout",TimeOut); Reset(); } /*}}}*/ +bool HttpsServerState::InitHashes(HashStringList const &ExpectedHashes) /*{{{*/ +{ + delete Hash; + Hash = new Hashes(ExpectedHashes); + return true; +} + /*}}}*/ +APT_PURE Hashes * HttpsServerState::GetHashes() /*{{{*/ +{ + return Hash; +} + /*}}}*/ void HttpsMethod::SetupProxy() /*{{{*/ { @@ -365,6 +380,8 @@ bool HttpsMethod::Fetch(FetchItem *Itm) // go for it - if the file exists, append on it File = new FileFd(Itm->DestFile, FileFd::WriteAny); Server = CreateServerState(Itm->Uri); + if (Server->InitHashes(Itm->ExpectedHashes) == false) + return false; // keep apt updated Res.Filename = Itm->DestFile; @@ -443,10 +460,7 @@ bool HttpsMethod::Fetch(FetchItem *Itm) Res.LastModified = resultStat.st_mtime; // take hashes - Hashes Hash(Itm->ExpectedHashes); - FileFd Fd(Res.Filename, FileFd::ReadOnly); - Hash.AddFD(Fd); - Res.TakeHashes(Hash); + Res.TakeHashes(*(Server->GetHashes())); // keep apt updated URIDone(Res); diff --git a/methods/https.h b/methods/https.h index dc0ff3322..6e32e8d3d 100644 --- a/methods/https.h +++ b/methods/https.h @@ -29,6 +29,8 @@ class FileFd; class HttpsServerState : public ServerState { + Hashes * Hash; + protected: virtual bool ReadHeaderLines(std::string &/*Data*/) { return false; } virtual bool LoadNextResponse(bool const /*ToFile*/, FileFd * const /*File*/) { return false; } @@ -42,8 +44,8 @@ class HttpsServerState : public ServerState virtual bool Open() { return false; } virtual bool IsOpen() { return false; } virtual bool Close() { return false; } - virtual bool InitHashes(FileFd &/*File*/, HashStringList const &/*ExpectedHashes*/) { return false; } - virtual Hashes * GetHashes() { return NULL; } + virtual bool InitHashes(HashStringList const &ExpectedHashes); + virtual Hashes * GetHashes(); virtual bool Die(FileFd &/*File*/) { return false; } virtual bool Flush(FileFd * const /*File*/) { return false; } virtual bool Go(bool /*ToFile*/, FileFd * const /*File*/) { return false; } diff --git a/methods/server.cc b/methods/server.cc index e403f1071..2116926b0 100644 --- a/methods/server.cc +++ b/methods/server.cc @@ -235,6 +235,12 @@ ServerState::ServerState(URI Srv, ServerMethod *Owner) : ServerName(Srv), TimeOu Reset(); } /*}}}*/ +bool ServerState::AddPartialFileToHashes(FileFd &File) /*{{{*/ +{ + File.Truncate(StartPos); + return GetHashes()->AddFD(File, StartPos); +} + /*}}}*/ bool ServerMethod::Configuration(string Message) /*{{{*/ { @@ -357,7 +363,7 @@ ServerMethod::DealWithHeaders(FetchResult &Res) FailFd = File->Fd(); FailTime = Server->Date; - if (Server->InitHashes(*File, Queue->ExpectedHashes) == false) + if (Server->InitHashes(Queue->ExpectedHashes) == false || Server->AddPartialFileToHashes(*File) == false) { _error->Errno("read",_("Problem hashing file")); return ERROR_NOT_FROM_SERVER; diff --git a/methods/server.h b/methods/server.h index 45622dd34..1b1f754a3 100644 --- a/methods/server.h +++ b/methods/server.h @@ -72,6 +72,7 @@ struct ServerState }; /** \brief Get the headers before the data */ RunHeadersResult RunHeaders(FileFd * const File, const std::string &Uri); + bool AddPartialFileToHashes(FileFd &File); bool Comp(URI Other) const {return Other.Host == ServerName.Host && Other.Port == ServerName.Port;}; virtual void Reset() {Major = 0; Minor = 0; Result = 0; Code[0] = '\0'; Size = 0; JunkSize = 0; @@ -85,7 +86,7 @@ struct ServerState virtual bool Open() = 0; virtual bool IsOpen() = 0; virtual bool Close() = 0; - virtual bool InitHashes(FileFd &File, HashStringList const &ExpectedHashes) = 0; + virtual bool InitHashes(HashStringList const &ExpectedHashes) = 0; virtual Hashes * GetHashes() = 0; virtual bool Die(FileFd &File) = 0; virtual bool Flush(FileFd * const File) = 0; -- cgit v1.2.3 From dcbb364fc69e1108b3fea3adb12a7ba83d9af467 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 12 May 2015 00:30:16 +0200 Subject: detect 416 complete file in partial by expected hash If we have the expected hashes we can check with them if the file we have in partial we got a 416 for is the expected file. We detected this with same-size before, but not every server sends a good Content-Range header with a 416 response. --- methods/https.cc | 37 +++++++++++++++++++++++++++++-------- methods/https.h | 1 + methods/server.cc | 19 ++++++++++++++++--- 3 files changed, 46 insertions(+), 11 deletions(-) (limited to 'methods') diff --git a/methods/https.cc b/methods/https.cc index c6b75d9ad..712e9ee73 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -40,7 +40,9 @@ using namespace std; struct APT_HIDDEN CURLUserPointer { HttpsMethod * const https; HttpsMethod::FetchResult * const Res; - CURLUserPointer(HttpsMethod * const https, HttpsMethod::FetchResult * const Res) : https(https), Res(Res) {} + HttpsMethod::FetchItem const * const Itm; + CURLUserPointer(HttpsMethod * const https, HttpsMethod::FetchResult * const Res, + HttpsMethod::FetchItem const * const Itm) : https(https), Res(Res), Itm(Itm) {} }; size_t @@ -61,13 +63,32 @@ HttpsMethod::parse_header(void *buffer, size_t size, size_t nmemb, void *userp) { if (me->https->Server->Result != 416 && me->https->Server->StartPos != 0) ; - else if (me->https->Server->Result == 416 && me->https->Server->Size == me->https->File->FileSize()) + else if (me->https->Server->Result == 416) { - me->https->Server->Result = 200; - me->https->Server->StartPos = me->https->Server->Size; - // the actual size is not important for https as curl will deal with it - // by itself and e.g. doesn't bother us with transport-encoding… - me->https->Server->JunkSize = std::numeric_limits::max(); + bool partialHit = false; + if (me->Itm->ExpectedHashes.usable() == true) + { + Hashes resultHashes(me->Itm->ExpectedHashes); + FileFd file(me->Itm->DestFile, FileFd::ReadOnly); + me->https->Server->Size = file.FileSize(); + me->https->Server->Date = file.ModificationTime(); + resultHashes.AddFD(file); + HashStringList const hashList = resultHashes.GetHashStringList(); + partialHit = (me->Itm->ExpectedHashes == hashList); + } + else if (me->https->Server->Result == 416 && me->https->Server->Size == me->https->File->FileSize()) + partialHit = true; + + if (partialHit == true) + { + me->https->Server->Result = 200; + me->https->Server->StartPos = me->https->Server->Size; + // the actual size is not important for https as curl will deal with it + // by itself and e.g. doesn't bother us with transport-encoding… + me->https->Server->JunkSize = std::numeric_limits::max(); + } + else + me->https->Server->StartPos = 0; } else me->https->Server->StartPos = 0; @@ -221,7 +242,7 @@ bool HttpsMethod::Fetch(FetchItem *Itm) maybe_add_auth (Uri, _config->FindFile("Dir::Etc::netrc")); FetchResult Res; - CURLUserPointer userp(this, &Res); + CURLUserPointer userp(this, &Res, Itm); // callbacks curl_easy_setopt(curl, CURLOPT_URL, static_cast(Uri).c_str()); curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, parse_header); diff --git a/methods/https.h b/methods/https.h index 6e32e8d3d..57fc292ee 100644 --- a/methods/https.h +++ b/methods/https.h @@ -79,6 +79,7 @@ class HttpsMethod : public ServerMethod virtual bool Configuration(std::string Message); virtual ServerState * CreateServerState(URI uri); using pkgAcqMethod::FetchResult; + using pkgAcqMethod::FetchItem; HttpsMethod() : ServerMethod("1.2",Pipeline | SendConfig), File(NULL) { diff --git a/methods/server.cc b/methods/server.cc index 2116926b0..bd01c3e98 100644 --- a/methods/server.cc +++ b/methods/server.cc @@ -269,7 +269,7 @@ ServerMethod::DealWithHeaders(FetchResult &Res) Res.LastModified = Queue->LastModified; return IMS_HIT; } - + /* Redirect * * Note that it is only OK for us to treat all redirection the same @@ -314,7 +314,20 @@ ServerMethod::DealWithHeaders(FetchResult &Res) struct stat SBuf; if (stat(Queue->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0) { - if ((unsigned long long)SBuf.st_size == Server->Size) + bool partialHit = false; + if (Queue->ExpectedHashes.usable() == true) + { + Hashes resultHashes(Queue->ExpectedHashes); + FileFd file(Queue->DestFile, FileFd::ReadOnly); + Server->Size = file.FileSize(); + Server->Date = file.ModificationTime(); + resultHashes.AddFD(file); + HashStringList const hashList = resultHashes.GetHashStringList(); + partialHit = (Queue->ExpectedHashes == hashList); + } + else if ((unsigned long long)SBuf.st_size == Server->Size) + partialHit = true; + if (partialHit == true) { // the file is completely downloaded, but was not moved if (Server->HaveContent == true) @@ -351,7 +364,7 @@ ServerMethod::DealWithHeaders(FetchResult &Res) // This is some sort of 2xx 'data follows' reply Res.LastModified = Server->Date; Res.Size = Server->Size; - + // Open the file delete File; File = new FileFd(Queue->DestFile,FileFd::WriteAny); -- cgit v1.2.3 From 8eafc759544298211cd0bfaa3919afc0fadd47d1 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 13 May 2015 16:09:12 +0200 Subject: detect Releasefile IMS hits even if the server doesn't Not all servers we are talking to support If-Modified-Since and some are not even sending Last-Modified for us, so in an effort to detect such hits we run a hashsum check on the 'old' compared to the 'new' file, we got the hashes for the 'new' already for "free" from the methods anyway and hence just need to calculated the old ones. This allows us to detect hits even with unsupported servers, which in turn means we benefit from all the new hit behavior also here. --- methods/https.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'methods') diff --git a/methods/https.cc b/methods/https.cc index 712e9ee73..fa143439a 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -444,7 +444,7 @@ bool HttpsMethod::Fetch(FetchItem *Itm) char err[255]; snprintf(err, sizeof(err) - 1, "HttpError%i", Server->Result); SetFailReason(err); - _error->Error("%s", err); + _error->Error("%i %s", Server->Result, Server->Code); // unlink, no need keep 401/404 page content in partial/ unlink(File->Name().c_str()); return false; -- cgit v1.2.3 From ceafe8a6edc815df2923ba892894617829e9d3c2 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 22 May 2015 15:28:53 +0200 Subject: Fix endless loop in apt-get update that can cause disk fillup The apt http code parses Content-Length and Content-Range. For both requests the variable "Size" is used and the semantic for this Size is the total file size. However Content-Length is not the entire file size for partital file requests. For servers that send the Content-Range header first and then the Content-Length header this can lead to globbing of Size so that its less than the real file size. This may lead to a subsequent passing of a negative number into the CircleBuf which leads to a endless loop that writes data. Thanks to Anton Blanchard for the analysis and initial patch. LP: #1445239 --- methods/http.cc | 2 +- methods/server.cc | 20 +++++++++++++++----- methods/server.h | 3 ++- 3 files changed, 18 insertions(+), 7 deletions(-) (limited to 'methods') diff --git a/methods/http.cc b/methods/http.cc index 1b996db98..ad90c9891 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -443,7 +443,7 @@ bool HttpServerState::RunData(FileFd * const File) else if (JunkSize != 0) In.Limit(JunkSize); else - In.Limit(Size - StartPos); + In.Limit(DownloadSize); // Just transfer the whole block. do diff --git a/methods/server.cc b/methods/server.cc index e321e0230..ba0a8864b 100644 --- a/methods/server.cc +++ b/methods/server.cc @@ -164,15 +164,22 @@ bool ServerState::HeaderLine(string Line) Encoding = Stream; HaveContent = true; - unsigned long long * SizePtr = &Size; + unsigned long long * DownloadSizePtr = &DownloadSize; if (Result == 416) - SizePtr = &JunkSize; + DownloadSizePtr = &JunkSize; - *SizePtr = strtoull(Val.c_str(), NULL, 10); - if (*SizePtr >= std::numeric_limits::max()) + *DownloadSizePtr = strtoull(Val.c_str(), NULL, 10); + if (*DownloadSizePtr >= std::numeric_limits::max()) return _error->Errno("HeaderLine", _("The HTTP server sent an invalid Content-Length header")); - else if (*SizePtr == 0) + else if (*DownloadSizePtr == 0) HaveContent = false; + + // On partial content (206) the Content-Length less than the real + // size, so do not set it here but leave that to the Content-Range + // header instead + if(Result != 206 && Size == 0) + Size = DownloadSize; + return true; } @@ -193,6 +200,9 @@ bool ServerState::HeaderLine(string Line) return _error->Error(_("The HTTP server sent an invalid Content-Range header")); if ((unsigned long long)StartPos > Size) return _error->Error(_("This HTTP server has broken range support")); + + // figure out what we will download + DownloadSize = Size - StartPos; return true; } diff --git a/methods/server.h b/methods/server.h index 1b81e3549..ed3cb456a 100644 --- a/methods/server.h +++ b/methods/server.h @@ -34,7 +34,8 @@ struct ServerState char Code[360]; // These are some statistics from the last parsed header lines - unsigned long long Size; // size of the usable content (aka: the file) + unsigned long long Size; // total size of the usable content (aka: the file) + unsigned long long DownloadSize; // size we actually download (can be smaller than Size if we have partial content) unsigned long long JunkSize; // size of junk content (aka: server error pages) unsigned long long StartPos; time_t Date; -- cgit v1.2.3 From 6291f60e86718697f261519a6818e1d5ee433216 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 22 May 2015 15:40:18 +0200 Subject: Rename "Size" in ServerState to TotalFileSize The variable "Size" was misleading and caused bug #1445239. To avoid similar issues in the future, rename it to make the meaning more obvious. git-dch: ignore --- methods/https.cc | 4 ++-- methods/server.cc | 20 ++++++++++---------- methods/server.h | 14 ++++++++++---- 3 files changed, 22 insertions(+), 16 deletions(-) (limited to 'methods') diff --git a/methods/https.cc b/methods/https.cc index 3a5981b58..12fc6c70f 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -55,10 +55,10 @@ HttpsMethod::parse_header(void *buffer, size_t size, size_t nmemb, void *userp) { if (me->Server->Result != 416 && me->Server->StartPos != 0) ; - else if (me->Server->Result == 416 && me->Server->Size == me->File->FileSize()) + else if (me->Server->Result == 416 && me->Server->TotalFileSize == me->File->FileSize()) { me->Server->Result = 200; - me->Server->StartPos = me->Server->Size; + me->Server->StartPos = me->Server->TotalFileSize; // the actual size is not important for https as curl will deal with it // by itself and e.g. doesn't bother us with transport-encoding… me->Server->JunkSize = std::numeric_limits::max(); diff --git a/methods/server.cc b/methods/server.cc index ba0a8864b..6c05700a5 100644 --- a/methods/server.cc +++ b/methods/server.cc @@ -54,7 +54,7 @@ ServerState::RunHeadersResult ServerState::RunHeaders(FileFd * const File, Major = 0; Minor = 0; Result = 0; - Size = 0; + TotalFileSize = 0; JunkSize = 0; StartPos = 0; Encoding = Closes; @@ -177,8 +177,8 @@ bool ServerState::HeaderLine(string Line) // On partial content (206) the Content-Length less than the real // size, so do not set it here but leave that to the Content-Range // header instead - if(Result != 206 && Size == 0) - Size = DownloadSize; + if(Result != 206 && TotalFileSize == 0) + TotalFileSize = DownloadSize; return true; } @@ -194,15 +194,15 @@ bool ServerState::HeaderLine(string Line) HaveContent = true; // §14.16 says 'byte-range-resp-spec' should be a '*' in case of 416 - if (Result == 416 && sscanf(Val.c_str(), "bytes */%llu",&Size) == 1) + if (Result == 416 && sscanf(Val.c_str(), "bytes */%llu",&TotalFileSize) == 1) ; // we got the expected filesize which is all we wanted - else if (sscanf(Val.c_str(),"bytes %llu-%*u/%llu",&StartPos,&Size) != 2) + else if (sscanf(Val.c_str(),"bytes %llu-%*u/%llu",&StartPos,&TotalFileSize) != 2) return _error->Error(_("The HTTP server sent an invalid Content-Range header")); - if ((unsigned long long)StartPos > Size) + if ((unsigned long long)StartPos > TotalFileSize) return _error->Error(_("This HTTP server has broken range support")); // figure out what we will download - DownloadSize = Size - StartPos; + DownloadSize = TotalFileSize - StartPos; return true; } @@ -313,7 +313,7 @@ ServerMethod::DealWithHeaders(FetchResult &Res) struct stat SBuf; if (stat(Queue->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0) { - if ((unsigned long long)SBuf.st_size == Server->Size) + if ((unsigned long long)SBuf.st_size == Server->TotalFileSize) { // the file is completely downloaded, but was not moved if (Server->HaveContent == true) @@ -323,7 +323,7 @@ ServerMethod::DealWithHeaders(FetchResult &Res) Server->RunData(&DevNull); } Server->HaveContent = false; - Server->StartPos = Server->Size; + Server->StartPos = Server->TotalFileSize; Server->Result = 200; } else if (unlink(Queue->DestFile.c_str()) == 0) @@ -349,7 +349,7 @@ ServerMethod::DealWithHeaders(FetchResult &Res) // This is some sort of 2xx 'data follows' reply Res.LastModified = Server->Date; - Res.Size = Server->Size; + Res.Size = Server->TotalFileSize; // Open the file delete File; diff --git a/methods/server.h b/methods/server.h index ed3cb456a..8c14282b6 100644 --- a/methods/server.h +++ b/methods/server.h @@ -34,10 +34,16 @@ struct ServerState char Code[360]; // These are some statistics from the last parsed header lines - unsigned long long Size; // total size of the usable content (aka: the file) - unsigned long long DownloadSize; // size we actually download (can be smaller than Size if we have partial content) - unsigned long long JunkSize; // size of junk content (aka: server error pages) + + // total size of the usable content (aka: the file) + unsigned long long TotalFileSize; + // size we actually download (can be smaller than Size if we have partial content) + unsigned long long DownloadSize; + // size of junk content (aka: server error pages) + unsigned long long JunkSize; + // The start of the data (for partial content) unsigned long long StartPos; + time_t Date; bool HaveContent; enum {Chunked,Stream,Closes} Encoding; @@ -73,7 +79,7 @@ struct ServerState RunHeadersResult RunHeaders(FileFd * const File, const std::string &Uri); bool Comp(URI Other) const {return Other.Host == ServerName.Host && Other.Port == ServerName.Port;}; - virtual void Reset() {Major = 0; Minor = 0; Result = 0; Code[0] = '\0'; Size = 0; JunkSize = 0; + virtual void Reset() {Major = 0; Minor = 0; Result = 0; Code[0] = '\0'; TotalFileSize = 0; JunkSize = 0; StartPos = 0; Encoding = Closes; time(&Date); HaveContent = false; State = Header; Persistent = false; Pipeline = true;}; virtual bool WriteResponse(std::string const &Data) = 0; -- cgit v1.2.3 From 65759e00eff0513c34f584b99420b72fe0e5073e Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 22 May 2015 16:27:08 +0200 Subject: Update methods/https.cc now that ServerState::Size is renamed Git-Dch: ignore --- methods/https.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'methods') diff --git a/methods/https.cc b/methods/https.cc index 81060122c..c97367323 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -69,7 +69,7 @@ HttpsMethod::parse_header(void *buffer, size_t size, size_t nmemb, void *userp) me->File->Truncate(me->Server->StartPos); me->File->Seek(me->Server->StartPos); - me->Res.Size = me->Server->Size; + me->Res.Size = me->Server->TotalFileSize; } else if (me->Server->HeaderLine(line) == false) return 0; -- cgit v1.2.3