diff options
Diffstat (limited to 'methods')
-rw-r--r-- | methods/CMakeLists.txt | 8 | ||||
-rw-r--r-- | methods/basehttp.cc (renamed from methods/server.cc) | 200 | ||||
-rw-r--r-- | methods/basehttp.h (renamed from methods/server.h) | 85 | ||||
-rw-r--r-- | methods/connect.cc | 10 | ||||
-rw-r--r-- | methods/ftp.cc | 4 | ||||
-rw-r--r-- | methods/gpgv.cc | 2 | ||||
-rw-r--r-- | methods/http.cc | 26 | ||||
-rw-r--r-- | methods/http.h | 7 | ||||
-rw-r--r-- | methods/https.cc | 113 | ||||
-rw-r--r-- | methods/https.h | 16 | ||||
-rw-r--r-- | methods/rred.cc | 9 |
11 files changed, 247 insertions, 233 deletions
diff --git a/methods/CMakeLists.txt b/methods/CMakeLists.txt index c4d28de64..128c61005 100644 --- a/methods/CMakeLists.txt +++ b/methods/CMakeLists.txt @@ -4,10 +4,10 @@ add_executable(copy copy.cc) add_executable(store store.cc) add_executable(gpgv gpgv.cc) add_executable(cdrom cdrom.cc) -#add_executable(http http.cc http_main.cc rfc2553emu.cc connect.cc server.cc) -add_executable(http http.cc rfc2553emu.cc connect.cc server.cc) -#add_executable(mirror mirror.cc http.cc rfc2553emu.cc connect.cc server.cc) -add_executable(https https.cc server.cc) +#add_executable(http http.cc http_main.cc rfc2553emu.cc connect.cc basehttp.cc) +add_executable(http http.cc rfc2553emu.cc connect.cc basehttp.cc) +#add_executable(mirror mirror.cc http.cc rfc2553emu.cc connect.cc basehttp.cc) +add_executable(https https.cc basehttp.cc) add_executable(ftp ftp.cc rfc2553emu.cc connect.cc) add_executable(rred rred.cc) add_executable(rsh rsh.cc) diff --git a/methods/server.cc b/methods/basehttp.cc index 0408dddfd..f79f26a7c 100644 --- a/methods/server.cc +++ b/methods/basehttp.cc @@ -29,26 +29,24 @@ #include <string> #include <vector> -#include "server.h" +#include "basehttp.h" #include <apti18n.h> /*}}}*/ using namespace std; -string ServerMethod::FailFile; -int ServerMethod::FailFd = -1; -time_t ServerMethod::FailTime = 0; +string BaseHttpMethod::FailFile; +int BaseHttpMethod::FailFd = -1; +time_t BaseHttpMethod::FailTime = 0; // ServerState::RunHeaders - Get the headers before the data /*{{{*/ // --------------------------------------------------------------------- /* Returns 0 if things are OK, 1 if an IO error occurred and 2 if a header parse error occurred */ -ServerState::RunHeadersResult ServerState::RunHeaders(FileFd * const File, +ServerState::RunHeadersResult ServerState::RunHeaders(RequestState &Req, const std::string &Uri) { - Reset(false); Owner->Status(_("Waiting for headers")); - do { string Data; @@ -57,35 +55,32 @@ ServerState::RunHeadersResult ServerState::RunHeaders(FileFd * const File, if (Owner->Debug == true) clog << "Answer for: " << Uri << endl << Data; - + for (string::const_iterator I = Data.begin(); I < Data.end(); ++I) { string::const_iterator J = I; for (; J != Data.end() && *J != '\n' && *J != '\r'; ++J); - if (HeaderLine(string(I,J)) == false) + if (Req.HeaderLine(string(I,J)) == false) return RUN_HEADERS_PARSE_ERROR; I = J; } // 100 Continue is a Nop... - if (Result == 100) + if (Req.Result == 100) continue; // Tidy up the connection persistence state. - if (Encoding == Closes && HaveContent == true) + if (Req.Encoding == RequestState::Closes && Req.HaveContent == true) Persistent = false; return RUN_HEADERS_OK; } - while (LoadNextResponse(false, File) == true); + while (LoadNextResponse(false, Req) == true); return RUN_HEADERS_IO_ERROR; } /*}}}*/ -// ServerState::HeaderLine - Process a header line /*{{{*/ -// --------------------------------------------------------------------- -/* */ -bool ServerState::HeaderLine(string Line) +bool RequestState::HeaderLine(string const &Line) /*{{{*/ { if (Line.empty() == true) return true; @@ -116,18 +111,18 @@ bool ServerState::HeaderLine(string Line) /* Check the HTTP response header to get the default persistence state. */ if (Major < 1) - Persistent = false; + Server->Persistent = false; else { if (Major == 1 && Minor == 0) { - Persistent = false; + Server->Persistent = false; } else { - Persistent = true; - if (PipelineAllowed) - Pipeline = true; + Server->Persistent = true; + if (Server->PipelineAllowed) + Server->Pipeline = true; } } @@ -151,6 +146,9 @@ bool ServerState::HeaderLine(string Line) if (stringcasecmp(Tag,"Content-Length:") == 0) { + auto ContentLength = strtoull(Val.c_str(), NULL, 10); + if (ContentLength == 0) + return true; if (Encoding == Closes) Encoding = Stream; HaveContent = true; @@ -159,7 +157,7 @@ bool ServerState::HeaderLine(string Line) if (Result == 416 || (Result >= 300 && Result < 400)) DownloadSizePtr = &JunkSize; - *DownloadSizePtr = strtoull(Val.c_str(), NULL, 10); + *DownloadSizePtr = ContentLength; if (*DownloadSizePtr >= std::numeric_limits<unsigned long long>::max()) return _error->Errno("HeaderLine", _("The HTTP server sent an invalid Content-Length header")); else if (*DownloadSizePtr == 0) @@ -180,7 +178,11 @@ bool ServerState::HeaderLine(string Line) return true; } - if (stringcasecmp(Tag,"Content-Range:") == 0) + // The Content-Range field only has a meaning in HTTP/1.1 for the + // 206 (Partial Content) and 416 (Range Not Satisfiable) responses + // according to RFC7233 "Range Requests", §4.2, so only consider it + // for such responses. + if ((Result == 416 || Result == 206) && stringcasecmp(Tag,"Content-Range:") == 0) { HaveContent = true; @@ -209,16 +211,16 @@ bool ServerState::HeaderLine(string Line) { if (stringcasecmp(Val,"close") == 0) { - Persistent = false; - Pipeline = false; + Server->Persistent = false; + Server->Pipeline = false; /* Some servers send error pages (as they are dynamically generated) for simplicity via a connection close instead of e.g. chunked, so assuming an always closing server only if we get a file + close */ if (Result >= 200 && Result < 300) - PipelineAllowed = false; + Server->PipelineAllowed = false; } else if (stringcasecmp(Val,"keep-alive") == 0) - Persistent = true; + Server->Persistent = true; return true; } @@ -240,7 +242,7 @@ bool ServerState::HeaderLine(string Line) std::string ranges = ',' + Val + ','; ranges.erase(std::remove(ranges.begin(), ranges.end(), ' '), ranges.end()); if (ranges.find(",bytes,") == std::string::npos) - RangesAllowed = false; + Server->RangesAllowed = false; return true; } @@ -248,42 +250,37 @@ bool ServerState::HeaderLine(string Line) } /*}}}*/ // ServerState::ServerState - Constructor /*{{{*/ -ServerState::ServerState(URI Srv, ServerMethod *Owner) : - DownloadSize(0), ServerName(Srv), TimeOut(120), Owner(Owner) +ServerState::ServerState(URI Srv, BaseHttpMethod *Owner) : + ServerName(Srv), TimeOut(120), Owner(Owner) { Reset(); } /*}}}*/ -bool ServerState::AddPartialFileToHashes(FileFd &File) /*{{{*/ +bool RequestState::AddPartialFileToHashes(FileFd &File) /*{{{*/ { File.Truncate(StartPos); - return GetHashes()->AddFD(File, StartPos); + return Server->GetHashes()->AddFD(File, StartPos); } /*}}}*/ -void ServerState::Reset(bool const Everything) /*{{{*/ +void ServerState::Reset() /*{{{*/ { - Major = 0; Minor = 0; Result = 0; Code[0] = '\0'; - TotalFileSize = 0; JunkSize = 0; StartPos = 0; - Encoding = Closes; time(&Date); HaveContent = false; - State = Header; MaximumSize = 0; - if (Everything) - { - Persistent = false; Pipeline = false; PipelineAllowed = true; - RangesAllowed = true; - } + Persistent = false; + Pipeline = false; + PipelineAllowed = true; + RangesAllowed = true; } /*}}}*/ -// ServerMethod::DealWithHeaders - Handle the retrieved header data /*{{{*/ +// BaseHttpMethod::DealWithHeaders - Handle the retrieved header data /*{{{*/ // --------------------------------------------------------------------- /* We look at the header data we got back from the server and decide what to do. Returns DealWithHeadersResult (see http.h for details). */ -ServerMethod::DealWithHeadersResult -ServerMethod::DealWithHeaders(FetchResult &Res) +BaseHttpMethod::DealWithHeadersResult +BaseHttpMethod::DealWithHeaders(FetchResult &Res, RequestState &Req) { // Not Modified - if (Server->Result == 304) + if (Req.Result == 304) { RemoveFile("server", Queue->DestFile); Res.IMSHit = true; @@ -300,26 +297,26 @@ ServerMethod::DealWithHeaders(FetchResult &Res) * redirect. Pass on those codes so the error handling kicks in. */ if (AllowRedirect - && (Server->Result > 300 && Server->Result < 400) - && (Server->Result != 300 // Multiple Choices - && Server->Result != 304 // Not Modified - && Server->Result != 306)) // (Not part of HTTP/1.1, reserved) + && (Req.Result > 300 && Req.Result < 400) + && (Req.Result != 300 // Multiple Choices + && Req.Result != 304 // Not Modified + && Req.Result != 306)) // (Not part of HTTP/1.1, reserved) { - if (Server->Location.empty() == true) + if (Req.Location.empty() == true) ; - else if (Server->Location[0] == '/' && Queue->Uri.empty() == false) + else if (Req.Location[0] == '/' && Queue->Uri.empty() == false) { URI Uri = Queue->Uri; if (Uri.Host.empty() == false) NextURI = URI::SiteOnly(Uri); else NextURI.clear(); - NextURI.append(DeQuoteString(Server->Location)); + NextURI.append(DeQuoteString(Req.Location)); if (Queue->Uri == NextURI) { SetFailReason("RedirectionLoop"); _error->Error("Redirection loop encountered"); - if (Server->HaveContent == true) + if (Req.HaveContent == true) return ERROR_WITH_CONTENT_PAGE; return ERROR_UNRECOVERABLE; } @@ -327,12 +324,12 @@ ServerMethod::DealWithHeaders(FetchResult &Res) } else { - NextURI = DeQuoteString(Server->Location); + NextURI = DeQuoteString(Req.Location); URI tmpURI = NextURI; if (tmpURI.Access.find('+') != std::string::npos) { _error->Error("Server tried to trick us into using a specific implementation: %s", tmpURI.Access.c_str()); - if (Server->HaveContent == true) + if (Req.HaveContent == true) return ERROR_WITH_CONTENT_PAGE; return ERROR_UNRECOVERABLE; } @@ -358,7 +355,7 @@ ServerMethod::DealWithHeaders(FetchResult &Res) { SetFailReason("RedirectionLoop"); _error->Error("Redirection loop encountered"); - if (Server->HaveContent == true) + if (Req.HaveContent == true) return ERROR_WITH_CONTENT_PAGE; return ERROR_UNRECOVERABLE; } @@ -390,7 +387,7 @@ ServerMethod::DealWithHeaders(FetchResult &Res) /* else pass through for error message */ } // retry after an invalid range response without partial data - else if (Server->Result == 416) + else if (Req.Result == 416) { struct stat SBuf; if (stat(Queue->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0) @@ -400,25 +397,25 @@ ServerMethod::DealWithHeaders(FetchResult &Res) { Hashes resultHashes(Queue->ExpectedHashes); FileFd file(Queue->DestFile, FileFd::ReadOnly); - Server->TotalFileSize = file.FileSize(); - Server->Date = file.ModificationTime(); + Req.TotalFileSize = file.FileSize(); + Req.Date = file.ModificationTime(); resultHashes.AddFD(file); HashStringList const hashList = resultHashes.GetHashStringList(); partialHit = (Queue->ExpectedHashes == hashList); } - else if ((unsigned long long)SBuf.st_size == Server->TotalFileSize) + else if ((unsigned long long)SBuf.st_size == Req.TotalFileSize) partialHit = true; if (partialHit == true) { // the file is completely downloaded, but was not moved - if (Server->HaveContent == true) + if (Req.HaveContent == true) { // nuke the sent error page - Server->RunDataToDevNull(); - Server->HaveContent = false; + Server->RunDataToDevNull(Req); + Req.HaveContent = false; } - Server->StartPos = Server->TotalFileSize; - Server->Result = 200; + Req.StartPos = Req.TotalFileSize; + Req.Result = 200; } else if (RemoveFile("server", Queue->DestFile)) { @@ -430,31 +427,31 @@ ServerMethod::DealWithHeaders(FetchResult &Res) /* We have a reply we don't handle. This should indicate a perm server failure */ - if (Server->Result < 200 || Server->Result >= 300) + if (Req.Result < 200 || Req.Result >= 300) { if (_error->PendingError() == false) { std::string err; - strprintf(err, "HttpError%u", Server->Result); + strprintf(err, "HttpError%u", Req.Result); SetFailReason(err); - _error->Error("%u %s", Server->Result, Server->Code); + _error->Error("%u %s", Req.Result, Req.Code); } - if (Server->HaveContent == true) + if (Req.HaveContent == true) return ERROR_WITH_CONTENT_PAGE; return ERROR_UNRECOVERABLE; } // This is some sort of 2xx 'data follows' reply - Res.LastModified = Server->Date; - Res.Size = Server->TotalFileSize; + Res.LastModified = Req.Date; + Res.Size = Req.TotalFileSize; return FILE_IS_OPEN; } /*}}}*/ -// ServerMethod::SigTerm - Handle a fatal signal /*{{{*/ +// BaseHttpMethod::SigTerm - Handle a fatal signal /*{{{*/ // --------------------------------------------------------------------- /* This closes and timestamps the open file. This is necessary to get - resume behavoir on user abort */ -void ServerMethod::SigTerm(int) + resume behavior on user abort */ +void BaseHttpMethod::SigTerm(int) { if (FailFd == -1) _exit(100); @@ -469,11 +466,11 @@ void ServerMethod::SigTerm(int) _exit(100); } /*}}}*/ -// ServerMethod::Fetch - Fetch an item /*{{{*/ +// BaseHttpMethod::Fetch - Fetch an item /*{{{*/ // --------------------------------------------------------------------- /* This adds an item to the pipeline. We keep the pipeline at a fixed depth. */ -bool ServerMethod::Fetch(FetchItem *) +bool BaseHttpMethod::Fetch(FetchItem *) { if (Server == nullptr || QueueBack == nullptr) return true; @@ -540,8 +537,8 @@ bool ServerMethod::Fetch(FetchItem *) return true; } /*}}}*/ -// ServerMethod::Loop - Main loop /*{{{*/ -int ServerMethod::Loop() +// BaseHttpMethod::Loop - Main loop /*{{{*/ +int BaseHttpMethod::Loop() { signal(SIGTERM,SigTerm); signal(SIGINT,SigTerm); @@ -595,7 +592,7 @@ int ServerMethod::Loop() if (Server->IsOpen() == false) QueueBack = Queue; - // Connnect to the host + // Connect to the host if (Server->Open() == false) { Fail(true); @@ -605,9 +602,10 @@ int ServerMethod::Loop() // Fill the pipeline. Fetch(0); - + + RequestState Req(this, Server.get()); // Fetch the next URL header data from the server. - switch (Server->RunHeaders(File, Queue->Uri)) + switch (Server->RunHeaders(Req, Queue->Uri)) { case ServerState::RUN_HEADERS_OK: break; @@ -646,7 +644,7 @@ int ServerMethod::Loop() // Decide what to do. FetchResult Res; Res.Filename = Queue->DestFile; - switch (DealWithHeaders(Res)) + switch (DealWithHeaders(Res, Req)) { // Ok, the file is Open case FILE_IS_OPEN: @@ -660,24 +658,23 @@ int ServerMethod::Loop() // we could do "Server->MaximumSize = Queue->MaximumSize" here // but that would break the clever pipeline messup detection // so instead we use the size of the biggest item in the queue - Server->MaximumSize = FindMaximumObjectSizeInQueue(); + Req.MaximumSize = FindMaximumObjectSizeInQueue(); - if (Server->HaveContent) - Result = Server->RunData(File); + if (Req.HaveContent) + Result = Server->RunData(Req); /* If the server is sending back sizeless responses then fill in the size now */ if (Res.Size == 0) - Res.Size = File->Size(); - + Res.Size = Req.File.Size(); + // Close the file, destroy the FD object and timestamp it FailFd = -1; - delete File; - File = 0; - + Req.File.Close(); + // Timestamp struct timeval times[2]; - times[0].tv_sec = times[1].tv_sec = Server->Date; + times[0].tv_sec = times[1].tv_sec = Req.Date; times[0].tv_usec = times[1].tv_usec = 0; utimes(Queue->DestFile.c_str(), times); @@ -758,9 +755,6 @@ int ServerMethod::Loop() // Hard internal error, kill the connection and fail case ERROR_NOT_FROM_SERVER: { - delete File; - File = 0; - Fail(); RotateDNS(); Server->Close(); @@ -770,7 +764,7 @@ int ServerMethod::Loop() // We need to flush the data, the header is like a 404 w/ error text case ERROR_WITH_CONTENT_PAGE: { - Server->RunDataToDevNull(); + Server->RunDataToDevNull(Req); Fail(); break; } @@ -779,8 +773,8 @@ int ServerMethod::Loop() case TRY_AGAIN_OR_REDIRECT: { // Clear rest of response if there is content - if (Server->HaveContent) - Server->RunDataToDevNull(); + if (Req.HaveContent) + Server->RunDataToDevNull(Req); Redirect(NextURI); break; } @@ -796,7 +790,7 @@ int ServerMethod::Loop() return 0; } /*}}}*/ -unsigned long long ServerMethod::FindMaximumObjectSizeInQueue() const /*{{{*/ +unsigned long long BaseHttpMethod::FindMaximumObjectSizeInQueue() const /*{{{*/ { unsigned long long MaxSizeInQueue = 0; for (FetchItem *I = Queue; I != 0 && I != QueueBack; I = I->Next) @@ -804,13 +798,13 @@ unsigned long long ServerMethod::FindMaximumObjectSizeInQueue() const /*{{{*/ return MaxSizeInQueue; } /*}}}*/ -ServerMethod::ServerMethod(std::string &&Binary, char const * const Ver,unsigned long const Flags) :/*{{{*/ - aptMethod(std::move(Binary), Ver, Flags), Server(nullptr), File(NULL), PipelineDepth(10), +BaseHttpMethod::BaseHttpMethod(std::string &&Binary, char const * const Ver,unsigned long const Flags) :/*{{{*/ + aptMethod(std::move(Binary), Ver, Flags), Server(nullptr), PipelineDepth(10), AllowRedirect(false), Debug(false) { } /*}}}*/ -bool ServerMethod::Configuration(std::string Message) /*{{{*/ +bool BaseHttpMethod::Configuration(std::string Message) /*{{{*/ { if (aptMethod::Configuration(Message) == false) return false; @@ -820,7 +814,7 @@ bool ServerMethod::Configuration(std::string Message) /*{{{*/ return true; } /*}}}*/ -bool ServerMethod::AddProxyAuth(URI &Proxy, URI const &Server) const /*{{{*/ +bool BaseHttpMethod::AddProxyAuth(URI &Proxy, URI const &Server) const /*{{{*/ { if (std::find(methodNames.begin(), methodNames.end(), "tor") != methodNames.end() && Proxy.User == "apt-transport-tor" && Proxy.Password.empty()) diff --git a/methods/server.h b/methods/basehttp.h index c3adba87a..41a9a4306 100644 --- a/methods/server.h +++ b/methods/basehttp.h @@ -12,6 +12,7 @@ #define APT_SERVER_H #include <apt-pkg/strutl.h> +#include <apt-pkg/fileutl.h> #include "aptmethod.h" #include <time.h> @@ -23,53 +24,63 @@ using std::cout; using std::endl; class Hashes; -class ServerMethod; -class FileFd; +class BaseHttpMethod; +struct ServerState; -struct ServerState +struct RequestState { - // This is the last parsed Header Line - unsigned int Major; - unsigned int Minor; - unsigned int Result; + unsigned int Major = 0; + unsigned int Minor = 0; + unsigned int Result = 0; char Code[360]; - // These are some statistics from the last parsed header lines - // total size of the usable content (aka: the file) - unsigned long long TotalFileSize; + unsigned long long TotalFileSize = 0; // size we actually download (can be smaller than Size if we have partial content) - unsigned long long DownloadSize; + unsigned long long DownloadSize = 0; // size of junk content (aka: server error pages) - unsigned long long JunkSize; + unsigned long long JunkSize = 0; // The start of the data (for partial content) - unsigned long long StartPos; + unsigned long long StartPos = 0; + + unsigned long long MaximumSize = 0; time_t Date; - bool HaveContent; - enum {Chunked,Stream,Closes} Encoding; - enum {Header, Data} State; + bool HaveContent = false; + enum {Chunked,Stream,Closes} Encoding = Closes; + enum {Header, Data} State = Header; + std::string Location; + + FileFd File; + + BaseHttpMethod * const Owner; + ServerState * const Server; + + bool HeaderLine(std::string const &Line); + bool AddPartialFileToHashes(FileFd &File); + + RequestState(BaseHttpMethod * const Owner, ServerState * const Server) : + Owner(Owner), Server(Server) { time(&Date); } +}; + +struct ServerState +{ bool Persistent; bool PipelineAllowed; bool RangesAllowed; - std::string Location; - // This is a Persistent attribute of the server itself. bool Pipeline; URI ServerName; URI Proxy; unsigned long TimeOut; - unsigned long long MaximumSize; - protected: - ServerMethod *Owner; + BaseHttpMethod *Owner; virtual bool ReadHeaderLines(std::string &Data) = 0; - virtual bool LoadNextResponse(bool const ToFile, FileFd * const File) = 0; + virtual bool LoadNextResponse(bool const ToFile, RequestState &Req) = 0; public: - bool HeaderLine(std::string Line); /** \brief Result of the header acquire */ enum RunHeadersResult { @@ -81,38 +92,36 @@ struct ServerState RUN_HEADERS_PARSE_ERROR }; /** \brief Get the headers before the data */ - RunHeadersResult RunHeaders(FileFd * const File, const std::string &Uri); - bool AddPartialFileToHashes(FileFd &File); + RunHeadersResult RunHeaders(RequestState &Req, const std::string &Uri); bool Comp(URI Other) const {return Other.Host == ServerName.Host && Other.Port == ServerName.Port;}; - virtual void Reset(bool const Everything = true); + virtual void Reset(); virtual bool WriteResponse(std::string const &Data) = 0; /** \brief Transfer the data from the socket */ - virtual bool RunData(FileFd * const File) = 0; - virtual bool RunDataToDevNull() = 0; + virtual bool RunData(RequestState &Req) = 0; + virtual bool RunDataToDevNull(RequestState &Req) = 0; virtual bool Open() = 0; virtual bool IsOpen() = 0; virtual bool Close() = 0; virtual bool InitHashes(HashStringList const &ExpectedHashes) = 0; - virtual Hashes * GetHashes() = 0; - virtual bool Die(FileFd * const File) = 0; + virtual bool Die(RequestState &Req) = 0; virtual bool Flush(FileFd * const File) = 0; - virtual bool Go(bool ToFile, FileFd * const File) = 0; + virtual bool Go(bool ToFile, RequestState &Req) = 0; + virtual Hashes * GetHashes() = 0; - ServerState(URI Srv, ServerMethod *Owner); + ServerState(URI Srv, BaseHttpMethod *Owner); virtual ~ServerState() {}; }; -class ServerMethod : public aptMethod +class BaseHttpMethod : public aptMethod { protected: virtual bool Fetch(FetchItem *) APT_OVERRIDE; std::unique_ptr<ServerState> Server; std::string NextURI; - FileFd *File; unsigned long PipelineDepth; bool AllowRedirect; @@ -140,7 +149,7 @@ class ServerMethod : public aptMethod TRY_AGAIN_OR_REDIRECT }; /** \brief Handle the retrieved header data */ - virtual DealWithHeadersResult DealWithHeaders(FetchResult &Res); + virtual DealWithHeadersResult DealWithHeaders(FetchResult &Res, RequestState &Req); // In the event of a fatal signal this file will be closed and timestamped. static std::string FailFile; @@ -148,8 +157,6 @@ class ServerMethod : public aptMethod static time_t FailTime; static APT_NORETURN void SigTerm(int); - virtual bool Flush() { return Server->Flush(File); }; - int Loop(); virtual void SendReq(FetchItem *Itm) = 0; @@ -159,8 +166,8 @@ class ServerMethod : public aptMethod bool AddProxyAuth(URI &Proxy, URI const &Server) const; - ServerMethod(std::string &&Binary, char const * const Ver,unsigned long const Flags); - virtual ~ServerMethod() {}; + BaseHttpMethod(std::string &&Binary, char const * const Ver,unsigned long const Flags); + virtual ~BaseHttpMethod() {}; }; #endif diff --git a/methods/connect.cc b/methods/connect.cc index f6fb14769..dc2aee05c 100644 --- a/methods/connect.cc +++ b/methods/connect.cc @@ -144,7 +144,9 @@ static bool DoConnect(struct addrinfo *Addr,std::string const &Host, return _error->Errno("connect",_("Could not connect to %s:%s (%s)."),Host.c_str(), Service,Name); } - + + Owner->SetFailReason(""); + return true; } /*}}}*/ @@ -312,12 +314,15 @@ bool Connect(std::string Host,int Port,const char *Service, size_t stackSize = 0; // try to connect in the priority order of the srv records std::string initialHost{std::move(Host)}; + auto const initialPort = Port; while(SrvRecords.empty() == false) { _error->PushToStack(); ++stackSize; // PopFromSrvRecs will also remove the server - Host = PopFromSrvRecs(SrvRecords).target; + auto Srv = PopFromSrvRecs(SrvRecords); + Host = Srv.target; + Port = Srv.port; auto const ret = ConnectToHostname(Host, Port, Service, DefPort, Fd, TimeOut, Owner); if (ret) { @@ -327,6 +332,7 @@ bool Connect(std::string Host,int Port,const char *Service, } } Host = std::move(initialHost); + Port = initialPort; // we have no (good) SrvRecords for this host, connect right away _error->PushToStack(); diff --git a/methods/ftp.cc b/methods/ftp.cc index dbc5f25d1..d789637a8 100644 --- a/methods/ftp.cc +++ b/methods/ftp.cc @@ -6,7 +6,7 @@ FTP Acquire Method - This is the FTP acquire method for APT. This is a very simple implementation that does not try to optimize - at all. Commands are sent syncronously with the FTP server (as the + at all. Commands are sent synchronously with the FTP server (as the rfc recommends, but it is not really necessary..) and no tricks are done to speed things along. @@ -974,7 +974,7 @@ FtpMethod::FtpMethod() : aptMethod("ftp","1.0",SendConfig) // FtpMethod::SigTerm - Handle a fatal signal /*{{{*/ // --------------------------------------------------------------------- /* This closes and timestamps the open file. This is necessary to get - resume behavoir on user abort */ + resume behavior on user abort */ void FtpMethod::SigTerm(int) { if (FailFd == -1) diff --git a/methods/gpgv.cc b/methods/gpgv.cc index 95a86f890..51c268d02 100644 --- a/methods/gpgv.cc +++ b/methods/gpgv.cc @@ -120,7 +120,7 @@ static void PushEntryWithKeyID(std::vector<std::string> &Signers, char * const b // skip the message while (*p && !isspace(*p)) ++p; - // skip the seperator whitespace + // skip the separator whitespace ++p; // skip the hexdigit fingerprint while (*p && isxdigit(*p)) diff --git a/methods/http.cc b/methods/http.cc index 4703584cf..72e813cb9 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -174,32 +174,28 @@ void HttpMethod::RotateDNS() /*{{{*/ { } /*}}}*/ -ServerMethod::DealWithHeadersResult HttpMethod::DealWithHeaders(FetchResult &Res)/*{{{*/ +BaseHttpMethod::DealWithHeadersResult HttpMethod::DealWithHeaders(FetchResult &Res, RequestState &Req)/*{{{*/ { - auto ret = ServerMethod::DealWithHeaders(Res); - if (ret != ServerMethod::FILE_IS_OPEN) + auto ret = BaseHttpMethod::DealWithHeaders(Res, Req); + if (ret != BaseHttpMethod::FILE_IS_OPEN) return ret; - - // Open the file - delete File; - File = new FileFd(Queue->DestFile,FileFd::WriteAny); - if (_error->PendingError() == true) + if (Req.File.Open(Queue->DestFile, FileFd::WriteAny) == false) return ERROR_NOT_FROM_SERVER; FailFile = Queue->DestFile; FailFile.c_str(); // Make sure we don't do a malloc in the signal handler - FailFd = File->Fd(); - FailTime = Server->Date; + FailFd = Req.File.Fd(); + FailTime = Req.Date; - if (Server->InitHashes(Queue->ExpectedHashes) == false || Server->AddPartialFileToHashes(*File) == false) + if (Server->InitHashes(Queue->ExpectedHashes) == false || Req.AddPartialFileToHashes(Req.File) == false) { _error->Errno("read",_("Problem hashing file")); return ERROR_NOT_FROM_SERVER; } - if (Server->StartPos > 0) - Res.ResumePoint = Server->StartPos; + if (Req.StartPos > 0) + Res.ResumePoint = Req.StartPos; - SetNonBlock(File->Fd(),true); + SetNonBlock(Req.File.Fd(),true); return FILE_IS_OPEN; } @@ -554,7 +550,7 @@ int HttpMethod::Loop() return 0; } -HttpMethod::HttpMethod(std::string &&pProg) : ServerMethod(pProg.c_str(), "1.2", Pipeline | SendConfig)/*{{{*/ +HttpMethod::HttpMethod(std::string &&pProg) : BaseHttpMethod(pProg.c_str(), "1.2", Pipeline | SendConfig)/*{{{*/ { auto addName = std::inserter(methodNames, methodNames.begin()); if (Binary != "http") diff --git a/methods/http.h b/methods/http.h index 203ea9fd4..3b491dfda 100644 --- a/methods/http.h +++ b/methods/http.h @@ -17,7 +17,7 @@ #include <sys/time.h> #include <iostream> -#include "server.h" +#include "basehttp.h" using std::cout; using std::endl; @@ -26,17 +26,18 @@ class FileFd; class HttpMethod; class Hashes; -class HttpMethod : public ServerMethod +class HttpMethod : public BaseHttpMethod { public: virtual void SendReq(FetchItem *Itm) APT_OVERRIDE; virtual std::unique_ptr<ServerState> CreateServerState(URI const &uri) APT_OVERRIDE; virtual void RotateDNS() APT_OVERRIDE; - virtual DealWithHeadersResult DealWithHeaders(FetchResult &Res) APT_OVERRIDE; + virtual DealWithHeadersResult DealWithHeaders(FetchResult &Res, RequestState &Req) APT_OVERRIDE; protected: std::string AutoDetectProxyCmd; + FileFd *File; public: friend struct HttpServerState; diff --git a/methods/https.cc b/methods/https.cc index b2d05136c..d71ef0bf0 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -43,8 +43,9 @@ struct APT_HIDDEN CURLUserPointer { HttpsMethod * const https; HttpsMethod::FetchResult * const Res; HttpsMethod::FetchItem const * const Itm; + RequestState * const Req; CURLUserPointer(HttpsMethod * const https, HttpsMethod::FetchResult * const Res, - HttpsMethod::FetchItem const * const Itm) : https(https), Res(Res), Itm(Itm) {} + HttpsMethod::FetchItem const * const Itm, RequestState * const Req) : https(https), Res(Res), Itm(Itm), Req(Req) {} }; size_t @@ -63,55 +64,58 @@ HttpsMethod::parse_header(void *buffer, size_t size, size_t nmemb, void *userp) if (line.empty() == true) { - me->https->Server->JunkSize = 0; - if (me->https->Server->Result != 416 && me->https->Server->StartPos != 0) + if (me->Req->File.Open(me->Itm->DestFile, FileFd::WriteAny) == false) + return ERROR_NOT_FROM_SERVER; + + me->Req->JunkSize = 0; + if (me->Req->Result != 416 && me->Req->StartPos != 0) ; - else if (me->https->Server->Result == 416) + else if (me->Req->Result == 416) { bool partialHit = false; if (me->Itm->ExpectedHashes.usable() == true) { Hashes resultHashes(me->Itm->ExpectedHashes); FileFd file(me->Itm->DestFile, FileFd::ReadOnly); - me->https->Server->TotalFileSize = file.FileSize(); - me->https->Server->Date = file.ModificationTime(); + me->Req->TotalFileSize = file.FileSize(); + me->Req->Date = file.ModificationTime(); resultHashes.AddFD(file); HashStringList const hashList = resultHashes.GetHashStringList(); partialHit = (me->Itm->ExpectedHashes == hashList); } - else if (me->https->Server->Result == 416 && me->https->Server->TotalFileSize == me->https->File->FileSize()) + else if (me->Req->Result == 416 && me->Req->TotalFileSize == me->Req->File.FileSize()) partialHit = true; if (partialHit == true) { - me->https->Server->Result = 200; - me->https->Server->StartPos = me->https->Server->TotalFileSize; + me->Req->Result = 200; + me->Req->StartPos = me->Req->TotalFileSize; // the actual size is not important for https as curl will deal with it // by itself and e.g. doesn't bother us with transport-encoding… - me->https->Server->JunkSize = std::numeric_limits<unsigned long long>::max(); + me->Req->JunkSize = std::numeric_limits<unsigned long long>::max(); } else - me->https->Server->StartPos = 0; + me->Req->StartPos = 0; } else - me->https->Server->StartPos = 0; + me->Req->StartPos = 0; - me->Res->LastModified = me->https->Server->Date; - me->Res->Size = me->https->Server->TotalFileSize; - me->Res->ResumePoint = me->https->Server->StartPos; + me->Res->LastModified = me->Req->Date; + me->Res->Size = me->Req->TotalFileSize; + me->Res->ResumePoint = me->Req->StartPos; // we expect valid data, so tell our caller we get the file now - if (me->https->Server->Result >= 200 && me->https->Server->Result < 300) + if (me->Req->Result >= 200 && me->Req->Result < 300) { if (me->Res->Size != 0 && me->Res->Size > me->Res->ResumePoint) me->https->URIStart(*me->Res); - if (me->https->Server->AddPartialFileToHashes(*(me->https->File)) == false) + if (me->Req->AddPartialFileToHashes(me->Req->File) == false) return 0; } else - me->https->Server->JunkSize = std::numeric_limits<decltype(me->https->Server->JunkSize)>::max(); + me->Req->JunkSize = std::numeric_limits<decltype(me->Req->JunkSize)>::max(); } - else if (me->https->Server->HeaderLine(line) == false) + else if (me->Req->HeaderLine(line) == false) return 0; return size*nmemb; @@ -120,29 +124,29 @@ HttpsMethod::parse_header(void *buffer, size_t size, size_t nmemb, void *userp) size_t HttpsMethod::write_data(void *buffer, size_t size, size_t nmemb, void *userp) { - HttpsMethod *me = static_cast<HttpsMethod *>(userp); + CURLUserPointer *me = static_cast<CURLUserPointer *>(userp); size_t buffer_size = size * nmemb; // we don't need to count the junk here, just drop anything we get as // we don't always know how long it would be, e.g. in chunked encoding. - if (me->Server->JunkSize != 0) + if (me->Req->JunkSize != 0) return buffer_size; - if(me->File->Write(buffer, buffer_size) != true) + if(me->Req->File.Write(buffer, buffer_size) != true) return 0; - if(me->Queue->MaximumSize > 0) + if(me->https->Queue->MaximumSize > 0) { - unsigned long long const TotalWritten = me->File->Tell(); - if (TotalWritten > me->Queue->MaximumSize) + unsigned long long const TotalWritten = me->Req->File.Tell(); + if (TotalWritten > me->https->Queue->MaximumSize) { - me->SetFailReason("MaximumSizeExceeded"); + me->https->SetFailReason("MaximumSizeExceeded"); _error->Error("Writing more data than expected (%llu > %llu)", - TotalWritten, me->Queue->MaximumSize); + TotalWritten, me->https->Queue->MaximumSize); return 0; } } - if (me->Server->GetHashes()->Add((unsigned char const * const)buffer, buffer_size) == false) + if (me->https->Server->GetHashes()->Add((unsigned char const * const)buffer, buffer_size) == false) return 0; return buffer_size; @@ -268,15 +272,22 @@ bool HttpsMethod::Fetch(FetchItem *Itm) return _error->Error("Unsupported proxy configured: %s", URI::SiteOnly(Proxy).c_str()); maybe_add_auth (Uri, _config->FindFile("Dir::Etc::netrc")); + if (Server == nullptr || Server->Comp(Itm->Uri) == false) + Server = CreateServerState(Itm->Uri); + + // The "+" is encoded as a workaround for a amazon S3 bug + // see LP bugs #1003633 and #1086997. (taken from http method) + Uri.Path = QuoteString(Uri.Path, "+~ "); FetchResult Res; - CURLUserPointer userp(this, &Res, Itm); + RequestState Req(this, Server.get()); + CURLUserPointer userp(this, &Res, Itm, &Req); // callbacks curl_easy_setopt(curl, CURLOPT_URL, static_cast<string>(Uri).c_str()); curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, parse_header); curl_easy_setopt(curl, CURLOPT_WRITEHEADER, &userp); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, this); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &userp); // options curl_easy_setopt(curl, CURLOPT_NOPROGRESS, true); curl_easy_setopt(curl, CURLOPT_FILETIME, true); @@ -361,6 +372,11 @@ bool HttpsMethod::Fetch(FetchItem *Itm) curl_easy_setopt(curl, CURLOPT_LOW_SPEED_LIMIT, DL_MIN_SPEED); curl_easy_setopt(curl, CURLOPT_LOW_SPEED_TIME, timeout); + if(_config->FindB("Acquire::ForceIPv4", false) == true) + curl_easy_setopt(curl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); + else if(_config->FindB("Acquire::ForceIPv6", false) == true) + curl_easy_setopt(curl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V6); + // debug if (Debug == true) curl_easy_setopt(curl, CURLOPT_VERBOSE, true); @@ -382,13 +398,6 @@ bool HttpsMethod::Fetch(FetchItem *Itm) headers = curl_slist_append(headers, "Accept: text/*"); } - // go for it - if the file exists, append on it - File = new FileFd(Itm->DestFile, FileFd::WriteAny); - if (Server == nullptr || Server->Comp(Itm->Uri) == false) - Server = CreateServerState(Itm->Uri); - else - Server->Reset(false); - // if we have the file send an if-range query with a range header if (Server->RangesAllowed && stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0) { @@ -418,9 +427,9 @@ bool HttpsMethod::Fetch(FetchItem *Itm) long curl_condition_unmet = 0; curl_easy_getinfo(curl, CURLINFO_CONDITION_UNMET, &curl_condition_unmet); if (curl_condition_unmet == 1) - Server->Result = 304; + Req.Result = 304; - File->Close(); + Req.File.Close(); curl_slist_free_all(headers); // cleanup @@ -451,28 +460,28 @@ bool HttpsMethod::Fetch(FetchItem *Itm) return false; } - switch (DealWithHeaders(Res)) + switch (DealWithHeaders(Res, Req)) { - case ServerMethod::IMS_HIT: + case BaseHttpMethod::IMS_HIT: URIDone(Res); break; - case ServerMethod::ERROR_WITH_CONTENT_PAGE: + case BaseHttpMethod::ERROR_WITH_CONTENT_PAGE: // unlink, no need keep 401/404 page content in partial/ - RemoveFile(Binary.c_str(), File->Name()); - case ServerMethod::ERROR_UNRECOVERABLE: - case ServerMethod::ERROR_NOT_FROM_SERVER: + RemoveFile(Binary.c_str(), Req.File.Name()); + case BaseHttpMethod::ERROR_UNRECOVERABLE: + case BaseHttpMethod::ERROR_NOT_FROM_SERVER: return false; - case ServerMethod::TRY_AGAIN_OR_REDIRECT: + case BaseHttpMethod::TRY_AGAIN_OR_REDIRECT: Redirect(NextURI); break; - case ServerMethod::FILE_IS_OPEN: + case BaseHttpMethod::FILE_IS_OPEN: struct stat resultStat; - if (unlikely(stat(File->Name().c_str(), &resultStat) != 0)) + if (unlikely(stat(Req.File.Name().c_str(), &resultStat) != 0)) { - _error->Errno("stat", "Unable to access file %s", File->Name().c_str()); + _error->Errno("stat", "Unable to access file %s", Req.File.Name().c_str()); return false; } Res.Size = resultStat.st_size; @@ -485,7 +494,7 @@ bool HttpsMethod::Fetch(FetchItem *Itm) times[0].tv_sec = Res.LastModified; times[1].tv_sec = Res.LastModified; times[0].tv_usec = times[1].tv_usec = 0; - utimes(File->Name().c_str(), times); + utimes(Req.File.Name().c_str(), times); } else Res.LastModified = resultStat.st_mtime; @@ -497,8 +506,6 @@ bool HttpsMethod::Fetch(FetchItem *Itm) URIDone(Res); break; } - - delete File; return true; } /*}}}*/ @@ -507,7 +514,7 @@ std::unique_ptr<ServerState> HttpsMethod::CreateServerState(URI const &uri)/*{{{ return std::unique_ptr<ServerState>(new HttpsServerState(uri, this)); } /*}}}*/ -HttpsMethod::HttpsMethod(std::string &&pProg) : ServerMethod(std::move(pProg),"1.2",Pipeline | SendConfig)/*{{{*/ +HttpsMethod::HttpsMethod(std::string &&pProg) : BaseHttpMethod(std::move(pProg),"1.2",Pipeline | SendConfig)/*{{{*/ { auto addName = std::inserter(methodNames, methodNames.begin()); addName = "http"; diff --git a/methods/https.h b/methods/https.h index 04e72e815..fbbf34501 100644 --- a/methods/https.h +++ b/methods/https.h @@ -17,7 +17,7 @@ #include <string> #include <memory> -#include "server.h" +#include "basehttp.h" using std::cout; using std::endl; @@ -32,29 +32,29 @@ class HttpsServerState : public ServerState protected: virtual bool ReadHeaderLines(std::string &/*Data*/) APT_OVERRIDE { return false; } - virtual bool LoadNextResponse(bool const /*ToFile*/, FileFd * const /*File*/) APT_OVERRIDE { return false; } + virtual bool LoadNextResponse(bool const /*ToFile*/, RequestState &/*Req*/) APT_OVERRIDE { return false; } public: virtual bool WriteResponse(std::string const &/*Data*/) APT_OVERRIDE { return false; } /** \brief Transfer the data from the socket */ - virtual bool RunData(FileFd * const /*File*/) APT_OVERRIDE { return false; } - virtual bool RunDataToDevNull() APT_OVERRIDE { return false; } + virtual bool RunData(RequestState &) APT_OVERRIDE { return false; } + virtual bool RunDataToDevNull(RequestState &) APT_OVERRIDE { return false; } virtual bool Open() APT_OVERRIDE { return false; } virtual bool IsOpen() APT_OVERRIDE { return false; } virtual bool Close() APT_OVERRIDE { return false; } virtual bool InitHashes(HashStringList const &ExpectedHashes) APT_OVERRIDE; virtual Hashes * GetHashes() APT_OVERRIDE; - virtual bool Die(FileFd * const /*File*/) APT_OVERRIDE { return false; } + virtual bool Die(RequestState &/*Req*/) APT_OVERRIDE { return false; } virtual bool Flush(FileFd * const /*File*/) APT_OVERRIDE { return false; } - virtual bool Go(bool /*ToFile*/, FileFd * const /*File*/) APT_OVERRIDE { return false; } + virtual bool Go(bool /*ToFile*/, RequestState &/*Req*/) APT_OVERRIDE { return false; } HttpsServerState(URI Srv, HttpsMethod *Owner); virtual ~HttpsServerState() {Close();}; }; -class HttpsMethod : public ServerMethod +class HttpsMethod : public BaseHttpMethod { // minimum speed in bytes/se that triggers download timeout handling static const int DL_MIN_SPEED = 10; @@ -68,7 +68,7 @@ class HttpsMethod : public ServerMethod bool SetupProxy(); CURL *curl; - // Used by ServerMethods unused by https + // Used by BaseHttpMethods unused by https virtual void SendReq(FetchItem *) APT_OVERRIDE { exit(42); } virtual void RotateDNS() APT_OVERRIDE { exit(42); } diff --git a/methods/rred.cc b/methods/rred.cc index 958933a07..2e5008d46 100644 --- a/methods/rred.cc +++ b/methods/rred.cc @@ -662,12 +662,14 @@ class RredMethod : public aptMethod { FileFd inp, out; if (inp.Open(Path, FileFd::ReadOnly, FileFd::Extension) == false) { - std::cerr << "FAILED to open inp " << Path << std::endl; + if (Debug == true) + std::clog << "FAILED to open inp " << Path << std::endl; return _error->Error("Failed to open inp %s", Path.c_str()); } if (out.Open(Itm->DestFile, FileFd::WriteOnly | FileFd::Create | FileFd::Empty | FileFd::BufferedWrite, FileFd::Extension) == false) { - std::cerr << "FAILED to open out " << Itm->DestFile << std::endl; + if (Debug == true) + std::clog << "FAILED to open out " << Itm->DestFile << std::endl; return _error->Error("Failed to open out %s", Itm->DestFile.c_str()); } @@ -686,7 +688,8 @@ class RredMethod : public aptMethod { inp.Close(); if (_error->PendingError() == true) { - std::cerr << "FAILED to read or write files" << std::endl; + if (Debug == true) + std::clog << "FAILED to read or write files" << std::endl; return false; } |