From 64207dad49f1c803d2b004ccf8fc6432789a8cc2 Mon Sep 17 00:00:00 2001 From: Julian Andres Klode Date: Fri, 30 Jun 2017 13:24:04 +0200 Subject: http: Add support for CONNECT proxying to HTTPS locations Proxying HTTPS traffic requires the proxy providing the CONNECT method. This implements the client side of it, although it is a bit hacky. HTTP connect is a normal HTTP CONNECT request, followed by a normal HTTP response, just that the body of the response is the TCP stream of the target host. We use a special wrapper in case there are data bytes in the header packets - in that case, the bytes are stored in a buffer and the buffer will be drained first, afterwards the connection continues directly with the TCP stream (with one more vcall). Also: Do not send full URI to https destinations when proxying, as we are directly interfacing with the destination data stream. --- .clang-format | 1 + methods/http.cc | 129 +++++++++++++++++++++++++++++++++++++++++++++++++++++++- methods/http.h | 3 ++ 3 files changed, 132 insertions(+), 1 deletion(-) diff --git a/.clang-format b/.clang-format index 00edbfc92..3f5e0c127 100644 --- a/.clang-format +++ b/.clang-format @@ -7,3 +7,4 @@ IndentWidth: 3 ColumnLimit: 0 BreakBeforeBraces: Allman AccessModifierOffset: 0 +SortIncludes: false diff --git a/methods/http.cc b/methods/http.cc index 4ad4d389c..845e9c45b 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -266,6 +266,15 @@ bool CircleBuf::WriteTillEl(string &Data,bool Single) return false; } /*}}}*/ +// CircleBuf::Write - Write from the buffer to a string /*{{{*/ +// --------------------------------------------------------------------- +/* This copies everything */ +bool CircleBuf::Write(string &Data) +{ + Data = std::string((char *)Buf + (OutP % Size), LeftWrite()); + OutP += LeftWrite(); + return true; +} // CircleBuf::Stats - Print out stats information /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -287,6 +296,122 @@ CircleBuf::~CircleBuf() delete Hash; } +// UnwrapHTTPConnect - Does the HTTP CONNECT handshake /*{{{*/ +// --------------------------------------------------------------------- +/* Performs a TLS handshake on the socket */ +struct HttpConnectFd : public MethodFd +{ + std::unique_ptr UnderlyingFd; + std::string Buffer; + + int Fd() APT_OVERRIDE { return UnderlyingFd->Fd(); } + + ssize_t Read(void *buf, size_t count) APT_OVERRIDE + { + if (!Buffer.empty()) + { + auto read = count < Buffer.size() ? count : Buffer.size(); + + memcpy(buf, Buffer.data(), read); + Buffer.erase(Buffer.begin(), Buffer.begin() + read); + return read; + } + + return UnderlyingFd->Read(buf, count); + } + ssize_t Write(void *buf, size_t count) APT_OVERRIDE + { + return UnderlyingFd->Write(buf, count); + } + + int Close() APT_OVERRIDE + { + return UnderlyingFd->Close(); + } + + bool HasPending() APT_OVERRIDE + { + return !Buffer.empty(); + } +}; + +bool UnwrapHTTPConnect(std::string Host, int Port, URI Proxy, std::unique_ptr &Fd, + unsigned long Timeout, aptMethod *Owner) +{ + Owner->Status(_("Connecting to %s (%s)"), "HTTP proxy", URI::SiteOnly(Proxy).c_str()); + // The HTTP server expects a hostname with a trailing :port + std::stringstream Req; + std::string ProperHost; + + if (Host.find(':') != std::string::npos) + ProperHost = '[' + Proxy.Host + ']'; + else + ProperHost = Proxy.Host; + + // Build the connect + Req << "CONNECT " << Host << ":" << std::to_string(Port) << " HTTP/1.1\r\n"; + if (Proxy.Port != 0) + Req << "Host: " << ProperHost << ":" << std::to_string(Proxy.Port) << "\r\n"; + else + Req << "Host: " << ProperHost << "\r\n"; + ; + + maybe_add_auth(Proxy, _config->FindFile("Dir::Etc::netrc")); + if (Proxy.User.empty() == false || Proxy.Password.empty() == false) + Req << "Proxy-Authorization: Basic " + << Base64Encode(Proxy.User + ":" + Proxy.Password) << "\r\n"; + + Req << "User-Agent: " << Owner->ConfigFind("User-Agent", "Debian APT-HTTP/1.3 (" PACKAGE_VERSION ")") << "\r\n"; + + Req << "\r\n"; + + CircleBuf In(dynamic_cast(Owner), 4096); + CircleBuf Out(dynamic_cast(Owner), 4096); + std::string Headers; + + if (Owner->DebugEnabled() == true) + cerr << Req.str() << endl; + Out.Read(Req.str()); + + // Writing from proxy + while (Out.WriteSpace() > 0) + { + if (WaitFd(Fd->Fd(), true, Timeout) == false) + return _error->Errno("select", "Writing to proxy failed"); + if (Out.Write(Fd) == false) + return _error->Errno("write", "Writing to proxy failed"); + } + + while (In.ReadSpace() > 0) + { + if (WaitFd(Fd->Fd(), false, Timeout) == false) + return _error->Errno("select", "Reading from proxy failed"); + if (In.Read(Fd) == false) + return _error->Errno("read", "Reading from proxy failed"); + + if (In.WriteTillEl(Headers)) + break; + } + + if (Owner->DebugEnabled() == true) + cerr << Headers << endl; + + if (!(APT::String::Startswith(Headers, "HTTP/1.0 200") || APT::String::Startswith(Headers, "HTTP/1.1 200"))) + return _error->Error("Invalid response from proxy: %s", Headers.c_str()); + + if (In.WriteSpace() > 0) + { + // Maybe there is actual data already read, if so we need to buffer it + std::unique_ptr NewFd(new HttpConnectFd()); + In.Write(NewFd->Buffer); + NewFd->UnderlyingFd = std::move(Fd); + Fd = std::move(NewFd); + } + + return true; +} + /*}}}*/ + // HttpServerState::HttpServerState - Constructor /*{{{*/ HttpServerState::HttpServerState(URI Srv,HttpMethod *Owner) : ServerState(Srv, Owner), In(Owner, 64*1024), Out(Owner, 4*1024) { @@ -376,6 +501,8 @@ bool HttpServerState::Open() } if (!Connect(Host, Port, DefaultService, DefaultPort, ServerFd, TimeOut, Owner)) return false; + if (Host == Proxy.Host && tls && UnwrapHTTPConnect(ServerName.Host, ServerName.Port == 0 ? DefaultPort : ServerName.Port, Proxy, ServerFd, Owner->ConfigFindI("TimeOut", 120), Owner) == false) + return false; } if (tls && UnwrapTLS(ServerName.Host, ServerFd, TimeOut, Owner) == false) @@ -743,7 +870,7 @@ void HttpMethod::SendReq(FetchItem *Itm) but while its a must for all servers to accept absolute URIs, it is assumed clients will sent an absolute path for non-proxies */ std::string requesturi; - if (Server->Proxy.Access != "http" || Server->Proxy.empty() == true || Server->Proxy.Host.empty()) + if ((Server->Proxy.Access != "http" && Server->Proxy.Access != "https") || APT::String::Endswith(Uri.Access, "https") || Server->Proxy.empty() == true || Server->Proxy.Host.empty()) requesturi = Uri.Path; else requesturi = Uri; diff --git a/methods/http.h b/methods/http.h index 3336fb780..7a763675c 100644 --- a/methods/http.h +++ b/methods/http.h @@ -73,6 +73,7 @@ class CircleBuf // Write data out bool Write(std::unique_ptr const &Fd); + bool Write(std::string &Data); bool WriteTillEl(std::string &Data,bool Single = false); // Control the write limit @@ -92,6 +93,8 @@ class CircleBuf ~CircleBuf(); }; +bool UnwrapHTTPConnect(std::string To, int Port, URI Proxy, std::unique_ptr &Fd, unsigned long Timeout, aptMethod *Owner); + struct HttpServerState: public ServerState { // This is the connection itself. Output is data FROM the server -- cgit v1.2.3