methods: Add HTTPS support to http method, using GnuTLS

The http method will eventually replace the curl-based
https method, but for now, this is an opt-in experiment
that can be enabled by setting Dir::Bin::Methods::https
to "http".

Known issues:
- We do not support HTTPS proxies yet
- We do not support proxying HTTPS connections yet (CONNECT)
- IssuerCert and SslForceVersion are unsupported

Gbp-Dch: Full

1 files changed, 361 insertions, 0 deletions

diff --git a/methods/connect.cc b/methods/connect.cc
index 9e9e09ac4..c40f34f2d 100644
--- a/methods/connect.cc
+++ b/methods/connect.cc
@@ -20,6 +20,9 @@
 #include <apt-pkg/configuration.h>
 #include <apt-pkg/srvrec.h>
 
+#include <gnutls/gnutls.h>
+#include <gnutls/x509.h>
+
 #include <stdio.h>
 #include <errno.h>
 #include <unistd.h>
@@ -367,3 +370,361 @@ bool Connect(std::string Host, int Port, const char *Service,
 	 _error->MergeWithStack();
    return ret;
 }
+									/*}}}*/
+// UnwrapSocks - Handle SOCKS setup					/*{{{*/
+// ---------------------------------------------------------------------
+/* This does socks magic */
+static bool TalkToSocksProxy(int const ServerFd, std::string const &Proxy,
+			     char const *const type, bool const ReadWrite, uint8_t *const ToFrom,
+			     unsigned int const Size, unsigned int const Timeout)
+{
+   if (WaitFd(ServerFd, ReadWrite, Timeout) == false)
+      return _error->Error("Waiting for the SOCKS proxy %s to %s timed out", URI::SiteOnly(Proxy).c_str(), type);
+   if (ReadWrite == false)
+   {
+      if (FileFd::Read(ServerFd, ToFrom, Size) == false)
+	 return _error->Error("Reading the %s from SOCKS proxy %s failed", type, URI::SiteOnly(Proxy).c_str());
+   }
+   else
+   {
+      if (FileFd::Write(ServerFd, ToFrom, Size) == false)
+	 return _error->Error("Writing the %s to SOCKS proxy %s failed", type, URI::SiteOnly(Proxy).c_str());
+   }
+   return true;
+}
+
+bool UnwrapSocks(std::string Host, int Port, URI Proxy, std::unique_ptr<MethodFd> &Fd,
+		 unsigned long Timeout, aptMethod *Owner)
+{
+   /* We implement a very basic SOCKS5 client here complying mostly to RFC1928 expect
+    * for not offering GSSAPI auth which is a must (we only do no or user/pass auth).
+    * We also expect the SOCKS5 server to do hostname lookup (aka socks5h) */
+   std::string const ProxyInfo = URI::SiteOnly(Proxy);
+   Owner->Status(_("Connecting to %s (%s)"), "SOCKS5h proxy", ProxyInfo.c_str());
+#define APT_WriteOrFail(TYPE, DATA, LENGTH)                                               \
+   if (TalkToSocksProxy(Fd->Fd(), ProxyInfo, TYPE, true, DATA, LENGTH, Timeout) == false) \
+   return false
+#define APT_ReadOrFail(TYPE, DATA, LENGTH)                                                 \
+   if (TalkToSocksProxy(Fd->Fd(), ProxyInfo, TYPE, false, DATA, LENGTH, Timeout) == false) \
+   return false
+   if (Host.length() > 255)
+      return _error->Error("Can't use SOCKS5h as hostname %s is too long!", Host.c_str());
+   if (Proxy.User.length() > 255 || Proxy.Password.length() > 255)
+      return _error->Error("Can't use user&pass auth as they are too long (%lu and %lu) for the SOCKS5!", Proxy.User.length(), Proxy.Password.length());
+   if (Proxy.User.empty())
+   {
+      uint8_t greeting[] = {0x05, 0x01, 0x00};
+      APT_WriteOrFail("greet-1", greeting, sizeof(greeting));
+   }
+   else
+   {
+      uint8_t greeting[] = {0x05, 0x02, 0x00, 0x02};
+      APT_WriteOrFail("greet-2", greeting, sizeof(greeting));
+   }
+   uint8_t greeting[2];
+   APT_ReadOrFail("greet back", greeting, sizeof(greeting));
+   if (greeting[0] != 0x05)
+      return _error->Error("SOCKS proxy %s greets back with wrong version: %d", ProxyInfo.c_str(), greeting[0]);
+   if (greeting[1] == 0x00)
+      ; // no auth has no method-dependent sub-negotiations
+   else if (greeting[1] == 0x02)
+   {
+      if (Proxy.User.empty())
+	 return _error->Error("SOCKS proxy %s negotiated user&pass auth, but we had not offered it!", ProxyInfo.c_str());
+      // user&pass auth sub-negotiations are defined by RFC1929
+      std::vector<uint8_t> auth = {{0x01, static_cast<uint8_t>(Proxy.User.length())}};
+      std::copy(Proxy.User.begin(), Proxy.User.end(), std::back_inserter(auth));
+      auth.push_back(static_cast<uint8_t>(Proxy.Password.length()));
+      std::copy(Proxy.Password.begin(), Proxy.Password.end(), std::back_inserter(auth));
+      APT_WriteOrFail("user&pass auth", auth.data(), auth.size());
+      uint8_t authstatus[2];
+      APT_ReadOrFail("auth report", authstatus, sizeof(authstatus));
+      if (authstatus[0] != 0x01)
+	 return _error->Error("SOCKS proxy %s auth status response with wrong version: %d", ProxyInfo.c_str(), authstatus[0]);
+      if (authstatus[1] != 0x00)
+	 return _error->Error("SOCKS proxy %s reported authorization failure: username or password incorrect? (%d)", ProxyInfo.c_str(), authstatus[1]);
+   }
+   else
+      return _error->Error("SOCKS proxy %s greets back having not found a common authorization method: %d", ProxyInfo.c_str(), greeting[1]);
+   union {
+      uint16_t *i;
+      uint8_t *b;
+   } portu;
+   uint16_t port = htons(static_cast<uint16_t>(Port));
+   portu.i = &port;
+   std::vector<uint8_t> request = {{0x05, 0x01, 0x00, 0x03, static_cast<uint8_t>(Host.length())}};
+   std::copy(Host.begin(), Host.end(), std::back_inserter(request));
+   request.push_back(portu.b[0]);
+   request.push_back(portu.b[1]);
+   APT_WriteOrFail("request", request.data(), request.size());
+   uint8_t response[4];
+   APT_ReadOrFail("first part of response", response, sizeof(response));
+   if (response[0] != 0x05)
+      return _error->Error("SOCKS proxy %s response with wrong version: %d", ProxyInfo.c_str(), response[0]);
+   if (response[2] != 0x00)
+      return _error->Error("SOCKS proxy %s has unexpected non-zero reserved field value: %d", ProxyInfo.c_str(), response[2]);
+   std::string bindaddr;
+   if (response[3] == 0x01) // IPv4 address
+   {
+      uint8_t ip4port[6];
+      APT_ReadOrFail("IPv4+Port of response", ip4port, sizeof(ip4port));
+      portu.b[0] = ip4port[4];
+      portu.b[1] = ip4port[5];
+      port = ntohs(*portu.i);
+      strprintf(bindaddr, "%d.%d.%d.%d:%d", ip4port[0], ip4port[1], ip4port[2], ip4port[3], port);
+   }
+   else if (response[3] == 0x03) // hostname
+   {
+      uint8_t namelength;
+      APT_ReadOrFail("hostname length of response", &namelength, 1);
+      uint8_t hostname[namelength + 2];
+      APT_ReadOrFail("hostname of response", hostname, sizeof(hostname));
+      portu.b[0] = hostname[namelength];
+      portu.b[1] = hostname[namelength + 1];
+      port = ntohs(*portu.i);
+      hostname[namelength] = '\0';
+      strprintf(bindaddr, "%s:%d", hostname, port);
+   }
+   else if (response[3] == 0x04) // IPv6 address
+   {
+      uint8_t ip6port[18];
+      APT_ReadOrFail("IPv6+port of response", ip6port, sizeof(ip6port));
+      portu.b[0] = ip6port[16];
+      portu.b[1] = ip6port[17];
+      port = ntohs(*portu.i);
+      strprintf(bindaddr, "[%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X]:%d",
+		ip6port[0], ip6port[1], ip6port[2], ip6port[3], ip6port[4], ip6port[5], ip6port[6], ip6port[7],
+		ip6port[8], ip6port[9], ip6port[10], ip6port[11], ip6port[12], ip6port[13], ip6port[14], ip6port[15],
+		port);
+   }
+   else
+      return _error->Error("SOCKS proxy %s destination address is of unknown type: %d",
+			   ProxyInfo.c_str(), response[3]);
+   if (response[1] != 0x00)
+   {
+      char const *errstr = nullptr;
+      auto errcode = response[1];
+      // Tor error reporting can be a bit arcane, lets try to detect & fix it up
+      if (bindaddr == "0.0.0.0:0")
+      {
+	 auto const lastdot = Host.rfind('.');
+	 if (lastdot == std::string::npos || Host.substr(lastdot) != ".onion")
+	    ;
+	 else if (errcode == 0x01)
+	 {
+	    auto const prevdot = Host.rfind('.', lastdot - 1);
+	    if (lastdot == 16 && prevdot == std::string::npos)
+	       ; // valid .onion address
+	    else if (prevdot != std::string::npos && (lastdot - prevdot) == 17)
+	       ; // valid .onion address with subdomain(s)
+	    else
+	    {
+	       errstr = "Invalid hostname: onion service name must be 16 characters long";
+	       Owner->SetFailReason("SOCKS");
+	    }
+	 }
+	 // in all likelihood the service is either down or the address has
+	 // a typo and so "Host unreachable" is the better understood error
+	 // compared to the technically correct "TLL expired".
+	 else if (errcode == 0x06)
+	    errcode = 0x04;
+      }
+      if (errstr == nullptr)
+      {
+	 switch (errcode)
+	 {
+	 case 0x01:
+	    errstr = "general SOCKS server failure";
+	    Owner->SetFailReason("SOCKS");
+	    break;
+	 case 0x02:
+	    errstr = "connection not allowed by ruleset";
+	    Owner->SetFailReason("SOCKS");
+	    break;
+	 case 0x03:
+	    errstr = "Network unreachable";
+	    Owner->SetFailReason("ConnectionTimedOut");
+	    break;
+	 case 0x04:
+	    errstr = "Host unreachable";
+	    Owner->SetFailReason("ConnectionTimedOut");
+	    break;
+	 case 0x05:
+	    errstr = "Connection refused";
+	    Owner->SetFailReason("ConnectionRefused");
+	    break;
+	 case 0x06:
+	    errstr = "TTL expired";
+	    Owner->SetFailReason("Timeout");
+	    break;
+	 case 0x07:
+	    errstr = "Command not supported";
+	    Owner->SetFailReason("SOCKS");
+	    break;
+	 case 0x08:
+	    errstr = "Address type not supported";
+	    Owner->SetFailReason("SOCKS");
+	    break;
+	 default:
+	    errstr = "Unknown error";
+	    Owner->SetFailReason("SOCKS");
+	    break;
+	 }
+      }
+      return _error->Error("SOCKS proxy %s could not connect to %s (%s) due to: %s (%d)",
+			   ProxyInfo.c_str(), Host.c_str(), bindaddr.c_str(), errstr, response[1]);
+   }
+   else if (Owner->DebugEnabled())
+      ioprintf(std::clog, "http: SOCKS proxy %s connection established to %s (%s)\n",
+	       ProxyInfo.c_str(), Host.c_str(), bindaddr.c_str());
+
+   if (WaitFd(Fd->Fd(), true, Timeout) == false)
+      return _error->Error("SOCKS proxy %s reported connection to %s (%s), but timed out",
+			   ProxyInfo.c_str(), Host.c_str(), bindaddr.c_str());
+#undef APT_ReadOrFail
+#undef APT_WriteOrFail
+
+   return true;
+}
+									/*}}}*/
+// UnwrapTLS - Handle TLS connections 					/*{{{*/
+// ---------------------------------------------------------------------
+/* Performs a TLS handshake on the socket */
+struct TlsFd : public MethodFd
+{
+   std::unique_ptr<MethodFd> UnderlyingFd;
+   gnutls_session_t session;
+   gnutls_certificate_credentials_t credentials;
+   std::string hostname;
+
+   int Fd() APT_OVERRIDE { return UnderlyingFd->Fd(); }
+
+   ssize_t Read(void *buf, size_t count) APT_OVERRIDE
+   {
+      return HandleError(gnutls_record_recv(session, buf, count));
+   }
+   ssize_t Write(void *buf, size_t count) APT_OVERRIDE
+   {
+      return HandleError(gnutls_record_send(session, buf, count));
+   }
+
+   template <typename T>
+   T HandleError(T err)
+   {
+      if (err < 0 && gnutls_error_is_fatal(err))
+	 errno = EIO;
+      else if (err < 0)
+	 errno = EAGAIN;
+      else
+	 errno = 0;
+      return err;
+   }
+
+   int Close() APT_OVERRIDE
+   {
+      if (HandleError(gnutls_bye(session, GNUTLS_SHUT_RDWR)) < 0)
+	 return -1;
+      return UnderlyingFd->Close();
+   }
+};
+
+bool UnwrapTLS(std::string Host, std::unique_ptr<MethodFd> &Fd,
+	       unsigned long Timeout, aptMethod *Owner)
+{
+   int err;
+   TlsFd *tlsFd = new TlsFd();
+
+   tlsFd->hostname = Host;
+   tlsFd->UnderlyingFd = MethodFd::FromFd(-1); // For now
+
+   gnutls_init(&tlsFd->session, GNUTLS_CLIENT | GNUTLS_NONBLOCK);
+   gnutls_transport_set_int(tlsFd->session, dynamic_cast<FdFd *>(Fd.get())->fd);
+   gnutls_certificate_allocate_credentials(&tlsFd->credentials);
+
+   // Credential setup
+   if ((err = gnutls_certificate_set_x509_system_trust(tlsFd->credentials)) <= 0)
+      return _error->Error("Could not load TLS certificates: %s", gnutls_strerror(err));
+
+   std::string fileinfo = Owner->ConfigFind("CaInfo", "");
+   if (!fileinfo.empty())
+   {
+      gnutls_certificate_set_verify_flags(tlsFd->credentials, GNUTLS_VERIFY_ALLOW_X509_V1_CA_CRT);
+      err = gnutls_certificate_set_x509_trust_file(tlsFd->credentials, fileinfo.c_str(), GNUTLS_X509_FMT_PEM);
+      if (err < 0)
+	 return _error->Error("Could not load CaInfo certificate: %s", gnutls_strerror(err));
+   }
+
+   // TODO: IssuerCert AKA CURLOPT_ISSUERCERT
+   // TODO: Emulate SslForceVersion AKA CURLOPT_SSLVERSION?
+
+   // For client authentication, certificate file ...
+   std::string const cert = Owner->ConfigFind("SslCert", "");
+   std::string const key = Owner->ConfigFind("SslKey", "");
+   if (cert.empty() == false)
+   {
+      if ((err = gnutls_certificate_set_x509_key_file(
+	       tlsFd->credentials,
+	       cert.c_str(),
+	       key.empty() ? cert.c_str() : key.c_str(),
+	       GNUTLS_X509_FMT_PEM)) < 0)
+      {
+	 return _error->Error("Could not load client certificate or key: %s", gnutls_strerror(err));
+      }
+   }
+
+   // CRL file
+   std::string const crlfile = Owner->ConfigFind("CrlFile", "");
+   if (crlfile.empty() == false)
+   {
+      if ((err = gnutls_certificate_set_x509_crl_file(tlsFd->credentials,
+						      crlfile.c_str(),
+						      GNUTLS_X509_FMT_PEM)) < 0)
+	 return _error->Error("Could not load CrlFile: %s", gnutls_strerror(err));
+   }
+
+   if ((err = gnutls_credentials_set(tlsFd->session, GNUTLS_CRD_CERTIFICATE, tlsFd->credentials)) < 0)
+      return _error->Error("Could not set CaInfo certificate: %s", gnutls_strerror(err));
+
+   if ((err = gnutls_set_default_priority(tlsFd->session)) < 0)
+      return _error->Error("Could not set algorithm preferences: %s", gnutls_strerror(err));
+
+   if (Owner->ConfigFindB("Verify-Peer", true) || Owner->ConfigFindB("Verify-Host", true))
+   {
+      gnutls_session_set_verify_cert(tlsFd->session, tlsFd->hostname.c_str(), 0);
+   }
+   if ((err = gnutls_server_name_set(tlsFd->session, GNUTLS_NAME_DNS, tlsFd->hostname.c_str(), tlsFd->hostname.length())) < 0)
+      return _error->Error("Could not set SNI name: %s", gnutls_strerror(err));
+
+   // Do the handshake. Our socket is non-blocking, so we need to call WaitFd()
+   // accordingly.
+   do
+   {
+      err = gnutls_handshake(tlsFd->session);
+      if ((err == GNUTLS_E_INTERRUPTED || err == GNUTLS_E_AGAIN) &&
+	  WaitFd(Fd->Fd(), gnutls_record_get_direction(tlsFd->session) == 1, Timeout) == false)
+	 return _error->Errno("select", "Could not wait for server fd");
+   } while (err < 0 && gnutls_error_is_fatal(err) == 0);
+
+   if (err < 0)
+   {
+      // Print reason why validation failed.
+      if (err == GNUTLS_E_CERTIFICATE_VERIFICATION_ERROR)
+      {
+	 gnutls_datum_t txt;
+	 auto type = gnutls_certificate_type_get(tlsFd->session);
+	 auto status = gnutls_session_get_verify_cert_status(tlsFd->session);
+	 if (gnutls_certificate_verification_status_print(status,
+							  type, &txt, 0) == 0)
+	 {
+	    _error->Error("Certificate verification failed: %s", txt.data);
+	 }
+	 gnutls_free(txt.data);
+      }
+      return _error->Error("Could not handshake: %s", gnutls_strerror(err));
+   }
+
+   tlsFd->UnderlyingFd = std::move(Fd);
+   Fd.reset(tlsFd);
+   return true;
+}
+									/*}}}*/