Support for HTTP CONNECT Proxies

include/motis/config.h

@@ -209,6 +209,7 @@ struct config {
     unsigned http_timeout_{30};
     unsigned cache_size_{50};
     std::optional<std::string> proxy_{};
+    std::optional<bool> use_connect_{false};
     std::optional<ttl> ttl_{};
   };
   std::optional<gbfs> gbfs_{};

include/motis/http_req.h

@@ -1,23 +1,33 @@
 #pragma once
 
+#include <boost/beast/core/stream_traits.hpp>
 #include <chrono>
 #include <map>
+#include <print>
 #include <string>
 
 #include "boost/asio/awaitable.hpp"
+#include "boost/beast/core/flat_buffer.hpp"
 #include "boost/beast/http/dynamic_body.hpp"
-#include "boost/beast/http/message.hpp"
+#include "boost/beast/http/empty_body.hpp"
+#include "boost/beast/http/read.hpp"
+#include "boost/beast/http/write.hpp"
 #include "boost/url/url.hpp"
 
+#include "utl/verify.h"
+
 namespace motis {
 
-constexpr auto const kBodySizeLimit = 512U * 1024U * 1024U;  // 512 M
+namespace beast = boost::beast;
+namespace http = beast::http;
 
-using http_response =
-    boost::beast::http::response<boost::beast::http::dynamic_body>;
+using http_response = http::response<boost::beast::http::dynamic_body>;
+
+constexpr auto const kBodySizeLimit = 512U * 1024U * 1024U;  // 512 M
 
 struct proxy {
   bool use_tls_;
+  bool use_connect_;
   std::string host_, port_;
 };
 
@@ -34,6 +44,33 @@ boost::asio::awaitable<http_response> http_POST(
     std::chrono::seconds timeout,
     std::optional<proxy> const& = std::nullopt);
 
+template <typename Stream>
+boost::asio::awaitable<void> http_CONNECT(Stream& stream,
+                                          boost::urls::url const& url,
+                                          std::optional<proxy> const& proxy) {
+  if (!proxy) {
+    co_return;
+  }
+  std::println("making a connect request");
+  auto const target = std::string(url.host()) + ":" +
+                      (url.has_port() ? std::string(url.port()) : "443");
+
+  http::request<http::empty_body> req{http::verb::connect, target, 11};
+  req.set(http::field::host, target);
+  beast::flat_buffer buf;
+
+  co_await http::async_write(beast::get_lowest_layer(stream), req);
+
+  http::response_parser<http::empty_body> res;
+  res.skip(true);
+  co_await http::async_read_header(beast::get_lowest_layer(stream), buf, res);
+
+  if (res.get().result() != http::status::ok) {
+    throw utl::fail("CONNECT failed: target={}, status={}", target,
+                    res.get().result_int());
+  }
+}
+
 std::string get_http_body(http_response const&);
 
 }  // namespace motis

src/gbfs/update.cc

@@ -165,12 +165,13 @@ struct gbfs_update {
         d_{d},
         prev_d_{prev_d},
         timeout_{c.http_timeout_},
-        proxy_{c.proxy_.transform([](std::string const& u) {
+        proxy_{c.proxy_.transform([&](std::string const& u) {
           auto const url = boost::urls::url{u};
 
           auto p = proxy{};
           p.use_tls_ = url.scheme_id() == boost::urls::scheme::https;
           p.host_ = url.host();
+          p.use_connect_ = c.use_connect_.value_or(false);
           p.port_ = url.has_port() ? url.port() : (p.use_tls_ ? "443" : "80");
           return p;
         })} {}

src/http_req.cc

@@ -52,6 +52,7 @@ asio::awaitable<http_response> req_no_tls(
   stream.expires_after(timeout);
 
   co_await stream.async_connect(results);
+
   co_return co_await req(std::move(stream), url, headers, body);
 }
 
@@ -71,22 +72,27 @@ asio::awaitable<http_response> req_tls(
   auto resolver = asio::ip::tcp::resolver{executor};
   auto stream = ssl::stream<beast::tcp_stream>{executor, ssl_ctx};
 
-  auto const host = proxy ? proxy->host_ : url.host();
-  auto const port =
-      proxy ? proxy->port_ : std::string{url.has_port() ? url.port() : "443"};
-
-  if (!SSL_set_tlsext_host_name(stream.native_handle(),
-                                const_cast<char*>(host.c_str()))) {
-    throw boost::system::system_error{
-        {static_cast<int>(::ERR_get_error()), asio::error::get_ssl_category()}};
-  }
+  auto const target_port = std::string{url.has_port() ? url.port() : "443"};
+  auto const target_host = url.host();
 
   auto const results = co_await resolver.async_resolve(
-      host, port, asio::cancel_after(timeout, asio::use_awaitable));
+      proxy ? proxy->host_ : target_host, proxy ? proxy->port_ : target_port,
+      asio::cancel_after(timeout, asio::use_awaitable));
 
   stream.next_layer().expires_after(timeout);
 
   co_await beast::get_lowest_layer(stream).async_connect(results);
+
+  if (proxy && proxy->use_connect_) {
+    co_await http_CONNECT(stream, url, proxy);
+  }
+
+  if (!SSL_set_tlsext_host_name(stream.native_handle(),
+                                const_cast<char*>(target_host.c_str()))) {
+    throw boost::system::system_error{
+        {static_cast<int>(::ERR_get_error()), asio::error::get_ssl_category()}};
+  }
+
   co_await stream.async_handshake(ssl::stream_base::client);
   co_return co_await req(std::move(stream), url, headers, body);
 }
@@ -134,9 +140,7 @@ asio::awaitable<http::response<http::dynamic_body>> http_GET(
   auto n_redirects = 0U;
   auto next_url = url;
   while (n_redirects < 3U) {
-    auto const use_tls =
-        proxy.has_value() ? proxy->use_tls_
-                          : next_url.scheme_id() == boost::urls::scheme::https;
+    auto const use_tls = next_url.scheme_id() == boost::urls::scheme::https;
     auto const res = co_await (
         use_tls ? req_tls(next_url, headers, std::nullopt, timeout, proxy)
                 : req_no_tls(next_url, headers, std::nullopt, timeout, proxy));
@@ -162,9 +166,7 @@ asio::awaitable<http::response<http::dynamic_body>> http_POST(
   auto n_redirects = 0U;
   auto next_url = url;
   while (n_redirects < 3U) {
-    auto const use_tls =
-        proxy.has_value() ? proxy->use_tls_
-                          : next_url.scheme_id() == boost::urls::scheme::https;
+    auto const use_tls = next_url.scheme_id() == boost::urls::scheme::https;
     auto const res = co_await (
         use_tls ? req_tls(next_url, headers, body, timeout, proxy)
                 : req_no_tls(next_url, headers, body, timeout, proxy));