HTTP/3

Author: puzza007

.travis.yml

@@ -1,3 +1,4 @@
+---
 sudo: required
 dist: xenial
 group: edge
@@ -9,27 +10,47 @@ services:
 
 install:
   - ORIG_DIR=$PWD
+  - export PATH=$PATH:$HOME/.cargo/bin
+  - git clone --recursive https://github.com/cloudflare/quiche
+  - cd quiche/deps/boringssl
+  - mkdir build
+  - cd build
+  - cmake -DCMAKE_POSITION_INDEPENDENT_CODE=on ..
+  - make -j`nproc`
+  - cd ..
+  - mkdir .openssl/lib -p
+  - cp build/crypto/libcrypto.a build/ssl/libssl.a .openssl/lib
+  - ln -s $PWD/include .openssl
+  - cd ../..
+  - QUICHE_BSSL_PATH=$PWD/deps/boringssl cargo build --release --features pkg-config-meta
+  - cd ..
   - wget https://github.com/erlang/rebar3/releases/download/3.10.0/rebar3 && chmod 755 rebar3
-  - wget https://github.com/curl/curl/releases/download/curl-7_64_1/curl-7.64.1.tar.gz && tar xf curl-7.64.1.tar.gz
-  - cd curl-7.64.1 && ./configure && make && sudo make install
+  - git clone https://github.com/curl/curl
+  - cd curl
+  - ./buildconf
+  - ./configure LDFLAGS="-Wl,-rpath,$PWD/../quiche/target/release" --with-ssl=$PWD/../quiche/deps/boringssl/.openssl --with-quiche=$PWD/../quiche/target/release
+  - make -j`nproc`
+  - sudo make install
   - cd $ORIG_DIR
 
 otp_release:
-  - 20.3
   - 21.3
+  - 22.0
 
 script:
-  - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib
+  - export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
   - ./rebar3 update && ./rebar3 ct && ./rebar3 dialyzer && ./rebar3 coveralls send
 
 before_install:
   - sudo ifconfig
   - sudo apt-get update -q
-  - sudo apt-get install -qy build-essential libssl-dev libnghttp2-dev
+  - sudo apt-get install -qy build-essential libssl-dev libnghttp2-dev curl git cmake
+  - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs |  sh -s -- -y
+  - sudo apt purge -y curl && sudo apt autoremove -y
 
 cache:
   directories:
-  - $HOME/.cache/rebar3/
+    - $HOME/.cache/rebar3/
 
 addons:
   apt:

README.md

@@ -1,7 +1,7 @@
 katipo
 =====
 
-An HTTP/HTTP2 client library for Erlang built around libcurl-multi and libevent.
+An HTTP/HTTP2/HTTP3 client library for Erlang built around libcurl-multi and libevent.
 
 ### Status
 
@@ -122,7 +122,7 @@ katipo:Method(Pool :: atom(), URL :: binary(), ReqOptions :: map()).
 | `unix_socket_path`      | `binary()`                    | `undefined` | [docs](https://curl.haxx.se/libcurl/c/CURLOPT_UNIX_SOCKET_PATH.html) curl >= 7.40.0 |
 | `lock_data_ssl_session` | `boolean()`                   | `false`     | [docs](https://curl.haxx.se/libcurl/c/curl_share_setopt.html) curl >= 7.23.0        |
 | `doh_url`               | `binary()`                    | `undefined` | [docs](https://curl.haxx.se/libcurl/c/CURLOPT_DOH_URL.html) curl >= 7.62.0          |
-| `http_version`          | `curl_http_version_none` <br> `curl_http_version_1_0` <br> `curl_http_version_1_1` <br> `curl_http_version_2_0` <br> `curl_http_version_2tls` <br> `curl_http_version_2_prior_knowledge` | `curl_http_version_none` | [docs](https://curl.haxx.se/libcurl/c/CURLOPT_HTTP_VERSION.html) curl >= 7.62.0          |
+| `http_version`          | `curl_http_version_none` <br> `curl_http_version_1_0` <br> `curl_http_version_1_1` <br> `curl_http_version_2_0` <br> `curl_http_version_2tls` <br> `curl_http_version_2_prior_knowledge` <br> `curl_http_version_3` | `curl_http_version_none` | [docs](https://curl.haxx.se/libcurl/c/CURLOPT_HTTP_VERSION.html) curl >= 7.62.0          |
 
 #### Responses
 

c_src/Makefile

@@ -25,6 +25,7 @@ else ifeq ($(UNAME_SYS), Linux)
 	CC ?= gcc
 	CFLAGS ?= -O3 -std=c99 -finline-functions -Wall -Wmissing-prototypes
 	CXXFLAGS ?= -O3 -finline-functions -Wall
+	LDLIBS += -L /usr/local/lib
 endif
 
 CFLAGS += -fPIC -I $(ERTS_INCLUDE_DIR) -I $(ERL_INTERFACE_INCLUDE_DIR)

c_src/katipo.c

@@ -56,6 +56,7 @@ typedef struct _GlobalInfo {
   CURLSH *shobject;
   int still_running;
   size_t to_get;
+  curl_version_info_data *ver;
 } GlobalInfo;
 
 typedef struct _ConnInfo {
@@ -665,19 +666,30 @@ static size_t write_cb(void *ptr, size_t size, size_t nmemb, void *data) {
 
 static size_t header_cb(void *ptr, size_t size, size_t nmemb, void *data) {
   size_t realsize = size * nmemb;
+  size_t adjusted_size = realsize;
   ConnInfo *conn = (ConnInfo *)data;
   char *header;
 
-  // the last two chars of headers are \r\n
+  // the last two chars of headers are \r\n except for http3...
   if (realsize > 2) {
     if (conn->resp_headers && is_status_line(ptr)) {
       curl_slist_free_all(conn->resp_headers);
       conn->resp_headers = NULL;
       conn->num_headers = 0;
     }
-    header = (char *)malloc(realsize - 1);
-    strncpy(header, ptr, realsize - 2);
-    header[realsize - 2] = '\0';
+    // TODO: understand better what's going on with http3
+    // headers. They don't seem to have the trailing \r\n that the
+    // former HTTP versions do
+    #if LIBCURL_VERSION_NUM >= 0x074200
+    long http_version;
+    curl_easy_getinfo(conn->easy, CURLINFO_HTTP_VERSION, &http_version);
+    if (http_version == CURL_HTTP_VERSION_3) {
+      adjusted_size += 1;
+    }
+    #endif
+    header = (char *)malloc(adjusted_size - 1);
+    strncpy(header, ptr, adjusted_size - 2);
+    header[adjusted_size - 2] = '\0';
     conn->resp_headers = curl_slist_append(conn->resp_headers, header);
     free(header);
     conn->num_headers++;
@@ -802,9 +814,11 @@ static void new_conn(long method, char *url, struct curl_slist *req_headers,
                      eopts.curlopt_interface);
   }
   #if LIBCURL_VERSION_NUM >= 0x072800 /* Available since 7.40.0 */
-  if (eopts.curlopt_unix_socket_path != NULL) {
-    curl_easy_setopt(conn->easy, CURLOPT_UNIX_SOCKET_PATH,
-                     eopts.curlopt_unix_socket_path);
+  if (global->ver->features & CURL_VERSION_UNIX_SOCKETS) {
+    if (eopts.curlopt_unix_socket_path != NULL) {
+      curl_easy_setopt(conn->easy, CURLOPT_UNIX_SOCKET_PATH,
+                       eopts.curlopt_unix_socket_path);
+    }
   }
   #endif
   #if LIBCURL_VERSION_NUM >= 0x073100 /* Available since 7.49.0 */
@@ -1131,6 +1145,7 @@ int main(int argc, char **argv) {
   int option_index = 0;
   int c;
   long pipelining = 0;
+  curl_version_info_data *ver;
 
   struct option long_options[] = {
     { "pipelining", required_argument, 0, 'p' },
@@ -1158,6 +1173,8 @@ int main(int argc, char **argv) {
   }
   global.timer_event = evtimer_new(global.evbase, timer_cb, &global);
   global.to_get = 0;
+  ver = curl_version_info(CURLVERSION_NOW);
+  global.ver = ver;
 
   curl_multi_setopt(global.multi, CURLMOPT_SOCKETFUNCTION, sock_cb);
   curl_multi_setopt(global.multi, CURLMOPT_SOCKETDATA, &global);

rebar.config

@@ -15,7 +15,6 @@
    [{deps,
      [{jsx, "2.9.0"},
       {meck, "0.8.10"},
-      {cowboy, "2.4.0"},
       {http_proxy, ".*", {git, "https://github.com/puzza007/http_proxy.git", {branch, "rebar3"}}}
      ]}]
   }]

src/katipo.erl

@@ -253,7 +253,8 @@
                                 curl_http_version_1_1 |
                                 curl_http_version_2_0 |
                                 curl_http_version_2tls |
-                                curl_http_version_2_prior_knowledge.
+                                curl_http_version_2_prior_knowledge |
+                                curl_http_version_3.
 -type curlmopts() :: [{max_pipeline_length, non_neg_integer()} |
                       {pipelining, pipelining()} |
                       {max_total_connections, non_neg_integer()}].
@@ -652,6 +653,8 @@ opt(http_version, curl_http_version_2tls, {Req, Errors}) ->
     {Req#req{http_version=4}, Errors};
 opt(http_version, curl_http_version_2_prior_knowledge, {Req, Errors}) ->
     {Req#req{http_version=5}, Errors};
+opt(http_version, curl_http_version_3, {Req, Errors}) ->
+    {Req#req{http_version=30}, Errors}; %% See https://github.com/curl/curl/blob/32d64b2e875f0d74cd433dff8bda9f8a98dcd44e/include/curl/curl.h#L1983
 opt(verbose, true, {Req, Errors}) ->
     {Req#req{verbose=?VERBOSE_TRUE}, Errors};
 opt(verbose, false, {Req, Errors}) ->

test/katipo_SUITE.erl

@@ -26,16 +26,8 @@ init_per_group(pool, Config) ->
     application:ensure_all_started(meck),
     Config;
 init_per_group(https, Config) ->
-    application:ensure_all_started(cowboy),
-    Dispatch = cowboy_router:compile([{'_', [{"/", get_handler, []}]}]),
     DataDir = ?config(data_dir, Config),
-    CACert = filename:join(DataDir, "cowboy-ca.crt"),
-    {ok, _} = cowboy:start_tls(ct_https,
-                               [{port, 8443},
-                                {cacertfile, CACert},
-                                {certfile, filename:join(DataDir, "server.crt")},
-                                {keyfile, filename:join(DataDir, "server.key")}],
-                               #{env => #{dispatch => Dispatch}}),
+    CACert = filename:join(DataDir, "ca-bundle.crt"),
     [{cacert_file, list_to_binary(CACert)} | Config];
 init_per_group(proxy, Config) ->
     application:ensure_all_started(http_proxy),
@@ -148,7 +140,9 @@ groups() ->
       [metrics_true,
        metrics_false]},
      {http2, [parallel],
-      [http2_get]}].
+      [http2_get]},
+     {http3, [parallel],
+      [http3_get]}].
 
 all() ->
     [{group, http},
@@ -158,7 +152,8 @@ all() ->
      {group, session},
      {group, port},
      {group, metrics},
-     {group, http2}].
+     {group, http2},
+     {group, http3}].
 
 get(_) ->
     {ok, #{status := 200, body := Body}} =
@@ -614,31 +609,31 @@ verify_host_verify_peer_ok(_) ->
 
 verify_host_verify_peer_error(_) ->
     {error, #{code := Code}} =
-         katipo:get(?POOL, <<"https://localhost:8443">>,
+         katipo:get(?POOL, <<"https://self-signed.badssl.com/">>,
                     #{ssl_verifyhost => true, ssl_verifypeer => true}),
     %% TODO: this could be made to reflect the ifdef from katipo.c...
     ok = case Code of
              ssl_cacert -> ok;
              peer_failed_verification -> ok
          end,
     {error, #{code := Code}} =
-         katipo:get(?POOL, <<"https://localhost:8443">>,
+         katipo:get(?POOL, <<"https://self-signed.badssl.com/">>,
                     #{ssl_verifyhost => false, ssl_verifypeer => true}),
     ok = case Code of
              ssl_cacert -> ok;
              peer_failed_verification -> ok
          end,
     {ok, #{status := 200}} =
-        katipo:get(?POOL, <<"https://localhost:8443">>,
+        katipo:get(?POOL, <<"https://self-signed.badssl.com/">>,
                    #{ssl_verifyhost => true, ssl_verifypeer => false}),
     {ok, #{status := 200}} =
-        katipo:get(?POOL, <<"https://localhost:8443">>,
+        katipo:get(?POOL, <<"https://self-signed.badssl.com/">>,
                    #{ssl_verifyhost => false, ssl_verifypeer => false}).
 
 cacert_self_signed(Config) ->
     CACert = ?config(cacert_file, Config),
-    {ok, #{status := 200}} =
-        katipo:get(?POOL, <<"https://localhost:8443">>,
+    {ok, #{status := 301}} =
+        katipo:get(?POOL, <<"https://google.com">>,
                    #{ssl_verifyhost => true, ssl_verifypeer => true, cacert => CACert}).
 
 badssl(_) ->
@@ -790,6 +785,12 @@ http2_get(_) ->
     Json = jsx:decode(Body),
     [{<<"a">>, <<"!@#$%^&*()_+">>}] = proplists:get_value(<<"args">>, Json).
 
+http3_get(_) ->
+    {ok, #{status := 404, headers := Headers}} =
+        katipo:get(?POOL, <<"https://quic.tech:8443">>,
+                   #{http_version => curl_http_version_3, verbose => true}),
+    <<"quiche">> = proplists:get_value(<<"server">>, Headers).
+
 repeat_until_true(Fun) ->
     try
         case Fun() of