Add support for QUIC - HTTP/3

[JCMais]

OpenSSL does not provide a production-ready version with QUIC enabled, we will need to wait until then: openssl/openssl#8797

For the QUIC/HTTP3 backend we will need to use ngtcp2 as quiche only works with BoringSSL.

To test the QUIC implementation, the ngtcp2 team offers a patched OpenSSL version with QUIC support here: https://github.com/tatsuhiro-t/openssl/tree/openssl-quic or we could just built openssl from the akamai branch in the PR above.

[BitFis]

Dockerfile to test with the latest curl version support quic-draft-29

Dockerfile

FROM node:14.4.0-stretch

# install dependencies
RUN apt-get update && \
    apt-get upgrade -y && \
    apt-get install -y \
    git \
    make \
    build-essential \
    autoconf \
    libtool \
    pkg-config

RUN mkdir "/build-curl"
WORKDIR "/build-curl"

RUN echo $(pwd)

#ENV OPENSSL_CHECKOUT OpenSSL_1_1_1d-quic-draft-27
ENV OPENSSL_CHECKOUT OpenSSL_1_1_1g-quic-draft-29

# clone and build openssl 1.1.1
# last tested commit/5332ff475df45a123e654aded3406737c6232335
RUN git clone --depth 1 -b "$OPENSSL_CHECKOUT" https://github.com/tatsuhiro-t/openssl && \
    cd openssl/ && \
    mkdir ../openssllib && \
    ./config enable-tls1_3 --prefix=$(pwd)/../openssllib/ && \
    make && \
    make install_sw && \
    cd ../

# clone and build nghttp3
# last tested commit/6c926bb01b4ce5fc4d14d362601a2c01a1a543ea
RUN git clone https://github.com/ngtcp2/nghttp3 && \
    cd nghttp3/ && \
    mkdir ../nghttp3lib && \
    autoreconf -i && \
    ./configure --prefix=$(pwd)/../nghttp3lib --enable-lib-only && \
    make && \
    make install && \
    cd ../

# clone and build ngtcp2
# last tested commit/d6ba5c63dfcd846746d91101c6c51b9dd0e28de0
RUN git clone https://github.com/ngtcp2/ngtcp2 && \
    cd ngtcp2/ && \
    mkdir ../ngtcp2lib && \
    autoreconf -i && \
    ./configure PKG_CONFIG_PATH=$(pwd)/../openssllib/lib/pkgconfig:$(pwd)/../nghttp3lib/lib/pkgconfig LDFLAGS="-Wl,-rpath,$(pwd)/../openssllib/lib" --prefix=$(pwd)/../ngtcp2lib && \
    make && \
    make install && \
    cd ../

# clone and build curl with built dependencies
# last tested commit/c585bad8e219994be127e044c57c6d0d84fe4f41
RUN git clone https://github.com/curl/curl && \
    cd curl/ && \
    ./buildconf && \
    PKG_CONFIG_PATH=../openssllib/lib/pkgconfig:$(pwd)/../nghttp3lib/lib/pkgconfig:$(pwd)/../ngtcp2lib/lib/pkgconfig \
        LDFLAGS="-Wl,-rpath,$(pwd)/../openssllib/lib" ./configure \
            --with-ssl=$(pwd)/../openssllib \
            --with-nghttp3=$(pwd)/../nghttp3lib \
            --with-ngtcp2=$(pwd)/../ngtcp2lib \
            --enable-alt-svc && \
    make && \
    make install

RUN echo "/usr/local/lib" > /etc/ld.so.conf.d/libcurl.conf && \
    ldconfig

RUN mkdir /app
WORKDIR /app
ADD simple-client.js ./

RUN npm install --build-from-source node-libcurl

simple-client.js

(async () => {
    const { curly, Curl, CurlHttpVersion, Easy } = require('node-libcurl');

    const curl = new Curl();

    console.log("getVersionInfo", Curl.getVersionInfo())

    // tried with custom implementation, does not work HTTP3 curl implementation seems to not work with nodejs
    curl.setOpt(Curl.option.HTTP_VERSION, 30);
    curl.setOpt('URL', 'https://quic.rocks:4433');

    curl.on('end', function (statusCode, data, headers) {
        console.info(statusCode);
        console.info('---');
        console.info(data.length);
        console.info('---');
        console.info(this.getInfo( 'TOTAL_TIME'));
        console.info(this.getInfo( 'HTTP_VERSION'));
        console.log(data);

        this.close();
    });
    curl.on('error', curl.close.bind(curl));
    curl.perform();

})();

Prebuilt version: https://hub.docker.com/repository/docker/bigsisl/node-libcurl-http3

[BitFis]

llnode dump;

Process 33180 stopped
* thread #1, name = 'node', stop reason = signal SIGSEGV: invalid address (fault address: 0x0)
    frame #0: 0x000000000166eb59 node`OPENSSL_cleanse + 41
node`OPENSSL_cleanse:
->  0x166eb59 <+41>: movb   %al, (%rdi)
    0x166eb5b <+43>: leaq   -0x1(%rsi), %rsi
    0x166eb5f <+47>: leaq   0x1(%rdi), %rdi
    0x166eb63 <+51>: jmp    0x166eb50                 ; <+32>

gdb stacktrace:

#0  __GI___libc_free (mem=0x2e621a6a1744140f) at malloc.c:3103
#1  0x00000000014ed05e in BN_free ()
#2  0x00000000014aa0dd in SSL_SRP_CTX_free ()
#3  0x00000000014662ce in ssl3_free ()
#4  0x0000000001471ae6 in SSL_free.part.26 ()
#5  0x00007fffe7de2bfa in qs_disconnect () from /usr/local/lib/libcurl.so.4
#6  0x00007fffe7de421b in Curl_quic_is_connected () from /usr/local/lib/libcurl.so.4
#7  0x00007fffe7d92798 in Curl_is_connected () from /usr/local/lib/libcurl.so.4
#8  0x00007fffe7dba4e0 in multi_runsingle () from /usr/local/lib/libcurl.so.4
#9  0x00007fffe7dbb883 in multi_socket () from /usr/local/lib/libcurl.so.4
#10 0x00007fffe7dbb9f4 in curl_multi_socket_action () from /usr/local/lib/libcurl.so.4
#11 0x00007ffff426da8c in NodeLibcurl::Multi::OnSocket (handle=<optimized out>, status=<optimized out>, events=<optimized out>) at ../src/Multi.cc:153
#12 0x0000000001336c50 in uv.io_poll () at ../deps/uv/src/unix/linux-core.c:431
#13 0x00000000013250f8 in uv_run (loop=0x4291ea0 <default_loop_struct>, mode=UV_RUN_DEFAULT) at ../deps/uv/src/unix/core.c:381
#14 0x0000000000a6b9b4 in node::NodeMainInstance::Run() ()
#15 0x00000000009fae81 in node::Start(int, char**) ()
#16 0x00007ffff6ca3b97 in __libc_start_main (main=0x993d80 <main>, argc=2, argv=0x7fffffffda58, init=<optimized out>, fini=<optimized out>, rtld_fini=<optimized out>, stack_end=0x7fffffffda48) at ../csu/libc-start.c:310
#17 0x000000000099529c in _start ()

[BitFis]

Found some things, a ng_flush_egress(conn, sockfd, qs) fails with CURLE_SEND_ERROR. This happens at:
https://github.com/curl/curl/blob/1313d7a35620e40c765bd8dc4138916264f20e61/lib/vquic/ngtcp2.c#L1692-L1718
because of:
https://github.com/curl/curl/blob/1313d7a35620e40c765bd8dc4138916264f20e61/lib/vquic/ngtcp2.c#L1862-L1866
(ngtcp2_conn_write_pkt returned error: -502 / NGTCP2_ERR_CALLBACK_FAILURE)

called by:

node-libcurl/src/Multi.cc

Lines 164 to 165 in c563f5e

	code = curl_multi_socket_action(ctx->multi->mh, ctx->sockfd, flags,
	&ctx->multi->runningHandles);

I am not sure if the bug lies with node-libcurl or curl it self. It does seems strange thought, since after the CURLE_SEND_ERROR libcurl tries to disconnect and the ssl pointer seems to be not correct, therefore it fails => SIGFAULT.

[JCMais]

Could you check if you are using the same OpenSSL version than the one being used by the Node.js you are building against?

You can get the Node.js OpenSSL version by running node -e "console.log(process.versions)".

If they are different, this can cause some weird issues, like the one you are having. The akamai fork has some branches for the latest OpenSSL versions, like: https://github.com/akamai/openssl/tree/OpenSSL_1_1_1f-quic, but I don't know if they do work or not.

[BitFis]

I am testing it locally again, but in the container i use OpenSSL_1_1_1g-quic-draft-29 and

root@4808d4ebfb95:/app# node -e "console.log(process.versions)"
{
  node: '14.4.0',
  v8: '8.1.307.31-node.33',
  uv: '1.37.0',
  zlib: '1.2.11',
  brotli: '1.0.7',
  ares: '1.16.0',
  modules: '83',
  nghttp2: '1.41.0',
  napi: '6',
  llhttp: '2.0.4',
  openssl: '1.1.1g',
  cldr: '37.0',
  icu: '67.1',
  tz: '2019c',
  unicode: '13.0'
}

Therefore might be something else

[BitFis]

Nope, getting the same error CURLE_SEND_ERROR followed by SIGFAULT

[JCMais]

The same error happens when using only an Easy instance?

Just in case you want to check if this happens with libcurl only, there are some sample code available on libcurl documentation that you can use, like https://curl.haxx.se/libcurl/c/http3.html and https://curl.haxx.se/libcurl/c/multi-uv.html (need to also build and link libuv).

The second example above would be almost exactly what we do in the addon when you use a Multi / Curl instance.

[BitFis]

Seems like bare multi-uv and http3 example work with the version I compiled, I will try to run those examples now as part as nodejs callbacks, hopefully I will figure something out.

Sourcecode & Results

CMakeLists.txt

cmake_minimum_required(VERSION 3.0.0)
project("LibCurl HTTP3 Client Example" VERSION 0.1.0)

set(CMAKE_EXPORT_COMPILE_COMMANDS ON)

set(CURL_LIBRARY "-lcurl")
find_package(CURL REQUIRED)

include(CTest)
enable_testing()

add_executable(http3-client main.cpp)

set_property(TARGET http3-client PROPERTY CXX_STANDARD 17)

MESSAGE("CURL_INCLUDE_DIR: ${CURL_INCLUDE_DIR}")
MESSAGE("CURL_LIBRARIES: ${CURL_LIBRARIES}")

target_include_directories(
    http3-client PRIVATE
    ${CURL_INCLUDE_DIR}
)

target_link_libraries(http3-client ${CURL_LIBRARIES})

add_executable(multi-uv mutli-uv.cpp)

set_property(TARGET multi-uv PROPERTY CXX_STANDARD 17)

target_include_directories(
    multi-uv PRIVATE
    ${CURL_INCLUDE_DIR}
)

target_link_libraries(multi-uv ${CURL_LIBRARIES} -luv)

set(CPACK_PROJECT_NAME ${PROJECT_NAME})
set(CPACK_PROJECT_VERSION ${PROJECT_VERSION})
include(CPack)

mutli-uv.cpp

/***************************************************************************
 *                                  _   _ ____  _
 *  Project                     ___| | | |  _ \| |
 *                             / __| | | | |_) | |
 *                            | (__| |_| |  _ <| |___
 *                             \___|\___/|_| \_\_____|
 *
 * Copyright (C) 1998 - 2020, Daniel Stenberg, <[email protected]>, et al.
 *
 * This software is licensed as described in the file COPYING, which
 * you should have received as part of this distribution. The terms
 * are also available at https://curl.haxx.se/docs/copyright.html.
 *
 * You may opt to use, copy, modify, merge, publish, distribute and/or sell
 * copies of the Software, and permit persons to whom the Software is
 * furnished to do so, under the terms of the COPYING file.
 *
 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
 * KIND, either express or implied.
 *
 ***************************************************************************/

/* <DESC>
 * multi_socket API using libuv
 * </DESC>
 */
/* Example application using the multi socket interface to download multiple
   files in parallel, powered by libuv.

   Requires libuv and (of course) libcurl.

   See https://nikhilm.github.io/uvbook/ for more information on libuv.
*/

#include <stdio.h>
#include <stdlib.h>
#include <uv.h>
#include <curl/curl.h>
#include <curl/easy.h>
#include <assert.h>

uv_loop_t *loop;
CURLM *curl_handle;
uv_timer_t timeout;

typedef struct curl_context_s {
  uv_poll_t poll_handle;
  curl_socket_t sockfd;
} curl_context_t;

static curl_context_t *create_curl_context(curl_socket_t sockfd)
{
  curl_context_t *context;

  context = (curl_context_t *) malloc(sizeof(*context));

  context->sockfd = sockfd;

  uv_poll_init_socket(loop, &context->poll_handle, sockfd);
  context->poll_handle.data = context;

  return context;
}

static void curl_close_cb(uv_handle_t *handle)
{
  curl_context_t *context = (curl_context_t *) handle->data;
  free(context);
}

static void destroy_curl_context(curl_context_t *context)
{
  uv_close((uv_handle_t *) &context->poll_handle, curl_close_cb);
}

static void add_download(const char *url, int num)
{
  char filename[50];
  FILE *file;
  CURL *handle;

  snprintf(filename, 50, "%d.download", num);

  file = fopen(filename, "wb");
  if(!file) {
    fprintf(stderr, "Error opening %s\n", filename);
    return;
  }

  handle = curl_easy_init();
//  curl_easy_setopt(handle, CURLOPT_WRITEDATA, file);
  curl_easy_setopt(handle, CURLOPT_PRIVATE, file);
  curl_easy_setopt(handle, CURLOPT_URL, url);

  CURLcode resHttp3 = curl_easy_setopt(handle, CURLOPT_HTTP_VERSION, (long)CURL_HTTP_VERSION_3);
  printf("resHttp3: [%d]\n", resHttp3);

  curl_multi_add_handle(curl_handle, handle);
//  fprintf(stderr, "Added download %s -> %s\n", url, filename);
}

static void check_multi_info(void)
{
  char *done_url;
  CURLMsg *message;
  int pending;
  CURL *easy_handle;
  FILE *file = nullptr;

  while((message = curl_multi_info_read(curl_handle, &pending))) {
    switch(message->msg) {
    case CURLMSG_DONE:
      /* Do not use message data after calling curl_multi_remove_handle() and
         curl_easy_cleanup(). As per curl_multi_info_read() docs:
         "WARNING: The data the returned pointer points to will not survive
         calling curl_multi_cleanup, curl_multi_remove_handle or
         curl_easy_cleanup." */
      easy_handle = message->easy_handle;

      curl_easy_getinfo(easy_handle, CURLINFO_EFFECTIVE_URL, &done_url);
      curl_easy_getinfo(easy_handle, CURLINFO_PRIVATE, &file);

      printf("%s DONE\n", done_url);

      printf("CODE [%s]\n", curl_easy_strerror(message->data.result));
      printf("Content");

      curl_multi_remove_handle(curl_handle, easy_handle);
      curl_easy_cleanup(easy_handle);
      if(file) {
        fclose(file);
      }
      break;

    default:
      fprintf(stderr, "CURLMSG default\n");
      break;
    }
  }
}

static void curl_perform(uv_poll_t *req, int status, int events)
{
  int running_handles;
  int flags = 0;
  curl_context_t *context;

  if(events & UV_READABLE)
    flags |= CURL_CSELECT_IN;
  if(events & UV_WRITABLE)
    flags |= CURL_CSELECT_OUT;

  context = (curl_context_t *) req->data;

  curl_multi_socket_action(curl_handle, context->sockfd, flags,
                           &running_handles);

  check_multi_info();
}

static void on_timeout(uv_timer_t *req)
{
  int running_handles;
  curl_multi_socket_action(curl_handle, CURL_SOCKET_TIMEOUT, 0,
                           &running_handles);
  check_multi_info();
}

static int start_timeout(CURLM *multi, long timeout_ms, void *userp)
{
  if(timeout_ms < 0) {
    uv_timer_stop(&timeout);
  }
  else {
    if(timeout_ms == 0)
      timeout_ms = 1; /* 0 means directly call socket_action, but we'll do it
                         in a bit */
    uv_timer_start(&timeout, on_timeout, timeout_ms, 0);
  }
  return 0;
}

static int handle_socket(CURL *easy, curl_socket_t s, int action, void *userp,
                  void *socketp)
{
  curl_context_t *curl_context;
  int events = 0;

  switch(action) {
  case CURL_POLL_IN:
  case CURL_POLL_OUT:
  case CURL_POLL_INOUT:
    curl_context = socketp ?
      (curl_context_t *) socketp : create_curl_context(s);

    curl_multi_assign(curl_handle, s, (void *) curl_context);

    if(action != CURL_POLL_IN)
      events |= UV_WRITABLE;
    if(action != CURL_POLL_OUT)
      events |= UV_READABLE;

    uv_poll_start(&curl_context->poll_handle, events, curl_perform);
    break;
  case CURL_POLL_REMOVE:
    if(socketp) {
      uv_poll_stop(&((curl_context_t*)socketp)->poll_handle);
      destroy_curl_context((curl_context_t*) socketp);
      curl_multi_assign(curl_handle, s, NULL);
    }
    break;
  default:
    abort();
  }

  return 0;
}

int main(int argc, char **argv)
{
  loop = uv_default_loop();

  if(argc <= 1)
    return 0;

  if(curl_global_init(CURL_GLOBAL_ALL)) {
    fprintf(stderr, "Could not init curl\n");
    return 1;
  }

  uv_timer_init(loop, &timeout);

  curl_handle = curl_multi_init();
  curl_easy_setopt(curl_handle, CURLOPT_HTTP_VERSION, (long)CURL_HTTP_VERSION_3);
  curl_multi_setopt(curl_handle, CURLMOPT_SOCKETFUNCTION, handle_socket);
  curl_multi_setopt(curl_handle, CURLMOPT_TIMERFUNCTION, start_timeout);

  while(argc-- > 1) {
    add_download(argv[argc], argc);
  }

  uv_run(loop, UV_RUN_DEFAULT);
  curl_multi_cleanup(curl_handle);

  return 0;
}

main.cpp

#include <stdio.h>
#include <curl/curl.h>
#include <stdlib.h>

#define QUIC_ROCKS_URL "https://quic.rocks:4433"

void function_pt(void *ptr, size_t size, size_t nmemb, void *stream){
    printf("%d", atoi((char*)ptr));
}

int main(int argc, char** args) {
    CURLcode res;

    curl_version_info_data *ver;

    curl_global_init(CURL_GLOBAL_ALL);

    char buffer[128];

    ver = curl_version_info(CURLVERSION_NOW);

    snprintf(buffer, sizeof(buffer), "Suppported: %s %s %s",
        (ver->features & CURL_VERSION_HTTP2) ? "HTTP/2" : "",
        (ver->features & CURL_VERSION_HTTP3) ? "HTTP/3" : "",
        (ver->features & CURL_VERSION_ALTSVC) ? "ALTSVC" : ""
    );

    curl_global_cleanup();

    printf("%s\n", buffer);

    CURL *curl = curl_easy_init();

    char* url = NULL;

    if(argc < 2) {
        printf("no argument provided, connecting to '%s':\n\n", QUIC_ROCKS_URL);
        url = (char*)QUIC_ROCKS_URL;
    } else {
        url = args[1];
        printf("connectiong to '%s'\n", url);
    }

    if(curl) {
        curl_easy_setopt(curl, CURLOPT_URL, url);

        //curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, function_pt);

        /* Forcing HTTP/3 will make the connection fail if the server isn't
        accessible over QUIC + HTTP/3 on the given host and port.
        Consider using CURLOPT_ALTSVC instead! */
        CURLcode resHttp3 = curl_easy_setopt(curl, CURLOPT_HTTP_VERSION, (long)CURL_HTTP_VERSION_3);

        printf("resHttp3: [%d]", resHttp3);

        /* Perform the request, res will get the return code */
        res = curl_easy_perform(curl);
        /* Check for errors */
        if(res != CURLE_OK)
        {
            fprintf(stderr, "curl_easy_perform() failed: %s\n",
                    curl_easy_strerror(res));
        }

        /* always cleanup */
        curl_easy_cleanup(curl);
    }
    return 0;
}

Res

osboxes@osboxes:~/http-3-test-environment/src/libcurl-client/build$ ./multi-uv https://quic.rocks:4433
resHttp3: [0]
<!doctype html>
<html>
<head><title>quic.rocks</title></head>
<body>
<h1>quic.rocks</h1>
<p>You have successfully loaded quic.rocks using QUIC!</p>
</body>
</html>
https://quic.rocks:4433/ DONE
CODE [No error]
Contentosboxes@osboxes:~/http-3-test-environment/src/libcurl-client/build$ ./http3-client https://quic.rocks:4433
Suppported:  HTTP/3 ALTSVC
connectiong to 'https://quic.rocks:4433'
resHttp3: [0]<!doctype html>
<html>
<head><title>quic.rocks</title></head>
<body>
<h1>quic.rocks</h1>
<p>You have successfully loaded quic.rocks using QUIC!</p>
</body>
</html>