feat: support to enable quic and add basic test case

.github/workflows/quic.yml

@@ -0,0 +1,197 @@
+name: QUIC
+
+on:
+  push:
+    branches: [master, 'release/**']
+    paths-ignore:
+      - 'docs/**'
+      - '**/*.md'
+  pull_request:
+    branches: [master, 'release/**']
+    paths-ignore:
+      - 'docs/**'
+      - '**/*.md'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/master' && github.run_number || github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+          - ubuntu-20.04
+        os_name:
+          - linux_openresty
+        events_module:
+          - lua-resty-worker-events
+          - lua-resty-events
+        test_dir:
+          - t/quic/admin
+
+    runs-on: ${{ matrix.platform }}
+    timeout-minutes: 90
+    env:
+      SERVER_NAME: ${{ matrix.os_name }}
+      OPENRESTY_VERSION: default
+
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: "1.17"
+
+      - name: Cache deps
+        uses: actions/cache@v4
+        env:
+          cache-name: cache-deps
+        with:
+          path: deps
+          key: ${{ runner.os }}-${{ env.cache-name }}-${{ matrix.os_name }}-${{ hashFiles('apisix-master-0.rockspec') }}
+
+      - name: Extract branch name
+        if: ${{ startsWith(github.ref, 'refs/heads/release/') }}
+        id: branch_env
+        shell: bash
+        run: |
+          echo "version=${GITHUB_REF##*/}" >>$GITHUB_OUTPUT
+          echo "fullname=apache-apisix-${GITHUB_REF##*/}-src.tgz" >>$GITHUB_OUTPUT
+
+      - name: Extract test type
+        shell: bash
+        id: test_env
+        run: |
+          test_dir="${{ matrix.test_dir }}"
+          if [[ $test_dir =~ 't/quic/plugin' ]]; then
+            echo "type=plugin" >>$GITHUB_OUTPUT
+          fi
+          if [[ $test_dir =~ 't/quic/admin' ]]; then
+            echo "type=first" >>$GITHUB_OUTPUT
+          fi
+          if [[ $test_dir =~ ' t/quic/xrpc' ]]; then
+            echo "type=last" >>$GITHUB_OUTPUT
+          fi
+
+      - name: Free disk space
+        run: |
+          bash ./ci/free_disk_space.sh
+
+      - name: Linux launch common services
+        run: |
+          make ci-env-up project_compose_ci=ci/pod/docker-compose.common.yml
+          sudo ./ci/init-common-test-service.sh
+
+      - name: Create tarball
+        if: ${{ startsWith(github.ref, 'refs/heads/release/') }}
+        run: |
+          make compress-tar VERSION=${{ steps.branch_env.outputs.version }}
+
+      - name: Remove source code
+        if: ${{ startsWith(github.ref, 'refs/heads/release/') }}
+        run: |
+          rm -rf $(ls -1 --ignore=*.tgz --ignore=ci --ignore=t --ignore=utils --ignore=.github)
+          tar zxvf ${{ steps.branch_env.outputs.fullname }}
+
+      - name: Cache images
+        id: cache-images
+        uses: actions/cache@v4
+        env:
+          cache-name: cache-apisix-docker-images
+        with:
+          path: docker-images-backup
+          key: ${{ runner.os }}-${{ env.cache-name }}-${{ steps.test_env.outputs.type }}-${{ hashFiles(format('./ci/pod/docker-compose.{0}.yml', steps.test_env.outputs.type )) }}
+
+      - if: ${{ steps.cache-images.outputs.cache-hit == 'true' }}
+        name: Load saved docker images
+        run: |
+          if [[ -f docker-images-backup/apisix-images.tar ]]; then
+            [[ ${{ steps.test_env.outputs.type }} != first ]] && sudo ./ci/init-${{ steps.test_env.outputs.type }}-test-service.sh before
+            docker load --input docker-images-backup/apisix-images.tar
+            echo "loaded docker images"
+
+            # preserve storage space
+            rm docker-images-backup/apisix-images.tar
+
+            make ci-env-up project_compose_ci=ci/pod/docker-compose.${{ steps.test_env.outputs.type }}.yml
+            if [[ ${{ steps.test_env.outputs.type }} != first ]]; then
+              sudo ./ci/init-${{ steps.test_env.outputs.type }}-test-service.sh after
+            fi
+          fi
+      - if: ${{ steps.cache-images.outputs.cache-hit != 'true' }}
+        name: Linux launch services
+        run: |
+          [[ ${{ steps.test_env.outputs.type }} != first ]] && sudo ./ci/init-${{ steps.test_env.outputs.type }}-test-service.sh before
+          [[ ${{ steps.test_env.outputs.type }} == plugin ]] && ./ci/pod/openfunction/build-function-image.sh
+          make ci-env-up project_compose_ci=ci/pod/docker-compose.${{ steps.test_env.outputs.type }}.yml
+          [[ ${{ steps.test_env.outputs.type }} != first ]] && sudo ./ci/init-${{ steps.test_env.outputs.type }}-test-service.sh after
+          echo "Linux launch services, done."
+      - name: Start Dubbo Backend
+        if: matrix.os_name == 'linux_openresty' && (steps.test_env.outputs.type == 'plugin' || steps.test_env.outputs.type == 'last')
+        run: |
+          cur_dir=$(pwd)
+          sudo apt update
+          sudo apt install -y maven
+          cd t/lib/dubbo-backend
+          mvn package
+          cd dubbo-backend-provider/target
+          java -Djava.net.preferIPv4Stack=true -jar dubbo-demo-provider.one-jar.jar > /tmp/java.log &
+          cd $cur_dir/t/lib/dubbo-serialization-backend
+          mvn package
+          cd dubbo-serialization-backend-provider/target
+          java -Djava.net.preferIPv4Stack=true -jar dubbo-demo-provider.one-jar.jar > /tmp/java2.log &
+
+      - name: Build xDS library
+        if: steps.test_env.outputs.type == 'last'
+        run: |
+          cd t/xds-library
+          go build -o libxds.so -buildmode=c-shared main.go export.go
+
+      - name: Build wasm code
+        if: matrix.os_name == 'linux_openresty' && steps.test_env.outputs.type == 'last'
+        run: |
+          export TINYGO_VER=0.20.0
+          wget https://github.com/tinygo-org/tinygo/releases/download/v${TINYGO_VER}/tinygo_${TINYGO_VER}_amd64.deb 2>/dev/null
+          sudo dpkg -i tinygo_${TINYGO_VER}_amd64.deb
+          cd t/wasm && find . -type f -name "*.go" | xargs -Ip tinygo build -o p.wasm -scheduler=none -target=wasi p
+
+      - name: Linux Before install
+        run: sudo ./ci/${{ matrix.os_name }}_runner.sh before_install
+
+      - name: Linux Install
+        run: |
+          sudo --preserve-env=OPENRESTY_VERSION \
+          ./ci/${{ matrix.os_name }}_runner.sh do_install
+
+      - name: Linux Install static-curl
+        shell: bash
+        run: |
+          sudo apt update && sudo apt install xz-utils -y
+          CURL_VERSION="8.6.0"
+          wget -q https://github.com/stunnel/static-curl/releases/download/${CURL_VERSION}/curl-linux-amd64-${CURL_VERSION}.tar.xz
+          tar -xf curl-linux-amd64-${CURL_VERSION}.tar.xz
+          sudo apt remove -y curl
+          sudo cp curl /usr/bin
+          curl -V
+
+      - name: Linux Script
+        env:
+          TEST_FILE_SUB_DIR: ${{ matrix.test_dir }}
+          TEST_EVENTS_MODULE: ${{ matrix.events_module }}
+        run: sudo -E ./ci/${{ matrix.os_name }}_runner.sh script
+
+      - if: ${{ steps.cache-images.outputs.cache-hit != 'true' }}
+        name: Save docker images
+        run: |
+          echo "start backing up, $(date)"
+          bash ./ci/backup-docker-images.sh ${{ steps.test_env.outputs.type }}
+          echo "backup done, $(date)"

apisix/cli/ngx_tpl.lua

@@ -636,9 +636,14 @@ http {
         {% end %}
         {% if ssl.enable then %}
         {% for _, item in ipairs(ssl.listen) do %}
+        {% if item.enable_quic then %}
+        listen {* item.ip *}:{* item.port *} quic default_server {% if enable_reuseport then %} reuseport {% end %};
+        listen {* item.ip *}:{* item.port *} ssl default_server;
+        {% else %}
         listen {* item.ip *}:{* item.port *} ssl default_server {% if enable_reuseport then %} reuseport {% end %};
         {% end %}
         {% end %}
+        {% end %}
         {% if proxy_protocol and proxy_protocol.listen_http_port then %}
         listen {* proxy_protocol.listen_http_port *} default_server proxy_protocol;
         {% end %}

apisix/cli/ops.lua

@@ -379,7 +379,8 @@ Please modify "admin_key" in conf/config.yaml .
 
     local ip_port_to_check = {}
 
-    local function listen_table_insert(listen_table, scheme, ip, port, enable_http2, enable_ipv6)
+    local function listen_table_insert(listen_table, scheme, ip, port,
+                                enable_http2, enable_quic, enable_ipv6)
         if type(ip) ~= "string" then
             util.die(scheme, " listen ip format error, must be string", "\n")
         end
@@ -397,7 +398,12 @@ Please modify "admin_key" in conf/config.yaml .
 
         if ip_port_to_check[addr] == nil then
             table_insert(listen_table,
-                    {ip = ip, port = port, enable_http2 = enable_http2})
+                    {
+                        ip = ip,
+                        port = port,
+                        enable_http2 = enable_http2,
+                        enable_quic = enable_quic
+                    })
             ip_port_to_check[addr] = scheme
         end
 
@@ -407,7 +413,12 @@ Please modify "admin_key" in conf/config.yaml .
 
             if ip_port_to_check[addr] == nil then
                 table_insert(listen_table,
-                        {ip = ip, port = port, enable_http2 = enable_http2})
+                        {
+                            ip = ip,
+                            port = port,
+                            enable_http2 = enable_http2,
+                            enable_quic = enable_quic
+                        })
                 ip_port_to_check[addr] = scheme
             end
         end
@@ -418,12 +429,12 @@ Please modify "admin_key" in conf/config.yaml .
     -- listen in http, support multiple ports and specific IP, compatible with the original style
     if type(yaml_conf.apisix.node_listen) == "number" then
         listen_table_insert(node_listen, "http", "0.0.0.0", yaml_conf.apisix.node_listen,
-                false, yaml_conf.apisix.enable_ipv6)
+                false, false, yaml_conf.apisix.enable_ipv6)
     elseif type(yaml_conf.apisix.node_listen) == "table" then
         for _, value in ipairs(yaml_conf.apisix.node_listen) do
             if type(value) == "number" then
                 listen_table_insert(node_listen, "http", "0.0.0.0", value,
-                        false, yaml_conf.apisix.enable_ipv6)
+                        false, false, yaml_conf.apisix.enable_ipv6)
             elseif type(value) == "table" then
                 local ip = value.ip
                 local port = value.port
@@ -449,7 +460,7 @@ Please modify "admin_key" in conf/config.yaml .
                 end
 
                 listen_table_insert(node_listen, "http", ip, port,
-                        enable_http2, enable_ipv6)
+                        enable_http2, false, enable_ipv6)
             end
         end
     end
@@ -462,6 +473,7 @@ Please modify "admin_key" in conf/config.yaml .
         local port = value.port
         local enable_ipv6 = false
         local enable_http2 = value.enable_http2
+        local enable_quic = value.enable_quic
 
         if ip == nil then
             ip = "0.0.0.0"
@@ -481,8 +493,12 @@ Please modify "admin_key" in conf/config.yaml .
             enable_http2_global = true
         end
 
+        if enable_quic == nil then
+            enable_quic = false
+        end
+
         listen_table_insert(ssl_listen, "https", ip, port,
-                enable_http2, enable_ipv6)
+                enable_http2, enable_quic, enable_ipv6)
     end
 
     yaml_conf.apisix.ssl.listen = ssl_listen

apisix/cli/schema.lua

@@ -220,7 +220,10 @@ local config_schema = {
                                     },
                                     enable_http2 = {
                                         type = "boolean",
-                                    }
+                                    },
+                                    enable_quic = {
+                                        type = "boolean",
+                                    },
                                 }
                             }
                         },

apisix/init.lua

@@ -576,6 +576,11 @@ end
 
 
 function _M.http_access_phase()
+    -- from HTTP/3 to HTTP/1.1 we need to convert :authority pesudo-header
+    -- to Host header, so we set upstream_host variable here.
+    if ngx.req.http_version() == 3 then
+        ngx.var.upstream_host = ngx.var.host .. ":" .. ngx.var.server_port
+    end
     local ngx_ctx = ngx.ctx
 
     -- always fetch table from the table pool, we don't need a reused api_ctx

ci/common.sh

@@ -177,7 +177,7 @@ GRPC_SERVER_EXAMPLE_VER=20210819
 
 linux_get_dependencies () {
     apt update
-    apt install -y cpanminus build-essential libncurses5-dev libreadline-dev libssl-dev perl libpcre3 libpcre3-dev libldap2-dev
+    apt install -y cpanminus build-essential libncurses5-dev libreadline-dev libssl-dev perl libpcre3 libpcre3-dev libldap2-dev xz-utils
 }
 
 function start_grpc_server_example() {

conf/config-default.yaml

@@ -97,9 +97,11 @@ apisix:
     listen:                                       # APISIX listening port for HTTPS traffic.
       - port: 9443
         enable_http2: true
+        enable_quic: false
       # - ip: 127.0.0.3                           # If not set, default to `0.0.0.0`.
       #   port: 9445
       #   enable_http2: true
+      #   enable_quic: true
     # ssl_trusted_certificate: /path/to/ca-cert   # Set the path to CA certificates used to verify client
                                                   # certificates in the PEM format.
     ssl_protocols: TLSv1.2 TLSv1.3                # TLS versions supported.

t/APISIX.pm

@@ -671,6 +671,7 @@ _EOC_
     $a6_ngx_directives
 
     server {
+        listen 1983 quic reuseport;
         listen 1983 ssl;
         ssl_certificate             cert/apisix.crt;
         ssl_certificate_key         cert/apisix.key;
@@ -726,6 +727,7 @@ _EOC_
     $config .= <<_EOC_;
         $ipv6_listen_conf
 
+        listen 1994 quic reuseport;
         listen 1994 ssl;
         http2 on;
         ssl_certificate             cert/apisix.crt;

t/cli/test_main.sh

@@ -142,6 +142,7 @@ apisix:
       - ip: 127.0.0.4
         port: 9445
         enable_http2: true
+        enable_quic: true
 " > conf/config.yaml
 
 make init
@@ -170,6 +171,12 @@ if [ $count_https_specific_ip_and_enable_http2 -ne 1 ]; then
     exit 1
 fi
 
+count_https_specific_ip_and_enable_quic=`grep -c "listen 127.0.0..:944. quic" conf/nginx.conf || true`
+if [ $count_https_specific_ip_and_enable_quic -ne 1 ]; then
+    echo "failed: failed to support specific IP and enable quic listen in https"
+    exit 1
+fi
+
 echo "passed: support specific IP listen in http and https"
 
 # check default env