From b83a20b6ee9fe72cd8489bdbca04b963a8a796b4 Mon Sep 17 00:00:00 2001
From: Leigang Zhang <71714656+zll600@users.noreply.github.com>
Date: Wed, 6 Mar 2024 15:12:16 +0800
Subject: [PATCH] feat: support to enable quic (#10989)

---
 .github/workflows/quic.yml | 197 +++++++++++++++++++++++++++++++++++++
 apisix/cli/ngx_tpl.lua     |   5 +
 apisix/cli/ops.lua         |  30 ++++--
 apisix/cli/schema.lua      |   5 +-
 apisix/init.lua            |   5 +
 conf/config-default.yaml   |   2 +
 t/APISIX.pm                |   2 +
 t/cli/test_main.sh         |   7 ++
 t/quic/admin/basic.t       | 108 ++++++++++++++++++++
 9 files changed, 353 insertions(+), 8 deletions(-)
 create mode 100644 .github/workflows/quic.yml
 create mode 100644 t/quic/admin/basic.t

diff --git a/.github/workflows/quic.yml b/.github/workflows/quic.yml
new file mode 100644
index 000000000000..abaf39988471
--- /dev/null
+++ b/.github/workflows/quic.yml
@@ -0,0 +1,197 @@
+name: QUIC
+
+on:
+  push:
+    branches: [master, 'release/**']
+    paths-ignore:
+      - 'docs/**'
+      - '**/*.md'
+  pull_request:
+    branches: [master, 'release/**']
+    paths-ignore:
+      - 'docs/**'
+      - '**/*.md'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/master' && github.run_number || github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+          - ubuntu-20.04
+        os_name:
+          - linux_openresty
+        events_module:
+          - lua-resty-worker-events
+          - lua-resty-events
+        test_dir:
+          - t/quic/admin
+
+    runs-on: ${{ matrix.platform }}
+    timeout-minutes: 90
+    env:
+      SERVER_NAME: ${{ matrix.os_name }}
+      OPENRESTY_VERSION: default
+
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: "1.17"
+
+      - name: Cache deps
+        uses: actions/cache@v4
+        env:
+          cache-name: cache-deps
+        with:
+          path: deps
+          key: ${{ runner.os }}-${{ env.cache-name }}-${{ matrix.os_name }}-${{ hashFiles('apisix-master-0.rockspec') }}
+
+      - name: Extract branch name
+        if: ${{ startsWith(github.ref, 'refs/heads/release/') }}
+        id: branch_env
+        shell: bash
+        run: |
+          echo "version=${GITHUB_REF##*/}" >>$GITHUB_OUTPUT
+          echo "fullname=apache-apisix-${GITHUB_REF##*/}-src.tgz" >>$GITHUB_OUTPUT
+
+      - name: Extract test type
+        shell: bash
+        id: test_env
+        run: |
+          test_dir="${{ matrix.test_dir }}"
+          if [[ $test_dir =~ 't/quic/plugin' ]]; then
+            echo "type=plugin" >>$GITHUB_OUTPUT
+          fi
+          if [[ $test_dir =~ 't/quic/admin' ]]; then
+            echo "type=first" >>$GITHUB_OUTPUT
+          fi
+          if [[ $test_dir =~ ' t/quic/xrpc' ]]; then
+            echo "type=last" >>$GITHUB_OUTPUT
+          fi
+
+      - name: Free disk space
+        run: |
+          bash ./ci/free_disk_space.sh
+
+      - name: Linux launch common services
+        run: |
+          make ci-env-up project_compose_ci=ci/pod/docker-compose.common.yml
+          sudo ./ci/init-common-test-service.sh
+
+      - name: Create tarball
+        if: ${{ startsWith(github.ref, 'refs/heads/release/') }}
+        run: |
+          make compress-tar VERSION=${{ steps.branch_env.outputs.version }}
+
+      - name: Remove source code
+        if: ${{ startsWith(github.ref, 'refs/heads/release/') }}
+        run: |
+          rm -rf $(ls -1 --ignore=*.tgz --ignore=ci --ignore=t --ignore=utils --ignore=.github)
+          tar zxvf ${{ steps.branch_env.outputs.fullname }}
+
+      - name: Cache images
+        id: cache-images
+        uses: actions/cache@v4
+        env:
+          cache-name: cache-apisix-docker-images
+        with:
+          path: docker-images-backup
+          key: ${{ runner.os }}-${{ env.cache-name }}-${{ steps.test_env.outputs.type }}-${{ hashFiles(format('./ci/pod/docker-compose.{0}.yml', steps.test_env.outputs.type )) }}
+
+      - if: ${{ steps.cache-images.outputs.cache-hit == 'true' }}
+        name: Load saved docker images
+        run: |
+          if [[ -f docker-images-backup/apisix-images.tar ]]; then
+            [[ ${{ steps.test_env.outputs.type }} != first ]] && sudo ./ci/init-${{ steps.test_env.outputs.type }}-test-service.sh before
+            docker load --input docker-images-backup/apisix-images.tar
+            echo "loaded docker images"
+
+            # preserve storage space
+            rm docker-images-backup/apisix-images.tar
+
+            make ci-env-up project_compose_ci=ci/pod/docker-compose.${{ steps.test_env.outputs.type }}.yml
+            if [[ ${{ steps.test_env.outputs.type }} != first ]]; then
+              sudo ./ci/init-${{ steps.test_env.outputs.type }}-test-service.sh after
+            fi
+          fi
+      - if: ${{ steps.cache-images.outputs.cache-hit != 'true' }}
+        name: Linux launch services
+        run: |
+          [[ ${{ steps.test_env.outputs.type }} != first ]] && sudo ./ci/init-${{ steps.test_env.outputs.type }}-test-service.sh before
+          [[ ${{ steps.test_env.outputs.type }} == plugin ]] && ./ci/pod/openfunction/build-function-image.sh
+          make ci-env-up project_compose_ci=ci/pod/docker-compose.${{ steps.test_env.outputs.type }}.yml
+          [[ ${{ steps.test_env.outputs.type }} != first ]] && sudo ./ci/init-${{ steps.test_env.outputs.type }}-test-service.sh after
+          echo "Linux launch services, done."
+      - name: Start Dubbo Backend
+        if: matrix.os_name == 'linux_openresty' && (steps.test_env.outputs.type == 'plugin' || steps.test_env.outputs.type == 'last')
+        run: |
+          cur_dir=$(pwd)
+          sudo apt update
+          sudo apt install -y maven
+          cd t/lib/dubbo-backend
+          mvn package
+          cd dubbo-backend-provider/target
+          java -Djava.net.preferIPv4Stack=true -jar dubbo-demo-provider.one-jar.jar > /tmp/java.log &
+          cd $cur_dir/t/lib/dubbo-serialization-backend
+          mvn package
+          cd dubbo-serialization-backend-provider/target
+          java -Djava.net.preferIPv4Stack=true -jar dubbo-demo-provider.one-jar.jar > /tmp/java2.log &
+
+      - name: Build xDS library
+        if: steps.test_env.outputs.type == 'last'
+        run: |
+          cd t/xds-library
+          go build -o libxds.so -buildmode=c-shared main.go export.go
+
+      - name: Build wasm code
+        if: matrix.os_name == 'linux_openresty' && steps.test_env.outputs.type == 'last'
+        run: |
+          export TINYGO_VER=0.20.0
+          wget https://github.com/tinygo-org/tinygo/releases/download/v${TINYGO_VER}/tinygo_${TINYGO_VER}_amd64.deb 2>/dev/null
+          sudo dpkg -i tinygo_${TINYGO_VER}_amd64.deb
+          cd t/wasm && find . -type f -name "*.go" | xargs -Ip tinygo build -o p.wasm -scheduler=none -target=wasi p
+
+      - name: Linux Before install
+        run: sudo ./ci/${{ matrix.os_name }}_runner.sh before_install
+
+      - name: Linux Install
+        run: |
+          sudo --preserve-env=OPENRESTY_VERSION \
+          ./ci/${{ matrix.os_name }}_runner.sh do_install
+
+      - name: Linux Install static-curl
+        shell: bash
+        run: |
+          sudo apt update && sudo apt install xz-utils -y
+          CURL_VERSION="8.6.0"
+          wget -q https://github.com/stunnel/static-curl/releases/download/${CURL_VERSION}/curl-linux-amd64-${CURL_VERSION}.tar.xz
+          tar -xf curl-linux-amd64-${CURL_VERSION}.tar.xz
+          sudo apt remove -y curl
+          sudo cp curl /usr/bin
+          curl -V
+
+      - name: Linux Script
+        env:
+          TEST_FILE_SUB_DIR: ${{ matrix.test_dir }}
+          TEST_EVENTS_MODULE: ${{ matrix.events_module }}
+        run: sudo -E ./ci/${{ matrix.os_name }}_runner.sh script
+
+      - if: ${{ steps.cache-images.outputs.cache-hit != 'true' }}
+        name: Save docker images
+        run: |
+          echo "start backing up, $(date)"
+          bash ./ci/backup-docker-images.sh ${{ steps.test_env.outputs.type }}
+          echo "backup done, $(date)"
diff --git a/apisix/cli/ngx_tpl.lua b/apisix/cli/ngx_tpl.lua
index 121f39f0501f..9642a3605279 100644
--- a/apisix/cli/ngx_tpl.lua
+++ b/apisix/cli/ngx_tpl.lua
@@ -636,9 +636,14 @@ http {
         {% end %}
         {% if ssl.enable then %}
         {% for _, item in ipairs(ssl.listen) do %}
+        {% if item.enable_quic then %}
+        listen {* item.ip *}:{* item.port *} quic default_server {% if enable_reuseport then %} reuseport {% end %};
+        listen {* item.ip *}:{* item.port *} ssl default_server;
+        {% else %}
         listen {* item.ip *}:{* item.port *} ssl default_server {% if enable_reuseport then %} reuseport {% end %};
         {% end %}
         {% end %}
+        {% end %}
         {% if proxy_protocol and proxy_protocol.listen_http_port then %}
         listen {* proxy_protocol.listen_http_port *} default_server proxy_protocol;
         {% end %}
diff --git a/apisix/cli/ops.lua b/apisix/cli/ops.lua
index 918d1c81bdfb..73b9c1d1336e 100644
--- a/apisix/cli/ops.lua
+++ b/apisix/cli/ops.lua
@@ -379,7 +379,8 @@ Please modify "admin_key" in conf/config.yaml .
 
     local ip_port_to_check = {}
 
-    local function listen_table_insert(listen_table, scheme, ip, port, enable_http2, enable_ipv6)
+    local function listen_table_insert(listen_table, scheme, ip, port,
+                                enable_http2, enable_quic, enable_ipv6)
         if type(ip) ~= "string" then
             util.die(scheme, " listen ip format error, must be string", "\n")
         end
@@ -397,7 +398,12 @@ Please modify "admin_key" in conf/config.yaml .
 
         if ip_port_to_check[addr] == nil then
             table_insert(listen_table,
-                    {ip = ip, port = port, enable_http2 = enable_http2})
+                    {
+                        ip = ip,
+                        port = port,
+                        enable_http2 = enable_http2,
+                        enable_quic = enable_quic
+                    })
             ip_port_to_check[addr] = scheme
         end
 
@@ -407,7 +413,12 @@ Please modify "admin_key" in conf/config.yaml .
 
             if ip_port_to_check[addr] == nil then
                 table_insert(listen_table,
-                        {ip = ip, port = port, enable_http2 = enable_http2})
+                        {
+                            ip = ip,
+                            port = port,
+                            enable_http2 = enable_http2,
+                            enable_quic = enable_quic
+                        })
                 ip_port_to_check[addr] = scheme
             end
         end
@@ -418,12 +429,12 @@ Please modify "admin_key" in conf/config.yaml .
     -- listen in http, support multiple ports and specific IP, compatible with the original style
     if type(yaml_conf.apisix.node_listen) == "number" then
         listen_table_insert(node_listen, "http", "0.0.0.0", yaml_conf.apisix.node_listen,
-                false, yaml_conf.apisix.enable_ipv6)
+                false, false, yaml_conf.apisix.enable_ipv6)
     elseif type(yaml_conf.apisix.node_listen) == "table" then
         for _, value in ipairs(yaml_conf.apisix.node_listen) do
             if type(value) == "number" then
                 listen_table_insert(node_listen, "http", "0.0.0.0", value,
-                        false, yaml_conf.apisix.enable_ipv6)
+                        false, false, yaml_conf.apisix.enable_ipv6)
             elseif type(value) == "table" then
                 local ip = value.ip
                 local port = value.port
@@ -449,7 +460,7 @@ Please modify "admin_key" in conf/config.yaml .
                 end
 
                 listen_table_insert(node_listen, "http", ip, port,
-                        enable_http2, enable_ipv6)
+                        enable_http2, false, enable_ipv6)
             end
         end
     end
@@ -462,6 +473,7 @@ Please modify "admin_key" in conf/config.yaml .
         local port = value.port
         local enable_ipv6 = false
         local enable_http2 = value.enable_http2
+        local enable_quic = value.enable_quic
 
         if ip == nil then
             ip = "0.0.0.0"
@@ -481,8 +493,12 @@ Please modify "admin_key" in conf/config.yaml .
             enable_http2_global = true
         end
 
+        if enable_quic == nil then
+            enable_quic = false
+        end
+
         listen_table_insert(ssl_listen, "https", ip, port,
-                enable_http2, enable_ipv6)
+                enable_http2, enable_quic, enable_ipv6)
     end
 
     yaml_conf.apisix.ssl.listen = ssl_listen
diff --git a/apisix/cli/schema.lua b/apisix/cli/schema.lua
index 836b88f6965a..3eae5ed75d31 100644
--- a/apisix/cli/schema.lua
+++ b/apisix/cli/schema.lua
@@ -220,7 +220,10 @@ local config_schema = {
                                     },
                                     enable_http2 = {
                                         type = "boolean",
-                                    }
+                                    },
+                                    enable_quic = {
+                                        type = "boolean",
+                                    },
                                 }
                             }
                         },
diff --git a/apisix/init.lua b/apisix/init.lua
index 91dc83f0b88e..0391ddd62787 100644
--- a/apisix/init.lua
+++ b/apisix/init.lua
@@ -576,6 +576,11 @@ end
 
 
 function _M.http_access_phase()
+    -- from HTTP/3 to HTTP/1.1 we need to convert :authority pesudo-header
+    -- to Host header, so we set upstream_host variable here.
+    if ngx.req.http_version() == 3 then
+        ngx.var.upstream_host = ngx.var.host .. ":" .. ngx.var.server_port
+    end
     local ngx_ctx = ngx.ctx
 
     -- always fetch table from the table pool, we don't need a reused api_ctx
diff --git a/conf/config-default.yaml b/conf/config-default.yaml
index 245e68f2b829..7b409a7ba8a0 100755
--- a/conf/config-default.yaml
+++ b/conf/config-default.yaml
@@ -97,9 +97,11 @@ apisix:
     listen:                                       # APISIX listening port for HTTPS traffic.
       - port: 9443
         enable_http2: true
+        enable_quic: false                        # Enable QUIC or HTTP/3. If not set default to `false`.
       # - ip: 127.0.0.3                           # If not set, default to `0.0.0.0`.
       #   port: 9445
       #   enable_http2: true
+      #   enable_quic: true
     # ssl_trusted_certificate: /path/to/ca-cert   # Set the path to CA certificates used to verify client
                                                   # certificates in the PEM format.
     ssl_protocols: TLSv1.2 TLSv1.3                # TLS versions supported.
diff --git a/t/APISIX.pm b/t/APISIX.pm
index be640c2bc98a..5e61dcf76dec 100644
--- a/t/APISIX.pm
+++ b/t/APISIX.pm
@@ -671,6 +671,7 @@ _EOC_
     $a6_ngx_directives
 
     server {
+        listen 1983 quic reuseport;
         listen 1983 ssl;
         ssl_certificate             cert/apisix.crt;
         ssl_certificate_key         cert/apisix.key;
@@ -726,6 +727,7 @@ _EOC_
     $config .= <<_EOC_;
         $ipv6_listen_conf
 
+        listen 1994 quic reuseport;
         listen 1994 ssl;
         http2 on;
         ssl_certificate             cert/apisix.crt;
diff --git a/t/cli/test_main.sh b/t/cli/test_main.sh
index 1835ef5bbe27..7248b2c7f527 100755
--- a/t/cli/test_main.sh
+++ b/t/cli/test_main.sh
@@ -142,6 +142,7 @@ apisix:
       - ip: 127.0.0.4
         port: 9445
         enable_http2: true
+        enable_quic: true
 " > conf/config.yaml
 
 make init
@@ -170,6 +171,12 @@ if [ $count_https_specific_ip_and_enable_http2 -ne 1 ]; then
     exit 1
 fi
 
+count_https_specific_ip_and_enable_quic=`grep -c "listen 127.0.0..:944. quic" conf/nginx.conf || true`
+if [ $count_https_specific_ip_and_enable_quic -ne 1 ]; then
+    echo "failed: failed to support specific IP and enable quic listen in https"
+    exit 1
+fi
+
 echo "passed: support specific IP listen in http and https"
 
 # check default env
diff --git a/t/quic/admin/basic.t b/t/quic/admin/basic.t
new file mode 100644
index 000000000000..b33e60c80408
--- /dev/null
+++ b/t/quic/admin/basic.t
@@ -0,0 +1,108 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+use t::APISIX 'no_plan';
+
+repeat_each(1);
+log_level('info');
+no_root_location();
+no_shuffle();
+run_tests();
+
+__DATA__
+
+=== TEST 1:  create ssl for test.com
+--- config
+    location /t {
+        content_by_lua_block {
+            local core = require("apisix.core")
+            local t = require("lib.test_admin")
+
+            local ssl_cert = t.read_file("t/certs/apisix.crt")
+            local ssl_key =  t.read_file("t/certs/apisix.key")
+            local data = {cert = ssl_cert, key = ssl_key, sni = "test.com"}
+
+            local code, body = t.test('/apisix/admin/ssls/1',
+                ngx.HTTP_PUT,
+                core.json.encode(data),
+                [[{
+                    "value": {
+                        "sni": "test.com"
+                    },
+                    "key": "/apisix/ssls/1"
+                }]]
+                )
+
+            ngx.status = code
+            ngx.say(body)
+        }
+    }
+--- request
+GET /t
+--- response_body
+passed
+
+
+
+=== TEST 2: Successfully access test.com with QUIC
+--- config
+    location /echo {
+        echo world;
+    }
+--- exec
+curl -k -v -H "Host: test.com" -H "content-length: 0" --http3-only --resolve "test.com:1994:127.0.0.1" https://test.com:1994/echo 2>&1 | cat
+--- response_body eval
+qr/world/
+
+
+
+=== TEST 3: set route
+--- config
+location /t {
+    content_by_lua_block {
+        local t = require("lib.test_admin").test
+        local code, body = t('/apisix/admin/routes/1',
+            ngx.HTTP_PUT,
+            [[{
+                "upstream": {
+                    "nodes": {
+                        "127.0.0.1:1980": 1
+                    },
+                    "type": "roundrobin"
+                },
+                "uri": "/hello"
+            }]]
+        )
+        if code >= 300 then
+            ngx.status = code
+            ngx.say(message)
+            return
+        end
+        ngx.say(body)
+    }
+}
+--- request
+GET /t
+--- response_body
+passed
+
+
+
+=== TEST 4: Successfully access route with QUIC
+--- exec
+curl -k -v -H "Host: test.com:1994" -H "content-length: 0" --http3-only --resolve "test.com:1994:127.0.0.1" https://test.com:1994/hello 2>&1 | cat
+--- response_body_like
+hello world