feat: add GREASEEncryptedClientHelloExtension (#266)

* dicttls: update ECH-related entries

* wip: GREASE ECH extension

* new: GREASE ECH extension

* fix: GREASE ECH Read must succeed with io.EOF

* new: GREASE ECH multiple payload len

* new: parse ECH in EncryptedExtensions

* fix: ECHConfig Length always 0

* new: GREASE ECH parrots

* new: (*Config).ECHConfigs

Add (*Config).ECHConfigs for future full ECH extension.

* new: add GREASE ECH example

Add an incomplete example of using GREASE ECH extension (Chrome 120 parrot).

* fix: invalid httpGetOverConn call

fix a problem in old example where httpGetOverConn was called with uTlsConn.HandshakeState.ServerHello.AlpnProtocol, which will not be populated in case TLS 1.3 is used.

* new: possible InnerClientHello length

alert.go

@@ -58,6 +58,7 @@ const (
 	alertUnknownPSKIdentity           alert = 115
 	alertCertificateRequired          alert = 116
 	alertNoApplicationProtocol        alert = 120
+	alertECHRequired                  alert = 121
 )
 
 var alertText = map[alert]string{
@@ -94,6 +95,7 @@ var alertText = map[alert]string{
 	alertUnknownPSKIdentity:           "unknown PSK identity",
 	alertCertificateRequired:          "certificate required",
 	alertNoApplicationProtocol:        "no application protocol",
+	alertECHRequired:                  "ECH required",
 }
 
 func (e alert) String() string {

common.go

@@ -306,6 +306,11 @@ type ConnectionState struct {
 
 	// ekm is a closure exposed via ExportKeyingMaterial.
 	ekm func(label string, context []byte, length int) ([]byte, error)
+
+	// ECHRetryConfigs contains the ECH retry configurations sent by the server in
+	// EncryptedExtensions message. It is only populated if the server sent the
+	// ech extension in EncryptedExtensions message.
+	ECHRetryConfigs []ECHConfig // [uTLS]
 }
 
 // ExportKeyingMaterial returns length bytes of exported key material in a new
@@ -836,6 +841,17 @@ type Config struct {
 	// autoSessionTicketKeys is like sessionTicketKeys but is owned by the
 	// auto-rotation logic. See Config.ticketKeys.
 	autoSessionTicketKeys []ticketKey
+
+	// ECHConfigs contains the ECH configurations to be used by the ECH
+	// extension if any.
+	// It could either be distributed by the server in EncryptedExtensions
+	// message or out-of-band.
+	//
+	// If ECHConfigs is nil and an ECH extension is present, GREASEd ECH
+	// extension will be sent.
+	//
+	// If GREASE ECH extension is present, this field will be ignored.
+	ECHConfigs []ECHConfig // [uTLS]
 }
 
 const (
@@ -921,6 +937,7 @@ func (c *Config) Clone() *Config {
 		autoSessionTicketKeys:       c.autoSessionTicketKeys,
 
 		PreferSkipResumptionOnNilExtension: c.PreferSkipResumptionOnNilExtension, // [UTLS]
+		ECHConfigs:                         c.ECHConfigs,                         // [uTLS]
 	}
 }
 

dicttls/hpke_aead_identifiers.go

@@ -0,0 +1,19 @@
+package dicttls
+
+// source: https://www.iana.org/assignments/hpke/hpke.xhtml
+// last updated: December 2023
+
+const (
+	AEAD_AES_128_GCM       uint16 = 0x0001 // NIST Special Publication 800-38D
+	AEAD_AES_256_GCM       uint16 = 0x0002 // NIST Special Publication 800-38D
+	AEAD_CHACHA20_POLY1305 uint16 = 0x0003 // RFC 8439
+	AEAD_EXPORT_ONLY       uint16 = 0xFFFF // RFC 9180
+)
+
+var DictAEADIdentifierValueIndexed = map[uint16]string{
+	0x0000: "Reserved", // RFC 9180
+	0x0001: "AES-128-GCM",
+	0x0002: "AES-256-GCM",
+	0x0003: "ChaCha20Poly1305",
+	0xFFFF: "Export-only", // RFC 9180
+}

dicttls/kdf_identifiers.go → dicttls/hpke_kdf_identifiers.go

@@ -1,19 +1,24 @@
 package dicttls
 
-// source: https://www.iana.org/assignments/tls-parameters/tls-kdf-ids.csv
-// last updated: March 2023
+// source: https://www.iana.org/assignments/hpke/hpke.xhtml
+// last updated: December 2023
 
 const (
 	HKDF_SHA256 uint16 = 0x0001
 	HKDF_SHA384 uint16 = 0x0002
+	HKDF_SHA512 uint16 = 0x0003
 )
 
 var DictKDFIdentifierValueIndexed = map[uint16]string{
+	0x0000: "Reserved", // RFC 9180
 	0x0001: "HKDF_SHA256",
 	0x0002: "HKDF_SHA384",
+	0x0003: "HKDF_SHA512",
 }
 
 var DictKDFIdentifierNameIndexed = map[string]uint16{
+	"Reserved":    0x0000, // RFC 9180
 	"HKDF_SHA256": 0x0001,
 	"HKDF_SHA384": 0x0002,
+	"HKDF_SHA512": 0x0003,
 }

dicttls/hpke_kem_identifiers.go

@@ -0,0 +1,53 @@
+package dicttls
+
+// source: https://www.iana.org/assignments/hpke/hpke.xhtml
+// last updated: December 2023
+
+const (
+	DHKEM_P256_HKDF_SHA256      uint16 = 0x0010 // RFC 5869
+	DHKEM_P384_HKDF_SHA384      uint16 = 0x0011 // RFC 5869
+	DHKEM_P521_HKDF_SHA512      uint16 = 0x0012 // RFC 5869
+	DHKEM_CP256_HKDF_SHA256     uint16 = 0x0013 // RFC 6090
+	DHKEM_CP384_HKDF_SHA384     uint16 = 0x0014 // RFC 6090
+	DHKEM_CP521_HKDF_SHA512     uint16 = 0x0015 // RFC 6090
+	DHKEM_SECP256K1_HKDF_SHA256 uint16 = 0x0016 // draft-wahby-cfrg-hpke-kem-secp256k1-01
+
+	DHKEM_X25519_HKDF_SHA256 uint16 = 0x0020 // RFC 7748
+	DHKEM_X448_HKDF_SHA512   uint16 = 0x0021 // RFC 7748
+
+	X25519_KYBER768_DRAFT00 uint16 = 0x0030 // draft-westerbaan-cfrg-hpke-xyber768d00-02
+)
+
+var DictKEMIdentifierValueIndexed = map[uint16]string{
+	0x0000: "Reserved", // RFC 9180
+
+	0x0010: "DHKEM(P-256, HKDF-SHA256)",
+	0x0011: "DHKEM(P-384, HKDF-SHA384)",
+	0x0012: "DHKEM(P-521, HKDF-SHA512)",
+	0x0013: "DHKEM(CP-256, HKDF-SHA256)",
+	0x0014: "DHKEM(CP-384, HKDF-SHA384)",
+	0x0015: "DHKEM(CP-521, HKDF-SHA512)",
+	0x0016: "DHKEM(secp256k1, HKDF-SHA256)",
+
+	0x0020: "DHKEM(X25519, HKDF-SHA256)",
+	0x0021: "DHKEM(X448, HKDF-SHA512)",
+
+	0x0030: "X25519Kyber768Draft00",
+}
+
+var DictKEMIdentifierNameIndexed = map[string]uint16{
+	"Reserved": 0x0000, // RFC 9180
+
+	"DHKEM(P-256, HKDF-SHA256)":     0x0010,
+	"DHKEM(P-384, HKDF-SHA384)":     0x0011,
+	"DHKEM(P-521, HKDF-SHA512)":     0x0012,
+	"DHKEM(CP-256, HKDF-SHA256)":    0x0013,
+	"DHKEM(CP-384, HKDF-SHA384)":    0x0014,
+	"DHKEM(CP-521, HKDF-SHA512)":    0x0015,
+	"DHKEM(secp256k1, HKDF-SHA256)": 0x0016,
+
+	"DHKEM(X25519, HKDF-SHA256)": 0x0020,
+	"DHKEM(X448, HKDF-SHA512)":   0x0021,
+
+	"X25519Kyber768Draft00": 0x0030,
+}

examples/ech/main.go

@@ -0,0 +1,127 @@
+package main
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"time"
+
+	tls "github.com/refraction-networking/utls"
+	"golang.org/x/net/http2"
+)
+
+var (
+	dialTimeout = time.Duration(15) * time.Second
+)
+
+// var requestHostname = "crypto.cloudflare.com" // speaks http2 and TLS 1.3 and ECH and PQ
+// var requestAddr = "crypto.cloudflare.com:443"
+// var requestPath = "/cdn-cgi/trace"
+
+// var requestHostname = "tls-ech.dev" // speaks http2 and TLS 1.3 and ECH and PQ
+// var requestAddr = "tls-ech.dev:443"
+// var requestPath = "/"
+
+var requestHostname = "defo.ie" // speaks http2 and TLS 1.3 and ECH and PQ
+var requestAddr = "defo.ie:443"
+var requestPath = "/ech-check.php"
+
+// var requestHostname = "client.tlsfingerprint.io" // speaks http2 and TLS 1.3 and ECH and PQ
+// var requestAddr = "client.tlsfingerprint.io:443"
+// var requestPath = "/"
+
+func HttpGetCustom(hostname string, addr string) (*http.Response, error) {
+	klw, err := os.OpenFile("./sslkeylogging.log", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
+	if err != nil {
+		return nil, fmt.Errorf("os.OpenFile error: %+v", err)
+	}
+	config := tls.Config{
+		ServerName:   hostname,
+		KeyLogWriter: klw,
+	}
+	dialConn, err := net.DialTimeout("tcp", addr, dialTimeout)
+	if err != nil {
+		return nil, fmt.Errorf("net.DialTimeout error: %+v", err)
+	}
+	uTlsConn := tls.UClient(dialConn, &config, tls.HelloCustom)
+	defer uTlsConn.Close()
+
+	// do not use this particular spec in production
+	// make sure to generate a separate copy of ClientHelloSpec for every connection
+	spec, err := tls.UTLSIdToSpec(tls.HelloChrome_120)
+	// spec, err := tls.UTLSIdToSpec(tls.HelloFirefox_120)
+	if err != nil {
+		return nil, fmt.Errorf("tls.UTLSIdToSpec error: %+v", err)
+	}
+
+	err = uTlsConn.ApplyPreset(&spec)
+	if err != nil {
+		return nil, fmt.Errorf("uTlsConn.Handshake() error: %+v", err)
+	}
+
+	err = uTlsConn.Handshake()
+	if err != nil {
+		return nil, fmt.Errorf("uTlsConn.Handshake() error: %+v", err)
+	}
+
+	return httpGetOverConn(uTlsConn, uTlsConn.ConnectionState().NegotiatedProtocol)
+}
+
+func httpGetOverConn(conn net.Conn, alpn string) (*http.Response, error) {
+	req := &http.Request{
+		Method: "GET",
+		URL:    &url.URL{Scheme: "https", Host: requestHostname, Path: requestPath},
+		Header: make(http.Header),
+		Host:   requestHostname,
+	}
+
+	switch alpn {
+	case "h2":
+		log.Println("HTTP/2 enabled")
+		req.Proto = "HTTP/2.0"
+		req.ProtoMajor = 2
+		req.ProtoMinor = 0
+
+		tr := http2.Transport{}
+		cConn, err := tr.NewClientConn(conn)
+		if err != nil {
+			return nil, err
+		}
+		return cConn.RoundTrip(req)
+	case "http/1.1", "":
+		log.Println("Using HTTP/1.1")
+		req.Proto = "HTTP/1.1"
+		req.ProtoMajor = 1
+		req.ProtoMinor = 1
+
+		err := req.Write(conn)
+		if err != nil {
+			return nil, err
+		}
+		return http.ReadResponse(bufio.NewReader(conn), req)
+	default:
+		return nil, fmt.Errorf("unsupported ALPN: %v", alpn)
+	}
+}
+
+func main() {
+	resp, err := HttpGetCustom(requestHostname, requestAddr)
+	if err != nil {
+		panic(err)
+	}
+	fmt.Printf("Response: %+v\n", resp)
+	// read from resp.Body
+	body := make([]byte, 65535)
+	n, err := resp.Body.Read(body)
+	if err != nil && !errors.Is(err, io.EOF) {
+		panic(err)
+	}
+
+	fmt.Printf("Body: %s\n", body[:n])
+}

examples/old/examples.go

@@ -49,7 +49,7 @@ func HttpGetByHelloID(hostname string, addr string, helloID tls.ClientHelloID) (
 		return nil, fmt.Errorf("uTlsConn.Handshake() error: %+v", err)
 	}
 
-	return httpGetOverConn(uTlsConn, uTlsConn.HandshakeState.ServerHello.AlpnProtocol)
+	return httpGetOverConn(uTlsConn, uTlsConn.ConnectionState().NegotiatedProtocol)
 }
 
 // this example generates a randomized fingeprint, then re-uses it in a follow-up connection
@@ -80,7 +80,7 @@ func HttpGetConsistentRandomized(hostname string, addr string) (*http.Response,
 		return nil, fmt.Errorf("uTlsConn.Handshake() error: %+v", err)
 	}
 
-	return httpGetOverConn(uTlsConn2, uTlsConn2.HandshakeState.ServerHello.AlpnProtocol)
+	return httpGetOverConn(uTlsConn2, uTlsConn2.ConnectionState().NegotiatedProtocol)
 }
 
 func HttpGetExplicitRandom(hostname string, addr string) (*http.Response, error) {
@@ -112,7 +112,7 @@ func HttpGetExplicitRandom(hostname string, addr string) (*http.Response, error)
 	fmt.Printf("#> ClientHello Random:\n%s", hex.Dump(uTlsConn.HandshakeState.Hello.Random))
 	fmt.Printf("#> ServerHello Random:\n%s", hex.Dump(uTlsConn.HandshakeState.ServerHello.Random))
 
-	return httpGetOverConn(uTlsConn, uTlsConn.HandshakeState.ServerHello.AlpnProtocol)
+	return httpGetOverConn(uTlsConn, uTlsConn.ConnectionState().NegotiatedProtocol)
 }
 
 // Note that the server will reject the fake ticket(unless you set up your server to accept them) and do full handshake
@@ -152,7 +152,7 @@ func HttpGetTicket(hostname string, addr string) (*http.Response, error) {
 	fmt.Println("#> This is how client hello with session ticket looked:")
 	fmt.Print(hex.Dump(uTlsConn.HandshakeState.Hello.Raw))
 
-	return httpGetOverConn(uTlsConn, uTlsConn.HandshakeState.ServerHello.AlpnProtocol)
+	return httpGetOverConn(uTlsConn, uTlsConn.ConnectionState().NegotiatedProtocol)
 }
 
 // Note that the server will reject the fake ticket(unless you set up your server to accept them) and do full handshake
@@ -183,7 +183,7 @@ func HttpGetTicketHelloID(hostname string, addr string, helloID tls.ClientHelloI
 	fmt.Println("#> This is how client hello with session ticket looked:")
 	fmt.Print(hex.Dump(uTlsConn.HandshakeState.Hello.Raw))
 
-	return httpGetOverConn(uTlsConn, uTlsConn.HandshakeState.ServerHello.AlpnProtocol)
+	return httpGetOverConn(uTlsConn, uTlsConn.ConnectionState().NegotiatedProtocol)
 }
 
 func HttpGetCustom(hostname string, addr string) (*http.Response, error) {
@@ -253,7 +253,7 @@ func HttpGetCustom(hostname string, addr string) (*http.Response, error) {
 		return nil, fmt.Errorf("uTlsConn.Handshake() error: %+v", err)
 	}
 
-	return httpGetOverConn(uTlsConn, uTlsConn.HandshakeState.ServerHello.AlpnProtocol)
+	return httpGetOverConn(uTlsConn, uTlsConn.ConnectionState().NegotiatedProtocol)
 }
 
 var roller *tls.Roller
@@ -277,7 +277,7 @@ func HttpGetGoogleWithRoller() (*http.Response, error) {
 		return nil, err
 	}
 
-	return httpGetOverConn(c, c.HandshakeState.ServerHello.AlpnProtocol)
+	return httpGetOverConn(c, c.ConnectionState().NegotiatedProtocol)
 }
 
 func forgeConn() {

go.mod

@@ -9,7 +9,7 @@ retract (
 
 require (
 	github.com/andybalholm/brotli v1.0.5
-	github.com/cloudflare/circl v1.3.3
+	github.com/cloudflare/circl v1.3.6
 	github.com/klauspost/compress v1.16.7
 	github.com/quic-go/quic-go v0.37.4
 	golang.org/x/crypto v0.14.0