Implement certificate compression

Certificate compression is defined in RFC 8879:
https://datatracker.ietf.org/doc/html/rfc8879

This implementation is client-side only, for server certificates.

Author: hwh33

conn.go

@@ -1057,6 +1057,10 @@ func (c *Conn) readHandshake() (interface{}, error) {
 		m = new(endOfEarlyDataMsg)
 	case typeKeyUpdate:
 		m = new(keyUpdateMsg)
+	// [UTLS SECTION BEGINS]
+	case typeCompressedCertificate:
+		m = new(compressedCertificateMsg)
+	// [UTLS SECTION ENDS]
 	default:
 		return nil, c.in.setErrorLocked(c.sendAlert(alertUnexpectedMessage))
 	}

go.mod

@@ -0,0 +1,10 @@
+module github.com/refraction-networking/utls
+
+go 1.16
+
+require (
+	github.com/andybalholm/brotli v1.0.4
+	github.com/klauspost/compress v1.13.6
+	golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa
+	golang.org/x/net v0.0.0-20211111160137-58aab5ef257a
+)

go.sum

@@ -0,0 +1,18 @@
+github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
+github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
+github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
+github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa h1:idItI2DDfCokpg0N51B2VtiLdJ4vAuXC9fnCb2gACo4=
+golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20211111160137-58aab5ef257a h1:c83jeVQW0KGKNaKBRfelNYNHaev+qawl9yaA825s8XE=
+golang.org/x/net v0.0.0-20211111160137-58aab5ef257a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

handshake_client_tls13.go

@@ -6,14 +6,19 @@ package tls
 
 import (
 	"bytes"
+	"compress/zlib"
 	"crypto"
 	"crypto/hmac"
 	"crypto/rsa"
 	"errors"
 	"fmt"
 	"hash"
+	"io"
 	"sync/atomic"
 	"time"
+
+	"github.com/andybalholm/brotli"
+	"github.com/klauspost/compress/zstd"
 )
 
 type clientHandshakeStateTLS13 struct {
@@ -484,6 +489,24 @@ func (hs *clientHandshakeStateTLS13) readServerCertificate() error {
 		}
 	}
 
+	// [UTLS SECTION BEGINS]
+	receivedCompressedCert := false
+	// Check to see if we advertised any compression algorithms
+	if hs.uconn != nil && len(hs.uconn.certCompressionAlgs) > 0 {
+		// Check to see if the message is a compressed certificate message, otherwise move on.
+		compressedCertMsg, ok := msg.(*compressedCertificateMsg)
+		if ok {
+			receivedCompressedCert = true
+			hs.transcript.Write(compressedCertMsg.marshal())
+
+			msg, err = hs.decompressCert(*compressedCertMsg)
+			if err != nil {
+				return fmt.Errorf("tls: failed to decompress certificate message: %w", err)
+			}
+		}
+	}
+	// [UTLS SECTION ENDS]
+
 	certMsg, ok := msg.(*certificateMsgTLS13)
 	if !ok {
 		c.sendAlert(alertUnexpectedMessage)
@@ -493,7 +516,12 @@ func (hs *clientHandshakeStateTLS13) readServerCertificate() error {
 		c.sendAlert(alertDecodeError)
 		return errors.New("tls: received empty certificates message")
 	}
-	hs.transcript.Write(certMsg.marshal())
+	// [UTLS SECTION BEGINS]
+	// Previously, this was simply 'hs.transcript.Write(certMsg.marshal())' (without the if).
+	if !receivedCompressedCert {
+		hs.transcript.Write(certMsg.marshal())
+	}
+	// [UTLS SECTION ENDS]
 
 	c.scts = certMsg.certificate.SignedCertificateTimestamps
 	c.ocspResponse = certMsg.certificate.OCSPStaple
@@ -690,6 +718,80 @@ func (hs *clientHandshakeStateTLS13) sendClientFinished() error {
 	return nil
 }
 
+// [UTLS SECTION BEGINS]
+func (hs *clientHandshakeStateTLS13) decompressCert(m compressedCertificateMsg) (*certificateMsgTLS13, error) {
+	var (
+		decompressed io.Reader
+		compressed   = bytes.NewReader(m.compressedCertificateMessage)
+		c            = hs.c
+	)
+
+	// Check to see if the peer responded with an algorithm we advertised.
+	supportedAlg := false
+	for _, alg := range hs.uconn.certCompressionAlgs {
+		if m.algorithm == uint16(alg) {
+			supportedAlg = true
+		}
+	}
+	if !supportedAlg {
+		c.sendAlert(alertBadCertificate)
+		return nil, fmt.Errorf("unadvertised algorithm (%d)", m.algorithm)
+	}
+
+	switch CertCompressionAlgo(m.algorithm) {
+	case CertCompressionBrotli:
+		decompressed = brotli.NewReader(compressed)
+
+	case CertCompressionZlib:
+		rc, err := zlib.NewReader(compressed)
+		if err != nil {
+			c.sendAlert(alertBadCertificate)
+			return nil, fmt.Errorf("failed to open zlib reader: %w", err)
+		}
+		defer rc.Close()
+		decompressed = rc
+
+	case CertCompressionZstd:
+		rc, err := zstd.NewReader(compressed)
+		if err != nil {
+			c.sendAlert(alertBadCertificate)
+			return nil, fmt.Errorf("failed to open zstd reader: %w", err)
+		}
+		defer rc.Close()
+		decompressed = rc
+
+	default:
+		c.sendAlert(alertBadCertificate)
+		return nil, fmt.Errorf("unsupported algorithm (%d)", m.algorithm)
+	}
+
+	rawMsg := make([]byte, m.uncompressedLength+4) // +4 for message type and uint24 length field
+	rawMsg[0] = typeCertificate
+	rawMsg[1] = uint8(m.uncompressedLength >> 16)
+	rawMsg[2] = uint8(m.uncompressedLength >> 8)
+	rawMsg[3] = uint8(m.uncompressedLength)
+
+	n, err := decompressed.Read(rawMsg[4:])
+	if err != nil {
+		c.sendAlert(alertBadCertificate)
+		return nil, err
+	}
+	if n < len(rawMsg)-4 {
+		// If, after decompression, the specified length does not match the actual length, the party
+		// receiving the invalid message MUST abort the connection with the "bad_certificate" alert.
+		// https://datatracker.ietf.org/doc/html/rfc8879#section-4
+		c.sendAlert(alertBadCertificate)
+		return nil, fmt.Errorf("decompressed len (%d) does not match specified len (%d)", n, m.uncompressedLength)
+	}
+	certMsg := new(certificateMsgTLS13)
+	if !certMsg.unmarshal(rawMsg) {
+		return nil, c.sendAlert(alertUnexpectedMessage)
+	}
+	return certMsg, nil
+}
+
+// [UTLS SECTION ENDS]
+
 func (c *Conn) handleNewSessionTicket(msg *newSessionTicketMsgTLS13) error {
 	if !c.isClient {
 		c.sendAlert(alertUnexpectedMessage)

handshake_messages_test.go

@@ -36,6 +36,7 @@ var tests = []interface{}{
 	&newSessionTicketMsgTLS13{},
 	&certificateRequestMsgTLS13{},
 	&certificateMsgTLS13{},
+	&compressedCertificateMsg{}, // [UTLS]
 }
 
 func TestMarshalUnmarshal(t *testing.T) {
@@ -420,6 +421,15 @@ func (*certificateMsgTLS13) Generate(rand *rand.Rand, size int) reflect.Value {
 	return reflect.ValueOf(m)
 }
 
+// [UTLS]
+func (*compressedCertificateMsg) Generate(rand *rand.Rand, size int) reflect.Value {
+	m := &compressedCertificateMsg{}
+	m.algorithm = uint16(rand.Intn(2 << 15))
+	m.uncompressedLength = uint32(rand.Intn(2 << 23))
+	m.compressedCertificateMessage = randomBytes(rand.Intn(500)+1, rand)
+	return reflect.ValueOf(m)
+}
+
 func TestRejectEmptySCTList(t *testing.T) {
 	// RFC 6962, Section 3.3.1 specifies that empty SCT lists are invalid.
 

u_common.go

@@ -19,11 +19,16 @@ const (
 	utlsExtensionPadding              uint16 = 21
 	utlsExtensionExtendedMasterSecret uint16 = 23 // https://tools.ietf.org/html/rfc7627
 
+	// https://datatracker.ietf.org/doc/html/rfc8879#section-7.1
+	utlsExtensionCompressCertificate uint16 = 27
+
 	// extensions with 'fake' prefix break connection, if server echoes them back
 	fakeExtensionChannelID uint16 = 30032 // not IANA assigned
 
-	fakeCertCompressionAlgs uint16 = 0x001b
-	fakeRecordSizeLimit     uint16 = 0x001c
+	fakeRecordSizeLimit uint16 = 0x001c
+
+	// https://datatracker.ietf.org/doc/html/rfc8879#section-7.2
+	typeCompressedCertificate uint8 = 25
 )
 
 const (
@@ -37,11 +42,11 @@ const (
 	FAKE_OLD_TLS_DHE_RSA_WITH_CHACHA20_POLY1305_SHA256 = uint16(0xcc15) // we can try to craft these ciphersuites
 	FAKE_TLS_DHE_RSA_WITH_AES_128_GCM_SHA256           = uint16(0x009e) // from existing pieces, if needed
 
-	FAKE_TLS_DHE_RSA_WITH_AES_128_CBC_SHA  = uint16(0x0033)
-	FAKE_TLS_DHE_RSA_WITH_AES_256_CBC_SHA  = uint16(0x0039)
-	FAKE_TLS_DHE_RSA_WITH_AES_256_GCM_SHA384  = uint16(0x009f)
-	FAKE_TLS_RSA_WITH_RC4_128_MD5          = uint16(0x0004)
-	FAKE_TLS_EMPTY_RENEGOTIATION_INFO_SCSV = uint16(0x00ff)
+	FAKE_TLS_DHE_RSA_WITH_AES_128_CBC_SHA    = uint16(0x0033)
+	FAKE_TLS_DHE_RSA_WITH_AES_256_CBC_SHA    = uint16(0x0039)
+	FAKE_TLS_DHE_RSA_WITH_AES_256_GCM_SHA384 = uint16(0x009f)
+	FAKE_TLS_RSA_WITH_RC4_128_MD5            = uint16(0x0004)
+	FAKE_TLS_EMPTY_RENEGOTIATION_INFO_SCSV   = uint16(0x00ff)
 )
 
 // newest signatures
@@ -65,6 +70,7 @@ type CertCompressionAlgo uint16
 const (
 	CertCompressionZlib   CertCompressionAlgo = 0x0001
 	CertCompressionBrotli CertCompressionAlgo = 0x0002
+	CertCompressionZstd   CertCompressionAlgo = 0x0003
 )
 
 const (

u_conn.go

@@ -32,6 +32,11 @@ type UConn struct {
 	greaseSeed [ssl_grease_last_index]uint16
 
 	omitSNIExtension bool
+
+	// certCompressionAlgs represents the set of advertised certificate compression
+	// algorithms, as specified in the ClientHello. This is only relevant client-side, for the
+	// server certificate. All other forms of certificate compression are unsupported.
+	certCompressionAlgs []CertCompressionAlgo
 }
 
 // UClient returns a new uTLS client, with behavior depending on clientHelloID.

u_fingerprinter.go

@@ -303,7 +303,22 @@ func (f *Fingerprinter) FingerprintClientHello(data []byte) (*ClientHelloSpec, e
 		case utlsExtensionPadding:
 			clientHelloSpec.Extensions = append(clientHelloSpec.Extensions, &UtlsPaddingExtension{GetPaddingLen: BoringPaddingStyle})
 
-		case fakeExtensionChannelID, fakeCertCompressionAlgs, fakeRecordSizeLimit:
+		case utlsExtensionCompressCertificate:
+			methods := []CertCompressionAlgo{}
+			methodsRaw := new(cryptobyte.String)
+			if !extData.ReadUint8LengthPrefixed(methodsRaw) {
+				return nil, errors.New("unable to read cert compression algorithms extension data")
+			}
+			for !methodsRaw.Empty() {
+				var method uint16
+				if !methodsRaw.ReadUint16(&method) {
+					return nil, errors.New("unable to read cert compression algorithms extension data")
+				}
+				methods = append(methods, CertCompressionAlgo(method))
+			}
+			clientHelloSpec.Extensions = append(clientHelloSpec.Extensions, &UtlsCompressCertExtension{methods})
+
+		case fakeExtensionChannelID, fakeRecordSizeLimit:
 			clientHelloSpec.Extensions = append(clientHelloSpec.Extensions, &GenericExtension{extension, extData})
 
 		case extensionPreSharedKey:

u_handshake_messages.go

@@ -0,0 +1,49 @@
+package tls
+
+import (
+	"golang.org/x/crypto/cryptobyte"
+)
+
+// Only implemented client-side, for server certificates.
+// Alternate certificate message formats (https://datatracker.ietf.org/doc/html/rfc7250) are not
+// supported.
+// https://datatracker.ietf.org/doc/html/rfc8879
+type compressedCertificateMsg struct {
+	raw []byte
+
+	algorithm                    uint16
+	uncompressedLength           uint32 // uint24
+	compressedCertificateMessage []byte
+}
+
+func (m *compressedCertificateMsg) marshal() []byte {
+	if m.raw != nil {
+		return m.raw
+	}
+
+	var b cryptobyte.Builder
+	b.AddUint8(typeCompressedCertificate)
+	b.AddUint24LengthPrefixed(func(b *cryptobyte.Builder) {
+		b.AddUint16(m.algorithm)
+		b.AddUint24(m.uncompressedLength)
+		b.AddUint24LengthPrefixed(func(b *cryptobyte.Builder) {
+			b.AddBytes(m.compressedCertificateMessage)
+		})
+	})
+
+	m.raw = b.BytesOrPanic()
+	return m.raw
+}
+
+func (m *compressedCertificateMsg) unmarshal(data []byte) bool {
+	*m = compressedCertificateMsg{raw: data}
+	s := cryptobyte.String(data)
+
+	if !s.Skip(4) || // message type and uint24 length field
+		!s.ReadUint16(&m.algorithm) ||
+		!s.ReadUint24(&m.uncompressedLength) ||
+		!readUint24LengthPrefixed(&s, &m.compressedCertificateMessage) {
+		return false
+	}
+	return true
+}

u_parrots.go

@@ -133,7 +133,7 @@ func utlsIdToSpec(id ClientHelloID) (ClientHelloSpec, error) {
 					CurveP256,
 					CurveP384,
 				}},
-				&FakeCertCompressionAlgsExtension{[]CertCompressionAlgo{CertCompressionBrotli}},
+				&UtlsCompressCertExtension{[]CertCompressionAlgo{CertCompressionBrotli}},
 				&UtlsGREASEExtension{},
 				&UtlsPaddingExtension{GetPaddingLen: BoringPaddingStyle},
 			},
@@ -205,7 +205,7 @@ func utlsIdToSpec(id ClientHelloID) (ClientHelloSpec, error) {
 					VersionTLS11,
 					VersionTLS10,
 				}},
-				&FakeCertCompressionAlgsExtension{[]CertCompressionAlgo{
+				&UtlsCompressCertExtension{[]CertCompressionAlgo{
 					CertCompressionBrotli,
 				}},
 				&UtlsGREASEExtension{},
@@ -277,7 +277,7 @@ func utlsIdToSpec(id ClientHelloID) (ClientHelloSpec, error) {
 					VersionTLS11,
 					VersionTLS10,
 				}},
-				&FakeCertCompressionAlgsExtension{[]CertCompressionAlgo{
+				&UtlsCompressCertExtension{[]CertCompressionAlgo{
 					CertCompressionBrotli,
 				}},
 				&UtlsGREASEExtension{},