use Huffman encoding for field names and values

Author: marten-seemann

encoder.go

@@ -2,6 +2,8 @@ package qpack
 
 import (
 	"io"
+
+	"golang.org/x/net/http2/hpack"
 )
 
 // An Encoder performs QPACK encoding.
@@ -64,31 +66,31 @@ func (e *Encoder) Close() error {
 
 func (e *Encoder) writeLiteralFieldWithoutNameReference(f HeaderField) {
 	offset := len(e.buf)
-	e.buf = appendVarInt(e.buf, 3, uint64(len(f.Name)))
-	e.buf[offset] ^= 0x20
-	e.buf = append(e.buf, []byte(f.Name)...)
-	e.buf = appendVarInt(e.buf, 7, uint64(len(f.Value)))
-	e.buf = append(e.buf, []byte(f.Value)...)
+	e.buf = appendVarInt(e.buf, 3, hpack.HuffmanEncodeLength(f.Name))
+	e.buf[offset] ^= 0x20 ^ 0x8
+	e.buf = hpack.AppendHuffmanString(e.buf, f.Name)
+	offset = len(e.buf)
+	e.buf = appendVarInt(e.buf, 7, hpack.HuffmanEncodeLength(f.Value))
+	e.buf[offset] ^= 0x80
+	e.buf = hpack.AppendHuffmanString(e.buf, f.Value)
 }
 
-// Encodes a header field whose name is present in one of the
-// tables.
-func (e *Encoder) writeLiteralFieldWithNameReference(
-	f *HeaderField, idx uint8) {
+// Encodes a header field whose name is present in one of the tables.
+func (e *Encoder) writeLiteralFieldWithNameReference(f *HeaderField, id uint8) {
 	offset := len(e.buf)
-	e.buf = appendVarInt(e.buf, 4, uint64(idx))
+	e.buf = appendVarInt(e.buf, 4, uint64(id))
 	// Set the 01NTxxxx pattern, forcing N to 0 and T to 1
 	e.buf[offset] ^= 0x50
-
-	e.buf = appendVarInt(e.buf, 7, uint64(len(f.Value)))
-	e.buf = append(e.buf, []byte(f.Value)...)
+	offset = len(e.buf)
+	e.buf = appendVarInt(e.buf, 7, hpack.HuffmanEncodeLength(f.Value))
+	e.buf[offset] ^= 0x80
+	e.buf = hpack.AppendHuffmanString(e.buf, f.Value)
 }
 
-// Encodes an indexed field, meaning it's entirely defined in one of the
-// tables.
-func (e *Encoder) writeIndexedField(idx uint8) {
+// Encodes an indexed field, meaning it's entirely defined in one of the tables.
+func (e *Encoder) writeIndexedField(id uint8) {
 	offset := len(e.buf)
-	e.buf = appendVarInt(e.buf, 6, uint64(idx))
+	e.buf = appendVarInt(e.buf, 6, uint64(id))
 	// Set the 1Txxxxxx pattern, forcing T to 1
 	e.buf[offset] ^= 0xc0
 }

encoder_test.go

@@ -3,6 +3,8 @@ package qpack
 import (
 	"bytes"
 
+	"golang.org/x/net/http2/hpack"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
@@ -29,41 +31,42 @@ var _ = Describe("Encoder", func() {
 
 	checkHeaderField := func(data []byte, hf HeaderField) []byte {
 		Expect(data[0] & (0x80 ^ 0x40 ^ 0x20)).To(Equal(uint8(0x20))) // 001xxxxx
-		Expect(data[0] & 0x8).To(BeZero())                            // no Huffman encoding
+		Expect(data[0] & 0x8).ToNot(BeZero())                         // Huffman encoding
 		nameLen, data, err := readVarInt(3, data)
 		Expect(err).ToNot(HaveOccurred())
-		Expect(nameLen).To(BeEquivalentTo(len(hf.Name)))
-		Expect(string(data[:len(hf.Name)])).To(Equal(hf.Name))
-		valueLen, data, err := readVarInt(7, data[len(hf.Name):])
-		Expect(err).ToNot(HaveOccurred())
-		Expect(valueLen).To(BeEquivalentTo(len(hf.Value)))
-		Expect(string(data[:len(hf.Value)])).To(Equal(hf.Value))
-		return data[len(hf.Value):]
+		l := hpack.HuffmanEncodeLength(hf.Name)
+		Expect(nameLen).To(BeEquivalentTo(l))
+		Expect(hpack.HuffmanDecodeToString(data[:l])).To(Equal(hf.Name))
+		valueLen, data, err := readVarInt(7, data[l:])
+		l = hpack.HuffmanEncodeLength(hf.Value)
+		Expect(valueLen).To(BeEquivalentTo(l))
+		Expect(hpack.HuffmanDecodeToString(data[:l])).To(Equal(hf.Value))
+		return data[l:]
 	}
 
-	// Reads one indexed field line representation from data and verifies it
-	// matches expected_hf.
+	// Reads one indexed field line representation from data and verifies it matches hf.
 	// Returns the leftover bytes from data.
-	checkIndexedHeaderField := func(data []byte, expected_hf HeaderField) []byte {
+	checkIndexedHeaderField := func(data []byte, hf HeaderField) []byte {
 		Expect(data[0] >> 7).To(Equal(uint8(1))) // 1Txxxxxx
 		index, data, err := readVarInt(6, data)
 		Expect(err).ToNot(HaveOccurred())
-		Expect(staticTableEntries[index]).To(Equal(expected_hf))
+		Expect(staticTableEntries[index]).To(Equal(hf))
 		return data
 	}
 
-	checkHeaderFieldWithNameRef := func(data []byte, expected_hf HeaderField) []byte {
+	checkHeaderFieldWithNameRef := func(data []byte, hf HeaderField) []byte {
 		// read name reference
 		Expect(data[0] >> 6).To(Equal(uint8(1))) // 01NTxxxx
 		index, data, err := readVarInt(4, data)
 		Expect(err).ToNot(HaveOccurred())
-		Expect(staticTableEntries[index].Name).To(Equal(expected_hf.Name))
+		Expect(staticTableEntries[index].Name).To(Equal(hf.Name))
 		// read literal value
 		valueLen, data, err := readVarInt(7, data)
 		Expect(err).ToNot(HaveOccurred())
-		Expect(valueLen).To(BeEquivalentTo(len(expected_hf.Value)))
-		Expect(string(data[:len(expected_hf.Value)])).To(Equal(expected_hf.Value))
-		return data[len(expected_hf.Value):]
+		l := hpack.HuffmanEncodeLength(hf.Value)
+		Expect(valueLen).To(BeEquivalentTo(l))
+		Expect(hpack.HuffmanDecodeToString(data[:l])).To(Equal(hf.Value))
+		return data[l:]
 	}
 
 	It("encodes a single field", func() {

integrationtests/self/integration_test.go

@@ -162,7 +162,9 @@ var _ = Describe("Self Tests", func() {
 	It("uses the static table for field names, for fields with values", func() {
 		var hf qpack.HeaderField
 		for {
-			if entry := staticTable[rand.Intn(len(staticTable))]; len(entry.Value) > 0 {
+			// Only use values with at least 2 characters.
+			// This makes sure that Huffman enocding doesn't compress them as much as encoding it using the static table would.
+			if entry := staticTable[rand.Intn(len(staticTable))]; len(entry.Value) > 1 {
 				hf = qpack.HeaderField{
 					Name:  entry.Name,
 					Value: randomString(20),
@@ -184,7 +186,9 @@ var _ = Describe("Self Tests", func() {
 	It("uses the static table for field values", func() {
 		var hf qpack.HeaderField
 		for {
-			if entry := staticTable[rand.Intn(len(staticTable))]; len(entry.Value) > 0 {
+			// Only use values with at least 2 characters.
+			// This makes sure that Huffman enocding doesn't compress them as much as encoding it using the static table would.
+			if entry := staticTable[rand.Intn(len(staticTable))]; len(entry.Value) > 1 {
 				hf = qpack.HeaderField{
 					Name:  entry.Name,
 					Value: entry.Value,