crypto/x509: store iOS root certs in compressed form

This change combines two changes from elsewhere:
tailscale/tailscale@28c632c
https://go-review.googlesource.com/c/go/+/229917

Author: bradfitz

src/crypto/x509/cert_pool.go

@@ -5,31 +5,65 @@
 package x509
 
 import (
+	"crypto/sha256"
 	"encoding/pem"
 	"errors"
 	"runtime"
+	"sync"
 )
 
+type sum224 [sha256.Size224]byte
+
 // CertPool is a set of certificates.
 type CertPool struct {
-	bySubjectKeyId map[string][]int
-	byName         map[string][]int
-	certs          []*Certificate
+	bySubjectKeyId map[string][]int // cert.SubjectKeyId => getCert index(es)
+	byName         map[string][]int // cert.RawSubject => getCert index(es)
+
+	// haveSum maps from sum224(cert.Raw) to true. It's used only
+	// for AddCert duplicate detection, to avoid CertPool.contains
+	// calls in the AddCert path (because the contains method can
+	// call getCert and otherwise negate savings from lazy getCert
+	// funcs).
+	haveSum map[sum224]bool
+
+	// getCert contains funcs that return the certificates.
+	getCert []func() (*Certificate, error)
+
+	// rawSubjects is each cert's RawSubject field.
+	// Its indexes correspond to the getCert indexes.
+	rawSubjects [][]byte
 }
 
 // NewCertPool returns a new, empty CertPool.
 func NewCertPool() *CertPool {
 	return &CertPool{
 		bySubjectKeyId: make(map[string][]int),
 		byName:         make(map[string][]int),
+		haveSum:        make(map[sum224]bool),
+	}
+}
+
+// len returns the number of certs in the set.
+// A nil set is a valid empty set.
+func (s *CertPool) len() int {
+	if s == nil {
+		return 0
 	}
+	return len(s.getCert)
+}
+
+// cert returns cert index n in s.
+func (s *CertPool) cert(n int) (*Certificate, error) {
+	return s.getCert[n]()
 }
 
 func (s *CertPool) copy() *CertPool {
 	p := &CertPool{
 		bySubjectKeyId: make(map[string][]int, len(s.bySubjectKeyId)),
 		byName:         make(map[string][]int, len(s.byName)),
-		certs:          make([]*Certificate, len(s.certs)),
+		haveSum:        make(map[sum224]bool, len(s.haveSum)),
+		getCert:        make([]func() (*Certificate, error), len(s.getCert)),
+		rawSubjects:    make([][]byte, len(s.rawSubjects)),
 	}
 	for k, v := range s.bySubjectKeyId {
 		indexes := make([]int, len(v))
@@ -41,7 +75,11 @@ func (s *CertPool) copy() *CertPool {
 		copy(indexes, v)
 		p.byName[k] = indexes
 	}
-	copy(p.certs, s.certs)
+	for k := range s.haveSum {
+		p.haveSum[k] = true
+	}
+	copy(p.getCert, s.getCert)
+	copy(p.rawSubjects, s.rawSubjects)
 	return p
 }
 
@@ -82,41 +120,61 @@ func (s *CertPool) findPotentialParents(cert *Certificate) []int {
 	return candidates
 }
 
-func (s *CertPool) contains(cert *Certificate) bool {
+func (s *CertPool) contains(cert *Certificate) (bool, error) {
 	if s == nil {
-		return false
+		return false, nil
 	}
-
 	candidates := s.byName[string(cert.RawSubject)]
-	for _, c := range candidates {
-		if s.certs[c].Equal(cert) {
-			return true
+	for _, i := range candidates {
+		c, err := s.cert(i)
+		if err != nil {
+			return false, err
+		}
+		if c.Equal(cert) {
+			return true, nil
 		}
 	}
 
-	return false
+	return false, nil
 }
 
 // AddCert adds a certificate to a pool.
 func (s *CertPool) AddCert(cert *Certificate) {
 	if cert == nil {
 		panic("adding nil Certificate to CertPool")
 	}
+	s.addCertFunc(sha256.Sum224(cert.Raw), string(cert.RawSubject), string(cert.SubjectKeyId), func() (*Certificate, error) {
+		return cert, nil
+	})
+}
 
+// addCertFunc adds metadata about a certificate to a pool, along with
+// a func to fetch that certificate later when needed.
+//
+// The rawSubject is Certificate.RawSubject and must be non-empty.
+// The subjectKeyID is Certificate.SubjectKeyId and may be empty.
+// The getCert func may be called 0 or more times.
+func (s *CertPool) addCertFunc(rawSum224 sum224, rawSubject, subjectKeyID string, getCert func() (*Certificate, error)) {
 	// Check that the certificate isn't being added twice.
-	if s.contains(cert) {
+	if s.haveSum[rawSum224] {
 		return
 	}
+	s.haveSum[rawSum224] = true
+	s.addCertFuncNotDup(rawSubject, subjectKeyID, getCert)
+}
 
-	n := len(s.certs)
-	s.certs = append(s.certs, cert)
+func (s *CertPool) addCertFuncNotDup(rawSubject, subjectKeyID string, getCert func() (*Certificate, error)) {
+	if getCert == nil {
+		panic("getCert can't be nil")
+	}
+	n := len(s.getCert)
+	s.getCert = append(s.getCert, getCert)
 
-	if len(cert.SubjectKeyId) > 0 {
-		keyId := string(cert.SubjectKeyId)
-		s.bySubjectKeyId[keyId] = append(s.bySubjectKeyId[keyId], n)
+	if subjectKeyID != "" {
+		s.bySubjectKeyId[subjectKeyID] = append(s.bySubjectKeyId[subjectKeyID], n)
 	}
-	name := string(cert.RawSubject)
-	s.byName[name] = append(s.byName[name], n)
+	s.byName[rawSubject] = append(s.byName[rawSubject], n)
+	s.rawSubjects = append(s.rawSubjects, []byte(rawSubject))
 }
 
 // AppendCertsFromPEM attempts to parse a series of PEM encoded certificates.
@@ -136,24 +194,34 @@ func (s *CertPool) AppendCertsFromPEM(pemCerts []byte) (ok bool) {
 			continue
 		}
 
-		cert, err := ParseCertificate(block.Bytes)
+		certBytes := block.Bytes
+		cert, err := ParseCertificate(certBytes)
 		if err != nil {
 			continue
 		}
-
-		s.AddCert(cert)
+		var lazyCert struct {
+			sync.Once
+			v *Certificate
+		}
+		s.addCertFunc(sha256.Sum224(cert.Raw), string(cert.RawSubject), string(cert.SubjectKeyId), func() (*Certificate, error) {
+			lazyCert.Do(func() {
+				// This can't fail, as the same bytes already parsed above.
+				lazyCert.v, _ = ParseCertificate(certBytes)
+				certBytes = nil
+			})
+			return lazyCert.v, nil
+		})
 		ok = true
 	}
-
-	return
+	return ok
 }
 
 // Subjects returns a list of the DER-encoded subjects of
 // all of the certificates in the pool.
 func (s *CertPool) Subjects() [][]byte {
-	res := make([][]byte, len(s.certs))
-	for i, c := range s.certs {
-		res[i] = c.RawSubject
+	res := make([][]byte, s.len())
+	for i, s := range s.rawSubjects {
+		res[i] = s
 	}
 	return res
 }

src/crypto/x509/root_darwin_armx.go renamed to src/crypto/x509/certs.pem

@@ -1,23 +1,3 @@
-// Code generated by root_darwin_arm_gen --output root_darwin_armx.go; DO NOT EDIT.
-
-// Copyright 2015 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build cgo
-// +build darwin
-// +build arm arm64 ios
-// +build !x509omitbundledroots
-
-package x509
-
-func loadSystemRoots() (*CertPool, error) {
-	p := NewCertPool()
-	p.AppendCertsFromPEM([]byte(systemRootsPEM))
-	return p, nil
-}
-
-const systemRootsPEM = `
 -----BEGIN CERTIFICATE-----
 MIIEMjCCAxqgAwIBAgIBATANBgkqhkiG9w0BAQUFADB7MQswCQYDVQQGEwJHQjEb
 MBkGA1UECAwSR3JlYXRlciBNYW5jaGVzdGVyMRAwDgYDVQQHDAdTYWxmb3JkMRow
@@ -4311,4 +4291,3 @@ IR9NmXmd4c8nnxCbHIgNsIpkQTG4DmyQJKSbXHGPurt+HBvbaoAPIbzp26a3QPSy
 i6mx5O+aGtA9aZnuqCij4Tyz8LIRnM98QObd50N9otg6tamN8jSZxNQQ4Qb9CYQQ
 O+7ETPTsJ3xCwnR8gooJybQDJbw=
 -----END CERTIFICATE-----
-`

src/crypto/x509/name_constraints_test.go

@@ -1993,7 +1993,7 @@ func TestConstraintCases(t *testing.T) {
 				pem.Encode(&buf, &pem.Block{Type: "CERTIFICATE", Bytes: cert.Raw})
 				return buf.String()
 			}
-			t.Errorf("#%d: root:\n%s", i, certAsPEM(rootPool.certs[0]))
+			t.Errorf("#%d: root:\n%s", i, certAsPEM(rootPool.mustCert(0)))
 			t.Errorf("#%d: leaf:\n%s", i, certAsPEM(leafCert))
 		}
 
@@ -2019,19 +2019,27 @@ func writePEMsToTempFile(certs []*Certificate) *os.File {
 	return file
 }
 
+func allCerts(p *CertPool) []*Certificate {
+	all := make([]*Certificate, p.len())
+	for i := range all {
+		all[i] = p.mustCert(i)
+	}
+	return all
+}
+
 func testChainAgainstOpenSSL(leaf *Certificate, intermediates, roots *CertPool) (string, error) {
 	args := []string{"verify", "-no_check_time"}
 
-	rootsFile := writePEMsToTempFile(roots.certs)
+	rootsFile := writePEMsToTempFile(allCerts(roots))
 	if debugOpenSSLFailure {
 		println("roots file:", rootsFile.Name())
 	} else {
 		defer os.Remove(rootsFile.Name())
 	}
 	args = append(args, "-CAfile", rootsFile.Name())
 
-	if len(intermediates.certs) > 0 {
-		intermediatesFile := writePEMsToTempFile(intermediates.certs)
+	if intermediates.len() > 0 {
+		intermediatesFile := writePEMsToTempFile(allCerts(intermediates))
 		if debugOpenSSLFailure {
 			println("intermediates file:", intermediatesFile.Name())
 		} else {

src/crypto/x509/omit_system_roots.go

@@ -0,0 +1,12 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build x509omitsystemroots
+
+package x509
+
+func loadSystemRoots() (*CertPool, error) {
+	p := NewCertPool()
+	return p, nil
+}

src/crypto/x509/pem_decrypt.go

@@ -2,8 +2,6 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !omitpemdecrypt
-
 package x509
 
 // RFC 1423 describes the encryption of PEM blocks. The algorithm used to

src/crypto/x509/pool_darwin_arm64.go

@@ -0,0 +1,36 @@
+// Copyright 2020 Tailscale Inc. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !x509omitbundledroots
+
+package x509
+
+import (
+	"compress/gzip"
+	"io/ioutil"
+	"strings"
+	"sync"
+)
+
+func certUncompressor(zcertBytes string) func() (*Certificate, error) {
+	var once sync.Once
+	var c *Certificate
+	var err error
+	return func() (*Certificate, error) {
+		once.Do(func() {
+			var certBytes []byte
+			var zr *gzip.Reader
+			zr, err = gzip.NewReader(strings.NewReader(zcertBytes))
+			if err != nil {
+				return
+			}
+			certBytes, err = ioutil.ReadAll(zr)
+			if err != nil {
+				return
+			}
+			c, err = ParseCertificate(certBytes)
+		})
+		return c, err
+	}
+}

src/crypto/x509/root_cgo_darwin.go

@@ -2,12 +2,12 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build cgo,!arm,!arm64,!ios
+// +build cgo,!arm64,!ios
 
 package x509
 
 /*
-#cgo CFLAGS: -mmacosx-version-min=10.10 -D__MAC_OS_X_VERSION_MAX_ALLOWED=101300
+#cgo CFLAGS: -mmacosx-version-min=10.11
 #cgo LDFLAGS: -framework CoreFoundation -framework Security
 
 #include <errno.h>
@@ -305,8 +305,16 @@ func loadSystemRoots() (*CertPool, error) {
 	untrustedRoots.AppendCertsFromPEM(buf)
 
 	trustedRoots := NewCertPool()
-	for _, c := range roots.certs {
-		if !untrustedRoots.contains(c) {
+	for i := 0; i < roots.len(); i++ {
+		c, err := roots.cert(i)
+		if err != nil {
+			return nil, err
+		}
+		contains, err := untrustedRoots.contains(c)
+		if err != nil {
+			return nil, err
+		}
+		if !contains {
 			trustedRoots.AddCert(c)
 		}
 	}