1 From 26741a15f77ebea1d5b72ff6f7c34c8494812288 Mon Sep 17 00:00:00 2001

     2 From: Quentin Smith <[email protected]>

     3 Date: Wed, 30 Nov 2016 15:16:37 -0500

     4 Subject: [PATCH 35/38] [release-branch.go1.7] crypto/x509: read Darwin trust

     5  settings for root CAs

     6 

     7 Darwin separately stores bits indicating whether a root certificate

     8 should be trusted; this changes Go to read and use those when

     9 initializing SystemCertPool.

    10 

    11 Unfortunately, the trust API is very slow. To avoid a delay of up to

    12 0.5s in initializing the system cert pool, we assume that

    13 the trust settings found in kSecTrustSettingsDomainSystem will always

    14 indicate trust. (That is, all root certs Apple distributes are trusted.)

    15 This is not guaranteed by the API but is true in practice.

    16 

    17 In the non-cgo codepath, we do not have that benefit, so we must check

    18 the trust status of every certificate. This causes about 0.5s of delay

    19 in initializing the SystemCertPool.

    20 

    21 On OS X 10.11 and older, the "security" command requires a certificate

    22 to be provided in a file and not on stdin, so the non-cgo codepath

    23 creates temporary files for each certificate, further slowing initialization.

    24 

    25 Updates #18141.

    26 

    27 Change-Id: If681c514047afe5e1a68de6c9d40ceabbce54755

    28 Reviewed-on: https://go-review.googlesource.com/33721

    29 Run-TryBot: Quentin Smith <[email protected]>

    30 TryBot-Result: Gobot Gobot <[email protected]>

    31 Reviewed-by: Russ Cox <[email protected]>

    32 Reviewed-on: https://go-review.googlesource.com/33727

    33 ---

    34  src/crypto/x509/cert_pool.go        |  15 +++++

    35  src/crypto/x509/root_cgo_darwin.go  |  81 ++++++++++++++++++++++---

    36  src/crypto/x509/root_darwin.go      | 114 +++++++++++++++++++++++++++++++++++-

    37  src/crypto/x509/root_darwin_test.go |   1 +

    38  4 files changed, 200 insertions(+), 11 deletions(-)

    39 

    40 diff --git a/src/crypto/x509/cert_pool.go b/src/crypto/x509/cert_pool.go

    41 index 59ab887..8438bf6 100644

    42 --- a/src/crypto/x509/cert_pool.go

    43 +++ b/src/crypto/x509/cert_pool.go

    44 @@ -64,6 +64,21 @@ func (s *CertPool) findVerifiedParents(cert *Certificate) (parents []int, errCer

    45  	return

    46  }

    47  

    48 +func (s *CertPool) contains(cert *Certificate) bool {

    49 +	if s == nil {

    50 +		return false

    51 +	}

    52 +

    53 +	candidates := s.byName[string(cert.RawSubject)]

    54 +	for _, c := range candidates {

    55 +		if s.certs[c].Equal(cert) {

    56 +			return true

    57 +		}

    58 +	}

    59 +

    60 +	return false

    61 +}

    62 +

    63  // AddCert adds a certificate to a pool.

    64  func (s *CertPool) AddCert(cert *Certificate) {

    65  	if cert == nil {

    66 diff --git a/src/crypto/x509/root_cgo_darwin.go b/src/crypto/x509/root_cgo_darwin.go

    67 index a4b33c7..d599174 100644

    68 --- a/src/crypto/x509/root_cgo_darwin.go

    69 +++ b/src/crypto/x509/root_cgo_darwin.go

    70 @@ -73,10 +73,11 @@ int useOldCode() {

    71  //

    72  // On success it returns 0 and fills pemRoots with a CFDataRef that contains the extracted root

    73  // certificates of the system. On failure, the function returns -1.

    74 +// Additionally, it fills untrustedPemRoots with certs that must be removed from pemRoots.

    75  //

    76 -// Note: The CFDataRef returned in pemRoots must be released (using CFRelease) after

    77 -// we've consumed its content.

    78 -int FetchPEMRoots(CFDataRef *pemRoots) {

    79 +// Note: The CFDataRef returned in pemRoots and untrustedPemRoots must

    80 +// be released (using CFRelease) after we've consumed its content.

    81 +int FetchPEMRoots(CFDataRef *pemRoots, CFDataRef *untrustedPemRoots) {

    82  	if (useOldCode()) {

    83  		return FetchPEMRoots_MountainLion(pemRoots);

    84  	}

    85 @@ -93,23 +94,69 @@ int FetchPEMRoots(CFDataRef *pemRoots) {

    86  		return -1;

    87  	}

    88  

    89 +	// kSecTrustSettingsResult is defined as CFSTR("kSecTrustSettingsResult"),

    90 +	// but the Go linker's internal linking mode can't handle CFSTR relocations.

    91 +	// Create our own dynamic string instead and release it below.

    92 +	CFStringRef policy = CFStringCreateWithCString(NULL, "kSecTrustSettingsResult", kCFStringEncodingUTF8);

    93 +

    94  	CFMutableDataRef combinedData = CFDataCreateMutable(kCFAllocatorDefault, 0);

    95 +	CFMutableDataRef combinedUntrustedData = CFDataCreateMutable(kCFAllocatorDefault, 0);

    96  	for (int i = 0; i < numDomains; i++) {

    97  		CFArrayRef certs = NULL;

    98 -		// Only get certificates from domain that are trusted

    99  		OSStatus err = SecTrustSettingsCopyCertificates(domains[i], &certs);

   100  		if (err != noErr) {

   101  			continue;

   102  		}

   103  

   104 -		int numCerts = CFArrayGetCount(certs);

   105 +		CFIndex numCerts = CFArrayGetCount(certs);

   106  		for (int j = 0; j < numCerts; j++) {

   107  			CFDataRef data = NULL;

   108  			CFErrorRef errRef = NULL;

   109 +			CFArrayRef trustSettings = NULL;

   110  			SecCertificateRef cert = (SecCertificateRef)CFArrayGetValueAtIndex(certs, j);

   111  			if (cert == NULL) {

   112  				continue;

   113  			}

   114 +			// We only want trusted certs.

   115 +			int untrusted = 0;

   116 +			if (i != 0) {

   117 +				// Certs found in the system domain are always trusted. If the user

   118 +				// configures "Never Trust" on such a cert, it will also be found in the

   119 +				// admin or user domain, causing it to be added to untrustedPemRoots. The

   120 +				// Go code will then clean this up.

   121 +

   122 +				// Trust may be stored in any of the domains. According to Apple's

   123 +				// SecTrustServer.c, "user trust settings overrule admin trust settings",

   124 +				// so take the last trust settings array we find.

   125 +				// Skip the system domain since it is always trusted.

   126 +				for (int k = 1; k < numDomains; k++) {

   127 +					CFArrayRef domainTrustSettings = NULL;

   128 +					err = SecTrustSettingsCopyTrustSettings(cert, domains[k], &domainTrustSettings);

   129 +					if (err == errSecSuccess && domainTrustSettings != NULL) {

   130 +						if (trustSettings) {

   131 +							CFRelease(trustSettings);

   132 +						}

   133 +						trustSettings = domainTrustSettings;

   134 +					}

   135 +				}

   136 +				if (trustSettings == NULL) {

   137 +					// "this certificate must be verified to a known trusted certificate"; aka not a root.

   138 +					continue;

   139 +				}

   140 +				for (CFIndex k = 0; k < CFArrayGetCount(trustSettings); k++) {

   141 +					CFNumberRef cfNum;

   142 +					CFDictionaryRef tSetting = (CFDictionaryRef)CFArrayGetValueAtIndex(trustSettings, k);

   143 +					if (CFDictionaryGetValueIfPresent(tSetting, policy, (const void**)&cfNum)){

   144 +						SInt32 result = 0;

   145 +						CFNumberGetValue(cfNum, kCFNumberSInt32Type, &result);

   146 +						// TODO: The rest of the dictionary specifies conditions for evaluation.

   147 +						if (result == kSecTrustSettingsResultDeny) {

   148 +							untrusted = 1;

   149 +						}

   150 +					}

   151 +				}

   152 +				CFRelease(trustSettings);

   153 +			}

   154  			// We only want to add Root CAs, so make sure Subject and Issuer Name match

   155  			CFDataRef subjectName = SecCertificateCopyNormalizedSubjectContent(cert, &errRef);

   156  			if (errRef != NULL) {

   157 @@ -138,13 +185,16 @@ int FetchPEMRoots(CFDataRef *pemRoots) {

   158  			}

   159  

   160  			if (data != NULL) {

   161 -				CFDataAppendBytes(combinedData, CFDataGetBytePtr(data), CFDataGetLength(data));

   162 +				CFMutableDataRef appendTo = untrusted ? combinedUntrustedData : combinedData;

   163 +				CFDataAppendBytes(appendTo, CFDataGetBytePtr(data), CFDataGetLength(data));

   164  				CFRelease(data);

   165  			}

   166  		}

   167  		CFRelease(certs);

   168  	}

   169 +	CFRelease(policy);

   170  	*pemRoots = combinedData;

   171 +	*untrustedPemRoots = combinedUntrustedData;

   172  	return 0;

   173  }

   174  */

   175 @@ -158,7 +208,8 @@ func loadSystemRoots() (*CertPool, error) {

   176  	roots := NewCertPool()

   177  

   178  	var data C.CFDataRef = nil

   179 -	err := C.FetchPEMRoots(&data)

   180 +	var untrustedData C.CFDataRef = nil

   181 +	err := C.FetchPEMRoots(&data, &untrustedData)

   182  	if err == -1 {

   183  		// TODO: better error message

   184  		return nil, errors.New("crypto/x509: failed to load darwin system roots with cgo")

   185 @@ -167,5 +218,19 @@ func loadSystemRoots() (*CertPool, error) {

   186  	defer C.CFRelease(C.CFTypeRef(data))

   187  	buf := C.GoBytes(unsafe.Pointer(C.CFDataGetBytePtr(data)), C.int(C.CFDataGetLength(data)))

   188  	roots.AppendCertsFromPEM(buf)

   189 -	return roots, nil

   190 +	if untrustedData == nil {

   191 +		return roots, nil

   192 +	}

   193 +	defer C.CFRelease(C.CFTypeRef(untrustedData))

   194 +	buf = C.GoBytes(unsafe.Pointer(C.CFDataGetBytePtr(untrustedData)), C.int(C.CFDataGetLength(untrustedData)))

   195 +	untrustedRoots := NewCertPool()

   196 +	untrustedRoots.AppendCertsFromPEM(buf)

   197 +

   198 +	trustedRoots := NewCertPool()

   199 +	for _, c := range roots.certs {

   200 +		if !untrustedRoots.contains(c) {

   201 +			trustedRoots.AddCert(c)

   202 +		}

   203 +	}

   204 +	return trustedRoots, nil

   205  }

   206 diff --git a/src/crypto/x509/root_darwin.go b/src/crypto/x509/root_darwin.go

   207 index 78de56c..59b303d 100644

   208 --- a/src/crypto/x509/root_darwin.go

   209 +++ b/src/crypto/x509/root_darwin.go

   210 @@ -6,12 +6,27 @@

   211  

   212  package x509

   213  

   214 -import "os/exec"

   215 +import (

   216 +	"bytes"

   217 +	"encoding/pem"

   218 +	"fmt"

   219 +	"io/ioutil"

   220 +	"os"

   221 +	"os/exec"

   222 +	"strconv"

   223 +	"sync"

   224 +	"syscall"

   225 +)

   226  

   227  func (c *Certificate) systemVerify(opts *VerifyOptions) (chains [][]*Certificate, err error) {

   228  	return nil, nil

   229  }

   230  

   231 +// This code is only used when compiling without cgo.

   232 +// It is here, instead of root_nocgo_darwin.go, so that tests can check it

   233 +// even if the tests are run with cgo enabled.

   234 +// The linker will not include these unused functions in binaries built with cgo enabled.

   235 +

   236  func execSecurityRoots() (*CertPool, error) {

   237  	cmd := exec.Command("/usr/bin/security", "find-certificate", "-a", "-p", "/System/Library/Keychains/SystemRootCertificates.keychain")

   238  	data, err := cmd.Output()

   239 @@ -19,7 +34,100 @@ func execSecurityRoots() (*CertPool, error) {

   240  		return nil, err

   241  	}

   242  

   243 -	roots := NewCertPool()

   244 -	roots.AppendCertsFromPEM(data)

   245 +	var (

   246 +		mu    sync.Mutex

   247 +		roots = NewCertPool()

   248 +	)

   249 +	add := func(cert *Certificate) {

   250 +		mu.Lock()

   251 +		defer mu.Unlock()

   252 +		roots.AddCert(cert)

   253 +	}

   254 +	blockCh := make(chan *pem.Block)

   255 +	var wg sync.WaitGroup

   256 +	for i := 0; i < 4; i++ {

   257 +		wg.Add(1)

   258 +		go func() {

   259 +			defer wg.Done()

   260 +			for block := range blockCh {

   261 +				verifyCertWithSystem(block, add)

   262 +			}

   263 +		}()

   264 +	}

   265 +	for len(data) > 0 {

   266 +		var block *pem.Block

   267 +		block, data = pem.Decode(data)

   268 +		if block == nil {

   269 +			break

   270 +		}

   271 +		if block.Type != "CERTIFICATE" || len(block.Headers) != 0 {

   272 +			continue

   273 +		}

   274 +		blockCh <- block

   275 +	}

   276 +	close(blockCh)

   277 +	wg.Wait()

   278  	return roots, nil

   279  }

   280 +

   281 +func verifyCertWithSystem(block *pem.Block, add func(*Certificate)) {

   282 +	data := pem.EncodeToMemory(block)

   283 +	var cmd *exec.Cmd

   284 +	if needsTmpFiles() {

   285 +		f, err := ioutil.TempFile("", "cert")

   286 +		if err != nil {

   287 +			fmt.Fprintf(os.Stderr, "can't create temporary file for cert: %v", err)

   288 +			return

   289 +		}

   290 +		defer os.Remove(f.Name())

   291 +		if _, err := f.Write(data); err != nil {

   292 +			fmt.Fprintf(os.Stderr, "can't write temporary file for cert: %v", err)

   293 +			return

   294 +		}

   295 +		if err := f.Close(); err != nil {

   296 +			fmt.Fprintf(os.Stderr, "can't write temporary file for cert: %v", err)

   297 +			return

   298 +		}

   299 +		cmd = exec.Command("/usr/bin/security", "verify-cert", "-c", f.Name(), "-l")

   300 +	} else {

   301 +		cmd = exec.Command("/usr/bin/security", "verify-cert", "-c", "/dev/stdin", "-l")

   302 +		cmd.Stdin = bytes.NewReader(data)

   303 +	}

   304 +	if cmd.Run() == nil {

   305 +		// Non-zero exit means untrusted

   306 +		cert, err := ParseCertificate(block.Bytes)

   307 +		if err != nil {

   308 +			return

   309 +		}

   310 +

   311 +		add(cert)

   312 +	}

   313 +}

   314 +

   315 +var versionCache struct {

   316 +	sync.Once

   317 +	major int

   318 +}

   319 +

   320 +// needsTmpFiles reports whether the OS is <= 10.11 (which requires real

   321 +// files as arguments to the security command).

   322 +func needsTmpFiles() bool {

   323 +	versionCache.Do(func() {

   324 +		release, err := syscall.Sysctl("kern.osrelease")

   325 +		if err != nil {

   326 +			return

   327 +		}

   328 +		for i, c := range release {

   329 +			if c == '.' {

   330 +				release = release[:i]

   331 +				break

   332 +			}

   333 +		}

   334 +		major, err := strconv.Atoi(release)

   335 +		if err != nil {

   336 +			return

   337 +		}

   338 +		versionCache.major = major

   339 +	})

   340 +	return versionCache.major <= 15

   341 +}

   342 diff --git a/src/crypto/x509/root_darwin_test.go b/src/crypto/x509/root_darwin_test.go

   343 index 8b6b151..c8ca3ea 100644

   344 --- a/src/crypto/x509/root_darwin_test.go

   345 +++ b/src/crypto/x509/root_darwin_test.go

   346 @@ -29,6 +29,7 @@ func TestSystemRoots(t *testing.T) {

   347  		// On Mavericks, there are 212 bundled certs; require only

   348  		// 150 here, since this is just a sanity check, and the

   349  		// exact number will vary over time.

   350 +		t.Logf("got %d roots", len(tt.certs))

   351  		if want, have := 150, len(tt.certs); have < want {

   352  			t.Fatalf("want at least %d system roots, have %d", want, have)

   353  		}

   354 -- 

   355 2.7.4

   356