Don't Load GVL v1 for TCF2 (+ TCF1 Cleanup) (#1693)

Author: SyntaxNode

gdpr/gdpr.go

@@ -44,8 +44,8 @@ func NewPermissions(ctx context.Context, cfg config.GDPR, vendorIDs map[openrtb_
 		cfg:       cfg,
 		vendorIDs: vendorIDs,
 		fetchVendorList: map[uint8]func(ctx context.Context, id uint16) (vendorlist.VendorList, error){
-			tcf1SpecVersion: newVendorListFetcher(ctx, cfg, client, vendorListURLMaker, tcf1SpecVersion),
-			tcf2SpecVersion: newVendorListFetcher(ctx, cfg, client, vendorListURLMaker, tcf2SpecVersion)},
+			tcf1SpecVersion: newVendorListFetcherTCF1(cfg),
+			tcf2SpecVersion: newVendorListFetcherTCF2(ctx, cfg, client, vendorListURLMaker)},
 	}
 
 	if cfg.HostVendorID == 0 {

gdpr/vendorlist-fetching.go

@@ -26,30 +26,40 @@ type saveVendors func(uint16, api.VendorList)
 //
 // Nothing in this file is exported. Public APIs can be found in gdpr.go
 
-func newVendorListFetcher(initCtx context.Context, cfg config.GDPR, client *http.Client, urlMaker func(uint16, uint8) string, tcfSpecVersion uint8) func(ctx context.Context, id uint16) (vendorlist.VendorList, error) {
-	var fallback api.VendorList
-
-	if tcfSpecVersion == tcf1SpecVersion && len(cfg.TCF1.FallbackGVLPath) == 0 {
-		return func(ctx context.Context, vendorListVersion uint16) (vendorlist.VendorList, error) {
+func newVendorListFetcherTCF1(cfg config.GDPR) func(ctx context.Context, id uint16) (vendorlist.VendorList, error) {
+	if len(cfg.TCF1.FallbackGVLPath) == 0 {
+		return func(_ context.Context, vendorListVersion uint16) (vendorlist.VendorList, error) {
 			return nil, makeVendorListNotFoundError(vendorListVersion)
 		}
 	}
 
-	if tcfSpecVersion == tcf1SpecVersion {
-		fallback = loadFallbackGVL(cfg.TCF1.FallbackGVLPath)
+	fallback := loadFallbackGVLForTCF1(cfg.TCF1.FallbackGVLPath)
+	return func(_ context.Context, _ uint16) (vendorlist.VendorList, error) {
+		return fallback, nil
+	}
+}
 
-		return func(ctx context.Context, vendorListVersion uint16) (vendorlist.VendorList, error) {
-			return fallback, nil
-		}
+func loadFallbackGVLForTCF1(fallbackGVLPath string) vendorlist.VendorList {
+	fallbackContents, err := ioutil.ReadFile(fallbackGVLPath)
+	if err != nil {
+		glog.Fatalf("Error reading from file %s: %v", fallbackGVLPath, err)
+	}
+
+	fallback, err := vendorlist.ParseEagerly(fallbackContents)
+	if err != nil {
+		glog.Fatalf("Error processing default GVL from %s: %v", fallbackGVLPath, err)
 	}
+	return fallback
+}
 
-	cacheSave, cacheLoad := newVendorListCache(fallback)
+func newVendorListFetcherTCF2(initCtx context.Context, cfg config.GDPR, client *http.Client, urlMaker func(uint16) string) func(ctx context.Context, id uint16) (vendorlist.VendorList, error) {
+	cacheSave, cacheLoad := newVendorListCache()
 
 	preloadContext, cancel := context.WithTimeout(initCtx, cfg.Timeouts.InitTimeout())
 	defer cancel()
-	preloadCache(preloadContext, client, urlMaker, cacheSave, tcfSpecVersion)
+	preloadCache(preloadContext, client, urlMaker, cacheSave)
 
-	saveOneRateLimited := newOccasionalSaver(cfg.Timeouts.ActiveTimeout(), tcfSpecVersion)
+	saveOneRateLimited := newOccasionalSaver(cfg.Timeouts.ActiveTimeout())
 	return func(ctx context.Context, vendorListVersion uint16) (vendorlist.VendorList, error) {
 		// Attempt To Load From Cache
 		if list := cacheLoad(vendorListVersion); list != nil {
@@ -58,19 +68,14 @@ func newVendorListFetcher(initCtx context.Context, cfg config.GDPR, client *http
 
 		// Attempt To Download
 		// - May not add to cache immediately.
-		saveOneRateLimited(ctx, client, urlMaker(vendorListVersion, tcfSpecVersion), cacheSave)
+		saveOneRateLimited(ctx, client, urlMaker(vendorListVersion), cacheSave)
 
 		// Attempt To Load From Cache Again
 		// - May have been added by the call to saveOneRateLimited.
 		if list := cacheLoad(vendorListVersion); list != nil {
 			return list, nil
 		}
 
-		// Attempt To Use Hardcoded Fallback
-		if fallback != nil {
-			return fallback, nil
-		}
-
 		// Give Up
 		return nil, makeVendorListNotFoundError(vendorListVersion)
 	}
@@ -81,35 +86,32 @@ func makeVendorListNotFoundError(vendorListVersion uint16) error {
 }
 
 // preloadCache saves all the known versions of the vendor list for future use.
-func preloadCache(ctx context.Context, client *http.Client, urlMaker func(uint16, uint8) string, saver saveVendors, tcfSpecVersion uint8) {
-	latestVersion := saveOne(ctx, client, urlMaker(0, tcfSpecVersion), saver, tcfSpecVersion)
+func preloadCache(ctx context.Context, client *http.Client, urlMaker func(uint16) string, saver saveVendors) {
+	latestVersion := saveOne(ctx, client, urlMaker(0), saver)
 
-	for i := uint16(1); i < latestVersion; i++ {
-		saveOne(ctx, client, urlMaker(i, tcfSpecVersion), saver, tcfSpecVersion)
+	// The GVL for TCF2 has no vendors defined in its first version. It's very unlikely to be used, so don't preload it.
+	firstVersionToLoad := uint16(2)
+
+	for i := firstVersionToLoad; i < latestVersion; i++ {
+		saveOne(ctx, client, urlMaker(i), saver)
 	}
 }
 
 // Make a URL which can be used to fetch a given version of the Global Vendor List. If the version is 0,
 // this will fetch the latest version.
-func vendorListURLMaker(vendorListVersion uint16, tcfSpecVersion uint8) string {
-	if tcfSpecVersion == tcf2SpecVersion {
-		if vendorListVersion == 0 {
-			return "https://vendor-list.consensu.org/v2/vendor-list.json"
-		}
-		return "https://vendor-list.consensu.org/v2/archives/vendor-list-v" + strconv.Itoa(int(vendorListVersion)) + ".json"
-	}
+func vendorListURLMaker(vendorListVersion uint16) string {
 	if vendorListVersion == 0 {
-		return "https://vendor-list.consensu.org/vendorlist.json"
+		return "https://vendor-list.consensu.org/v2/vendor-list.json"
 	}
-	return "https://vendor-list.consensu.org/v-" + strconv.Itoa(int(vendorListVersion)) + "/vendorlist.json"
+	return "https://vendor-list.consensu.org/v2/archives/vendor-list-v" + strconv.Itoa(int(vendorListVersion)) + ".json"
 }
 
 // newOccasionalSaver returns a wrapped version of saveOne() which only activates every few minutes.
 //
 // The goal here is to update quickly when new versions of the VendorList are released, but not wreck
 // server performance if a bad CMP starts sending us malformed consent strings that advertize a version
 // that doesn't exist yet.
-func newOccasionalSaver(timeout time.Duration, tcfSpecVersion uint8) func(ctx context.Context, client *http.Client, url string, saver saveVendors) {
+func newOccasionalSaver(timeout time.Duration) func(ctx context.Context, client *http.Client, url string, saver saveVendors) {
 	lastSaved := &atomic.Value{}
 	lastSaved.Store(time.Time{})
 
@@ -120,13 +122,13 @@ func newOccasionalSaver(timeout time.Duration, tcfSpecVersion uint8) func(ctx co
 		if timeSinceLastSave.Minutes() > 10 {
 			withTimeout, cancel := context.WithTimeout(ctx, timeout)
 			defer cancel()
-			saveOne(withTimeout, client, url, saver, tcfSpecVersion)
+			saveOne(withTimeout, client, url, saver)
 			lastSaved.Store(now)
 		}
 	}
 }
 
-func saveOne(ctx context.Context, client *http.Client, url string, saver saveVendors, tcfSpecVersion uint8) uint16 {
+func saveOne(ctx context.Context, client *http.Client, url string, saver saveVendors) uint16 {
 	req, err := http.NewRequest("GET", url, nil)
 	if err != nil {
 		glog.Errorf("Failed to build GET %s request. Cookie syncs may be affected: %v", url, err)
@@ -150,11 +152,7 @@ func saveOne(ctx context.Context, client *http.Client, url string, saver saveVen
 		return 0
 	}
 	var newList api.VendorList
-	if tcfSpecVersion == tcf2SpecVersion {
-		newList, err = vendorlist2.ParseEagerly(respBody)
-	} else {
-		newList, err = vendorlist.ParseEagerly(respBody)
-	}
+	newList, err = vendorlist2.ParseEagerly(respBody)
 	if err != nil {
 		glog.Errorf("GET %s returned malformed JSON. Cookie syncs may be affected. Error was %v. Body was %s", url, err, string(respBody))
 		return 0
@@ -164,7 +162,7 @@ func saveOne(ctx context.Context, client *http.Client, url string, saver saveVen
 	return newList.Version()
 }
 
-func newVendorListCache(fallbackVL api.VendorList) (save func(vendorListVersion uint16, list api.VendorList), load func(vendorListVersion uint16) api.VendorList) {
+func newVendorListCache() (save func(vendorListVersion uint16, list api.VendorList), load func(vendorListVersion uint16) api.VendorList) {
 	cache := &sync.Map{}
 
 	save = func(vendorListVersion uint16, list api.VendorList) {
@@ -180,16 +178,3 @@ func newVendorListCache(fallbackVL api.VendorList) (save func(vendorListVersion
 	}
 	return
 }
-
-func loadFallbackGVL(fallbackGVLPath string) vendorlist.VendorList {
-	fallbackContents, err := ioutil.ReadFile(fallbackGVLPath)
-	if err != nil {
-		glog.Fatalf("Error reading from file %s: %v", fallbackGVLPath, err)
-	}
-
-	fallback, err := vendorlist.ParseEagerly(fallbackContents)
-	if err != nil {
-		glog.Fatalf("Error processing default GVL from %s: %v", fallbackGVLPath, err)
-	}
-	return fallback
-}