Skip to content
This repository was archived by the owner on Dec 22, 2022. It is now read-only.

Commit 47e779b

Browse files
hhhjorthhhjort
authored
First pass at phase 1 TCF 2.0 support (prebid#1228)
* First pass at phase 1 TCF 2.0 support * minor fixes * Update go-gdpr library and fix stuff * Fixes for PR comments
1 parent dc403eb commit 47e779b

File tree

7 files changed

+122
-59
lines changed

7 files changed

+122
-59
lines changed

‎gdpr/gdpr.go

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55
"net/http"
66

7+
"github.com/prebid/go-gdpr/vendorlist"
78
"github.com/prebid/prebid-server/config"
89
"github.com/prebid/prebid-server/openrtb_ext"
910
)
@@ -25,6 +26,11 @@ type Permissions interface {
2526
PersonalInfoAllowed(ctx context.Context, bidder openrtb_ext.BidderName, PublisherID string, consent string) (bool, error)
2627
}
2728

29+
const (
30+
tCF1 uint8 = 1
31+
tCF2 uint8 = 2
32+
)
33+
2834
// NewPermissions gets an instance of the Permissions for use elsewhere in the project.
2935
func NewPermissions(ctx context.Context, cfg config.GDPR, vendorIDs map[openrtb_ext.BidderName]uint16, client *http.Client) Permissions {
3036
// If the host doesn't buy into the IAB GDPR consent framework, then save some cycles and let all syncs happen.
@@ -33,9 +39,11 @@ func NewPermissions(ctx context.Context, cfg config.GDPR, vendorIDs map[openrtb_
3339
}
3440

3541
return &permissionsImpl{
36-
cfg: cfg,
37-
vendorIDs: vendorIDs,
38-
fetchVendorList: newVendorListFetcher(ctx, cfg, client, vendorListURLMaker),
42+
cfg: cfg,
43+
vendorIDs: vendorIDs,
44+
fetchVendorList: map[uint8]func(ctx context.Context, id uint16) (vendorlist.VendorList, error){
45+
tCF1: newVendorListFetcher(ctx, cfg, client, vendorListURLMaker, tCF1),
46+
tCF2: newVendorListFetcher(ctx, cfg, client, vendorListURLMaker, tCF2)},
3947
}
4048
}
4149

‎gdpr/impl.go

Lines changed: 20 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,9 @@ package gdpr
33
import (
44
"context"
55

6-
"github.com/prebid/go-gdpr/consentconstants"
6+
"github.com/prebid/go-gdpr/api"
7+
tcf1constants "github.com/prebid/go-gdpr/consentconstants"
8+
consentconstants "github.com/prebid/go-gdpr/consentconstants/tcf2"
79
"github.com/prebid/go-gdpr/vendorconsent"
810
"github.com/prebid/go-gdpr/vendorlist"
911
"github.com/prebid/prebid-server/config"
@@ -18,7 +20,7 @@ import (
1820
type permissionsImpl struct {
1921
cfg config.GDPR
2022
vendorIDs map[openrtb_ext.BidderName]uint16
21-
fetchVendorList func(ctx context.Context, id uint16) (vendorlist.VendorList, error)
23+
fetchVendorList map[uint8]func(ctx context.Context, id uint16) (vendorlist.VendorList, error)
2224
}
2325

2426
func (p *permissionsImpl) HostCookiesAllowed(ctx context.Context, consent string) (bool, error) {
@@ -71,10 +73,10 @@ func (p *permissionsImpl) allowSync(ctx context.Context, vendorID uint16, consen
7173
return false, nil
7274
}
7375

76+
// InfoStorageAccess is the same across TCF 1 and TCF 2
7477
if vendor.Purpose(consentconstants.InfoStorageAccess) && parsedConsent.PurposeAllowed(consentconstants.InfoStorageAccess) && parsedConsent.VendorConsent(vendorID) {
7578
return true, nil
7679
}
77-
7880
return false, nil
7981
}
8082

@@ -93,14 +95,20 @@ func (p *permissionsImpl) allowPI(ctx context.Context, vendorID uint16, consent
9395
return false, nil
9496
}
9597

96-
if (vendor.Purpose(consentconstants.InfoStorageAccess) || vendor.LegitimateInterest(consentconstants.InfoStorageAccess)) && parsedConsent.PurposeAllowed(consentconstants.InfoStorageAccess) && (vendor.Purpose(consentconstants.AdSelectionDeliveryReporting) || vendor.LegitimateInterest(consentconstants.AdSelectionDeliveryReporting)) && parsedConsent.PurposeAllowed(consentconstants.AdSelectionDeliveryReporting) && parsedConsent.VendorConsent(vendorID) {
97-
return true, nil
98+
if parsedConsent.Version() == 2 {
99+
// Need to add the location special purpose once the library supports it.
100+
if (vendor.Purpose(consentconstants.InfoStorageAccess) || vendor.LegitimateInterest(consentconstants.InfoStorageAccess)) && parsedConsent.PurposeAllowed(consentconstants.InfoStorageAccess) && (vendor.Purpose(consentconstants.PersonalizationProfile) || vendor.LegitimateInterest(consentconstants.PersonalizationProfile)) && parsedConsent.PurposeAllowed(consentconstants.PersonalizationProfile) && parsedConsent.VendorConsent(vendorID) {
101+
return true, nil
102+
}
103+
} else {
104+
if (vendor.Purpose(tcf1constants.InfoStorageAccess) || vendor.LegitimateInterest(tcf1constants.InfoStorageAccess)) && parsedConsent.PurposeAllowed(tcf1constants.InfoStorageAccess) && (vendor.Purpose(tcf1constants.AdSelectionDeliveryReporting) || vendor.LegitimateInterest(tcf1constants.AdSelectionDeliveryReporting)) && parsedConsent.PurposeAllowed(tcf1constants.AdSelectionDeliveryReporting) && parsedConsent.VendorConsent(vendorID) {
105+
return true, nil
106+
}
98107
}
99-
100108
return false, nil
101109
}
102110

103-
func (p *permissionsImpl) parseVendor(ctx context.Context, vendorID uint16, consent string) (parsedConsent vendorconsent.VendorConsents, vendor vendorlist.Vendor, err error) {
111+
func (p *permissionsImpl) parseVendor(ctx context.Context, vendorID uint16, consent string) (parsedConsent api.VendorConsents, vendor api.Vendor, err error) {
104112
parsedConsent, err = vendorconsent.ParseString(consent)
105113
if err != nil {
106114
err = &ErrorMalformedConsent{
@@ -110,7 +118,11 @@ func (p *permissionsImpl) parseVendor(ctx context.Context, vendorID uint16, cons
110118
return
111119
}
112120

113-
vendorList, err := p.fetchVendorList(ctx, parsedConsent.VendorListVersion())
121+
version := parsedConsent.Version()
122+
if version < 1 || version > 2 {
123+
return
124+
}
125+
vendorList, err := p.fetchVendorList[version](ctx, parsedConsent.VendorListVersion())
114126
if err != nil {
115127
return
116128
}

‎gdpr/impl_test.go

Lines changed: 46 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,11 @@ func TestNoConsentButAllowByDefault(t *testing.T) {
1818
HostVendorID: 3,
1919
UsersyncIfAmbiguous: true,
2020
},
21-
vendorIDs: nil,
22-
fetchVendorList: failedListFetcher,
21+
vendorIDs: nil,
22+
fetchVendorList: map[uint8]func(ctx context.Context, id uint16) (vendorlist.VendorList, error){
23+
tCF1: failedListFetcher,
24+
tCF2: failedListFetcher,
25+
},
2326
}
2427
allowSync, err := perms.BidderSyncAllowed(context.Background(), openrtb_ext.BidderAppnexus, "")
2528
assertBoolsEqual(t, true, allowSync)
@@ -35,8 +38,11 @@ func TestNoConsentAndRejectByDefault(t *testing.T) {
3538
HostVendorID: 3,
3639
UsersyncIfAmbiguous: false,
3740
},
38-
vendorIDs: nil,
39-
fetchVendorList: failedListFetcher,
41+
vendorIDs: nil,
42+
fetchVendorList: map[uint8]func(ctx context.Context, id uint16) (vendorlist.VendorList, error){
43+
tCF1: failedListFetcher,
44+
tCF2: failedListFetcher,
45+
},
4046
}
4147
allowSync, err := perms.BidderSyncAllowed(context.Background(), openrtb_ext.BidderAppnexus, "")
4248
assertBoolsEqual(t, false, allowSync)
@@ -63,9 +69,14 @@ func TestAllowedSyncs(t *testing.T) {
6369
openrtb_ext.BidderAppnexus: 2,
6470
openrtb_ext.BidderPubmatic: 3,
6571
},
66-
fetchVendorList: listFetcher(map[uint16]vendorlist.VendorList{
67-
1: parseVendorListData(t, vendorListData),
68-
}),
72+
fetchVendorList: map[uint8]func(ctx context.Context, id uint16) (vendorlist.VendorList, error){
73+
tCF1: listFetcher(map[uint16]vendorlist.VendorList{
74+
1: parseVendorListData(t, vendorListData),
75+
}),
76+
tCF2: listFetcher(map[uint16]vendorlist.VendorList{
77+
1: parseVendorListData(t, vendorListData),
78+
}),
79+
},
6980
}
7081

7182
allowSync, err := perms.HostCookiesAllowed(context.Background(), "BON3PCUON3PCUABABBAAABoAAAAAMw")
@@ -94,9 +105,14 @@ func TestProhibitedPurposes(t *testing.T) {
94105
openrtb_ext.BidderAppnexus: 2,
95106
openrtb_ext.BidderPubmatic: 3,
96107
},
97-
fetchVendorList: listFetcher(map[uint16]vendorlist.VendorList{
98-
1: parseVendorListData(t, vendorListData),
99-
}),
108+
fetchVendorList: map[uint8]func(ctx context.Context, id uint16) (vendorlist.VendorList, error){
109+
tCF1: listFetcher(map[uint16]vendorlist.VendorList{
110+
1: parseVendorListData(t, vendorListData),
111+
}),
112+
tCF2: listFetcher(map[uint16]vendorlist.VendorList{
113+
1: parseVendorListData(t, vendorListData),
114+
}),
115+
},
100116
}
101117

102118
allowSync, err := perms.HostCookiesAllowed(context.Background(), "BON3PCUON3PCUABABBAAABAAAAAAMw")
@@ -125,9 +141,14 @@ func TestProhibitedVendors(t *testing.T) {
125141
openrtb_ext.BidderAppnexus: 2,
126142
openrtb_ext.BidderPubmatic: 3,
127143
},
128-
fetchVendorList: listFetcher(map[uint16]vendorlist.VendorList{
129-
1: parseVendorListData(t, vendorListData),
130-
}),
144+
fetchVendorList: map[uint8]func(ctx context.Context, id uint16) (vendorlist.VendorList, error){
145+
tCF1: listFetcher(map[uint16]vendorlist.VendorList{
146+
1: parseVendorListData(t, vendorListData),
147+
}),
148+
tCF2: listFetcher(map[uint16]vendorlist.VendorList{
149+
1: parseVendorListData(t, vendorListData),
150+
}),
151+
},
131152
}
132153

133154
allowSync, err := perms.HostCookiesAllowed(context.Background(), "BOS2bx5OS2bx5ABABBAAABoAAAAAFA")
@@ -144,7 +165,10 @@ func TestMalformedConsent(t *testing.T) {
144165
cfg: config.GDPR{
145166
HostVendorID: 2,
146167
},
147-
fetchVendorList: listFetcher(nil),
168+
fetchVendorList: map[uint8]func(ctx context.Context, id uint16) (vendorlist.VendorList, error){
169+
tCF1: listFetcher(nil),
170+
tCF2: listFetcher(nil),
171+
},
148172
}
149173

150174
sync, err := perms.HostCookiesAllowed(context.Background(), "BON")
@@ -169,9 +193,14 @@ func TestAllowPersonalInfo(t *testing.T) {
169193
openrtb_ext.BidderAppnexus: 2,
170194
openrtb_ext.BidderPubmatic: 3,
171195
},
172-
fetchVendorList: listFetcher(map[uint16]vendorlist.VendorList{
173-
1: parseVendorListData(t, vendorListData),
174-
}),
196+
fetchVendorList: map[uint8]func(ctx context.Context, id uint16) (vendorlist.VendorList, error){
197+
tCF1: listFetcher(map[uint16]vendorlist.VendorList{
198+
1: parseVendorListData(t, vendorListData),
199+
}),
200+
tCF2: listFetcher(map[uint16]vendorlist.VendorList{
201+
1: parseVendorListData(t, vendorListData),
202+
}),
203+
},
175204
}
176205

177206
// PI needs both purposes to succeed

‎gdpr/vendorlist-fetching.go

Lines changed: 29 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -11,35 +11,37 @@ import (
1111
"time"
1212

1313
"github.com/golang/glog"
14+
"github.com/prebid/go-gdpr/api"
1415
"github.com/prebid/go-gdpr/vendorlist"
16+
"github.com/prebid/go-gdpr/vendorlist2"
1517
"github.com/prebid/prebid-server/config"
1618
"golang.org/x/net/context/ctxhttp"
1719
)
1820

19-
type saveVendors func(uint16, vendorlist.VendorList)
21+
type saveVendors func(uint16, api.VendorList)
2022

2123
// This file provides the vendorlist-fetching function for Prebid Server.
2224
//
2325
// For more info, see https://github.com/prebid/prebid-server/issues/504
2426
//
2527
// Nothing in this file is exported. Public APIs can be found in gdpr.go
2628

27-
func newVendorListFetcher(initCtx context.Context, cfg config.GDPR, client *http.Client, urlMaker func(uint16) string) func(ctx context.Context, id uint16) (vendorlist.VendorList, error) {
29+
func newVendorListFetcher(initCtx context.Context, cfg config.GDPR, client *http.Client, urlMaker func(uint16, uint8) string, TCFVer uint8) func(ctx context.Context, id uint16) (vendorlist.VendorList, error) {
2830
// These save and load functions can be used to store & retrieve lists from our cache.
2931
save, load := newVendorListCache()
3032

3133
withTimeout, cancel := context.WithTimeout(initCtx, cfg.Timeouts.InitTimeout())
3234
defer cancel()
33-
populateCache(withTimeout, client, urlMaker, save)
35+
populateCache(withTimeout, client, urlMaker, save, TCFVer)
3436

35-
saveOneSometimes := newOccasionalSaver(cfg.Timeouts.ActiveTimeout())
37+
saveOneSometimes := newOccasionalSaver(cfg.Timeouts.ActiveTimeout(), TCFVer)
3638

3739
return func(ctx context.Context, id uint16) (vendorlist.VendorList, error) {
3840
list := load(id)
3941
if list != nil {
4042
return list, nil
4143
}
42-
saveOneSometimes(ctx, client, urlMaker(id), save)
44+
saveOneSometimes(ctx, client, urlMaker(id, TCFVer), save)
4345
list = load(id)
4446
if list != nil {
4547
return list, nil
@@ -49,17 +51,23 @@ func newVendorListFetcher(initCtx context.Context, cfg config.GDPR, client *http
4951
}
5052

5153
// populateCache saves all the known versions of the vendor list for future use.
52-
func populateCache(ctx context.Context, client *http.Client, urlMaker func(uint16) string, saver saveVendors) {
53-
latestVersion := saveOne(ctx, client, urlMaker(0), saver)
54+
func populateCache(ctx context.Context, client *http.Client, urlMaker func(uint16, uint8) string, saver saveVendors, TCFVer uint8) {
55+
latestVersion := saveOne(ctx, client, urlMaker(0, TCFVer), saver, TCFVer)
5456

5557
for i := uint16(1); i < latestVersion; i++ {
56-
saveOne(ctx, client, urlMaker(i), saver)
58+
saveOne(ctx, client, urlMaker(i, TCFVer), saver, TCFVer)
5759
}
5860
}
5961

6062
// Make a URL which can be used to fetch a given version of the Global Vendor List. If the version is 0,
6163
// this will fetch the latest version.
62-
func vendorListURLMaker(version uint16) string {
64+
func vendorListURLMaker(version uint16, TCFVer uint8) string {
65+
if TCFVer == 2 {
66+
if version == 0 {
67+
return "https://vendorlist.consensu.org/v2/vendor-list.json"
68+
}
69+
return "https://vendorlist.consensu.org/v2/archives/vendor-list-v" + strconv.Itoa(int(version)) + ".json"
70+
}
6371
if version == 0 {
6472
return "https://vendorlist.consensu.org/vendorlist.json"
6573
}
@@ -71,7 +79,7 @@ func vendorListURLMaker(version uint16) string {
7179
// The goal here is to update quickly when new versions of the VendorList are released, but not wreck
7280
// server performance if a bad CMP starts sending us malformed consent strings that advertize a version
7381
// that doesn't exist yet.
74-
func newOccasionalSaver(timeout time.Duration) func(ctx context.Context, client *http.Client, url string, saver saveVendors) {
82+
func newOccasionalSaver(timeout time.Duration, TCFVer uint8) func(ctx context.Context, client *http.Client, url string, saver saveVendors) {
7583
lastSaved := &atomic.Value{}
7684
lastSaved.Store(time.Time{})
7785

@@ -80,13 +88,13 @@ func newOccasionalSaver(timeout time.Duration) func(ctx context.Context, client
8088
if now.Sub(lastSaved.Load().(time.Time)).Minutes() > 10 {
8189
withTimeout, cancel := context.WithTimeout(ctx, timeout)
8290
defer cancel()
83-
saveOne(withTimeout, client, url, saver)
91+
saveOne(withTimeout, client, url, saver, TCFVer)
8492
lastSaved.Store(now)
8593
}
8694
}
8795
}
8896

89-
func saveOne(ctx context.Context, client *http.Client, url string, saver saveVendors) uint16 {
97+
func saveOne(ctx context.Context, client *http.Client, url string, saver saveVendors, cTFVer uint8) uint16 {
9098
req, err := http.NewRequest("GET", url, nil)
9199
if err != nil {
92100
glog.Errorf("Failed to build GET %s request. Cookie syncs may be affected: %v", url, err)
@@ -109,8 +117,12 @@ func saveOne(ctx context.Context, client *http.Client, url string, saver saveVen
109117
glog.Errorf("GET %s returned %d. Cookie syncs may be affected.", url, resp.StatusCode)
110118
return 0
111119
}
112-
113-
newList, err := vendorlist.ParseEagerly(respBody)
120+
var newList api.VendorList
121+
if cTFVer == 2 {
122+
newList, err = vendorlist2.ParseEagerly(respBody)
123+
} else {
124+
newList, err = vendorlist.ParseEagerly(respBody)
125+
}
114126
if err != nil {
115127
glog.Errorf("GET %s returned malformed JSON. Cookie syncs may be affected. Error was %v. Body was %s", url, err, string(respBody))
116128
return 0
@@ -120,13 +132,13 @@ func saveOne(ctx context.Context, client *http.Client, url string, saver saveVen
120132
return newList.Version()
121133
}
122134

123-
func newVendorListCache() (save func(id uint16, list vendorlist.VendorList), load func(id uint16) vendorlist.VendorList) {
135+
func newVendorListCache() (save func(id uint16, list api.VendorList), load func(id uint16) api.VendorList) {
124136
cache := &sync.Map{}
125137

126-
save = func(id uint16, list vendorlist.VendorList) {
138+
save = func(id uint16, list api.VendorList) {
127139
cache.Store(id, list)
128140
}
129-
load = func(id uint16) vendorlist.VendorList {
141+
load = func(id uint16) api.VendorList {
130142
list, ok := cache.Load(id)
131143
if ok {
132144
return list.(vendorlist.VendorList)

0 commit comments

Comments
 (0)