formatting

Author: max-ostapenko

sql/2024/privacy/most_common_client_hints.sql

@@ -35,9 +35,8 @@ WITH response_headers AS (
 SELECT
   client,
   IF(header_name = 'accept-ch', header_value, tag_value) AS value,
-  COUNT(DISTINCT page) AS number_of_websites,
-  ANY_VALUE(total_websites) AS total_websites,
-  COUNT(DISTINCT page) / ANY_VALUE(total_websites) AS pct_websites
+  COUNT(DISTINCT page) / ANY_VALUE(total_websites) AS pct_pages,
+  COUNT(DISTINCT page) AS number_of_pages
 FROM response_headers
 FULL OUTER JOIN meta_tags
 USING (client, page)
@@ -47,6 +46,5 @@ WHERE
 GROUP BY
   client,
   value
-ORDER BY
-  number_of_websites DESC
+ORDER BY pct_pages DESC
 LIMIT 200

sql/2024/privacy/most_common_cmps_for_iab_tcf_v2.sql

@@ -1,41 +1,26 @@
-#standardSQL
 # Counts of CMPs using IAB Transparency & Consent Framework
 # cf. https://github.com/InteractiveAdvertisingBureau/GDPR-Transparency-and-Consent-Framework/blob/master/TCFv2/IAB%20Tech%20Lab%20-%20CMP%20API%20v2.md#tcdata
 
-WITH totals AS (
+WITH cmps AS (
   SELECT
     client,
-    COUNT(0) AS total_websites
+    page,
+    JSON_VALUE(custom_metrics, '$.privacy.iab_tcf_v2.data.cmpId') AS cmpId,
+    COUNT(DISTINCT page) OVER (PARTITION BY client) AS total_pages
   FROM `httparchive.all.pages`
   WHERE
     date = '2024-06-01' AND
     is_root_page = TRUE
-  GROUP BY client
-), cmps AS (
-  SELECT
-    client,
-    JSON_VALUE(custom_metrics, '$.privacy.iab_tcf_v2.data.cmpId') AS cmpId
-  FROM `httparchive.all.pages`
-  WHERE
-    date = '2024-06-01' AND
-    is_root_page = TRUE AND
-    JSON_VALUE(custom_metrics, '$.privacy.iab_tcf_v2.data.cmpId') IS NOT NULL
 )
 
 SELECT
   client,
   cmpId,
-  COUNT(0) AS number_of_websites,
-  total_websites,
-  COUNT(0) / total_websites AS pct_websites
+  COUNT(0) / ANY_VALUE(total_pages) AS pct_pages,
+  COUNT(0) AS number_of_pages
 FROM cmps
-JOIN totals
-USING (client)
 GROUP BY
   client,
-  total_websites,
   cmpId
 ORDER BY
-  pct_websites DESC,
-  client,
-  cmpId
+  pct_pages DESC

sql/2024/privacy/most_common_cname_domains.sql

@@ -35,8 +35,8 @@ SELECT
   client,
   NET.REG_DOMAIN(cnames.cname) AS cname,
   NET.REG_DOMAIN(adguard_trackers.domain) AS adguard_cname,
-  COUNT(DISTINCT NET.REG_DOMAIN(cnames.origin)) AS request_domain_count,
-  COUNT(DISTINCT page) AS page_count
+  COUNT(DISTINCT NET.REG_DOMAIN(cnames.origin)) AS number_of_request_domains,
+  COUNT(DISTINCT page) AS number_of_pages
 --ARRAY_AGG(DISTINCT cnames.origin LIMIT 2) AS request_domain_examples,
 --ARRAY_AGG(DISTINCT page LIMIT 2) AS page_examples,
 FROM cnames
@@ -49,5 +49,5 @@ GROUP BY
 HAVING request_domain_count > 100
 ORDER BY
   client,
-  request_domain_count DESC
+  number_of_request_domains DESC
 LIMIT 500

sql/2024/privacy/most_common_countries_for_iab_tcf_v2.sql

@@ -1,4 +1,3 @@
-#standardSQL
 # Counts of countries for publishers using IAB Transparency & Consent Framework
 # cf. https://github.com/InteractiveAdvertisingBureau/GDPR-Transparency-and-Consent-Framework/blob/master/TCFv2/IAB%20Tech%20Lab%20-%20CMP%20API%20v2.md#tcdata
 # "Country code of the country that determines the legislation of
@@ -28,16 +27,14 @@ WITH totals AS (
 SELECT
   client,
   publisherCC,
-  COUNT(0) AS number_of_websites,
-  total_websites,
-  COUNT(0) / total_websites AS pct_websites
+  COUNT(0) / ANY_VALUE(total_websites) AS pct_pages,
+  COUNT(0) AS number_of_pages
 FROM cmps
 JOIN totals
 USING (client)
 GROUP BY
   client,
-  total_websites,
   publisherCC
 ORDER BY
   client,
-  number_of_websites DESC
+  pct_pages DESC

sql/2024/privacy/most_common_strings_for_iab_usp.sql

@@ -1,39 +1,27 @@
-#standardSQL
 # Counts of US Privacy String values for websites using IAB US Privacy Framework
 # cf. https://github.com/InteractiveAdvertisingBureau/USPrivacy/blob/master/CCPA/US%20Privacy%20String.md
 
-WITH totals AS (
-  SELECT
-    client,
-    COUNT(DISTINCT page) AS pages_total
-  FROM `httparchive.all.pages`
-  WHERE
-    date = '2024-06-01'
-  GROUP BY client
-), usp_data AS (
+WITH usp_data AS (
   SELECT
     client,
     page,
-    JSON_VALUE(custom_metrics, '$.privacy.iab_usp.privacy_string.uspString') AS uspString
+    JSON_VALUE(custom_metrics, '$.privacy.iab_usp.privacy_string.uspString') AS uspString,
+    COUNT(DISTINCT page) OVER (PARTITION BY client) AS pages_total
   FROM `httparchive.all.pages`
   WHERE
     date = '2024-06-01' AND
-    JSON_VALUE(custom_metrics, '$.privacy.iab_usp.privacy_string.uspString') IS NOT NULL
+    is_root_page = TRUE
 )
 
 SELECT
   client,
   uspString,
-  COUNT(DISTINCT page) AS pages_with_usp,
-  ANY_VALUE(pages_total) AS pages_total,
-  COUNT(DISTINCT page) / ANY_VALUE(pages_total) AS pages_pct
+  COUNT(DISTINCT page) / ANY_VALUE(pages_total) AS pct_pages,
+  COUNT(DISTINCT page) AS number_of_pages
 FROM usp_data
-JOIN totals
-USING (client)
 GROUP BY
   client,
   uspString
 ORDER BY
-  client,
-  pages_pct DESC
+  pct_pages DESC
 LIMIT 100

sql/2024/privacy/number_of_websites_per_technology.sql

@@ -16,10 +16,9 @@ WITH technologies AS (
 SELECT
   client,
   technology,
-  COUNT(DISTINCT page) / ANY_VALUE(total_websites) AS percent_of_websites,
-  COUNT(DISTINCT page) AS number_of_websites,
-  ARRAY_AGG(DISTINCT category) AS categories,
-  ANY_VALUE(total_websites) AS total_websites
+  COUNT(DISTINCT page) / ANY_VALUE(total_websites) AS pct_pages,
+  COUNT(DISTINCT page) AS number_of_pages,
+  ARRAY_AGG(DISTINCT category) AS categories
 FROM technologies
 WHERE
   category IN (
@@ -32,5 +31,4 @@ GROUP BY
   client,
   technology
 ORDER BY
-  client,
-  number_of_websites DESC
+  pct_pages DESC

sql/2024/privacy/number_of_websites_per_technology_category.sql

@@ -1,9 +1,8 @@
 SELECT
   client,
   category,
-  COUNT(DISTINCT IF(category = tech_category, page, NULL)) / COUNT(DISTINCT page) AS pct_websites_in_category,
-  COUNT(DISTINCT IF(category = tech_category, page, NULL)) AS number_of_websites_in_category,
-  COUNT(DISTINCT page) AS total_pages
+  COUNT(DISTINCT IF(category = tech_category, page, NULL)) / COUNT(DISTINCT page) AS pct_pages,
+  COUNT(DISTINCT IF(category = tech_category, page, NULL)) AS number_of_pages
 FROM `httparchive.all.pages`,
   UNNEST(technologies) AS tech,
   UNNEST(categories) AS tech_category,
@@ -20,5 +19,4 @@ GROUP BY
   client,
   category
 ORDER BY
-  client,
-  number_of_websites_in_category DESC
+  pct_pages DESC

sql/2024/privacy/number_of_websites_with_bounce_tracking.sql

@@ -72,9 +72,9 @@ WITH redirect_requests AS (
 -- Count the number of websites with bounce tracking per bounce hostname
 SELECT
   NET.HOST(navigation_redirect_location) AS bounce_hostname,
-  COUNT(DISTINCT page) AS pages_count
+  COUNT(DISTINCT page) AS number_of_pages
 --ARRAY_AGG(page LIMIT 2) AS page_examples
 FROM bounce_sequences
 GROUP BY bounce_hostname
-ORDER BY pages_count DESC
+ORDER BY number_of_pages DESC
 LIMIT 100

sql/2024/privacy/number_of_websites_with_client_hints.sql

@@ -30,13 +30,13 @@ WITH response_headers AS (
 
 SELECT
   client,
-  ANY_VALUE(total_websites) AS total_websites,
-  COUNT(DISTINCT page) / ANY_VALUE(total_websites) AS pct_websites
+  COUNT(DISTINCT page) / ANY_VALUE(total_websites) AS pct_pages,
+  COUNT(DISTINCT page) AS number_of_pages
 FROM response_headers
 FULL OUTER JOIN meta_tags
 USING (client, page)
 WHERE
   header_name = 'accept-ch' OR
   tag_name = 'accept-ch'
 GROUP BY client
-ORDER BY client
+ORDER BY pct_pages DESC

sql/2024/privacy/number_of_websites_with_dnt.sql

@@ -1,4 +1,3 @@
-#standardSQL
 # Pages that request DNT status
 
 WITH blink AS (
@@ -26,10 +25,10 @@ pages AS (
 
 SELECT
   COALESCE(blink.client, pages.client) AS client,
-  blink.num_urls AS number_of_websites_usage_per_blink,
-  blink.pct_urls AS percentage_of_websites_usage_per_blink,
-  pages.num_urls AS number_of_websites_usage_per_custom_metric,
-  pages.pct_urls AS percentage_of_websites_usage_per_custom_metric
+  blink.num_urls AS number_of_pages_usage_per_blink,
+  blink.pct_urls AS pct_of_websites_usage_per_blink,
+  pages.num_urls AS number_of_pages_usage_per_custom_metric,
+  pages.pct_urls AS pct_of_websites_usage_per_custom_metric
 FROM blink
 FULL OUTER JOIN pages
 ON blink.client = pages.client

sql/2024/privacy/number_of_websites_with_gpc.sql

@@ -1,13 +1,12 @@
-#standardSQL
 # Pages that provide `/.well-known/gpc.json` for Global Privacy Control
 
 WITH pages AS (
   SELECT
     client,
-    COUNT(DISTINCT IF(JSON_VALUE(custom_metrics, '$.well-known."/.well-known/gpc.json".found') = 'true', page, NULL)) AS well_known_pages_count,
-    COUNT(DISTINCT IF(JSON_VALUE(custom_metrics, '$.well-known."/.well-known/gpc.json".found') = 'true', page, NULL)) / COUNT(DISTINCT page) AS well_known_pages_pct,
-    COUNT(DISTINCT IF(JSON_VALUE(custom_metrics, '$.privacy.navigator_globalPrivacyControl') = 'true', page, NULL)) AS js_api_pages_count,
-    COUNT(DISTINCT IF(JSON_VALUE(custom_metrics, '$.privacy.navigator_globalPrivacyControl') = 'true', page, NULL)) / COUNT(DISTINCT page) AS js_api_pages_pct
+    COUNT(DISTINCT IF(JSON_VALUE(custom_metrics, '$.well-known."/.well-known/gpc.json".found') = 'true', page, NULL)) / COUNT(DISTINCT page) AS pct_pages_well_known,
+    COUNT(DISTINCT IF(JSON_VALUE(custom_metrics, '$.well-known."/.well-known/gpc.json".found') = 'true', page, NULL)) AS number_of_pages_well_known,
+    COUNT(DISTINCT IF(JSON_VALUE(custom_metrics, '$.privacy.navigator_globalPrivacyControl') = 'true', page, NULL)) / COUNT(DISTINCT page) AS pct_pages_js_api,
+    COUNT(DISTINCT IF(JSON_VALUE(custom_metrics, '$.privacy.navigator_globalPrivacyControl') = 'true', page, NULL)) AS number_of_pages_js_api
   FROM `httparchive.all.pages`
   WHERE
     date = '2024-06-01' AND
@@ -18,8 +17,8 @@ WITH pages AS (
 headers AS (
   SELECT
     client,
-    COUNT(DISTINCT IF(headers.name = 'sec-gpc' AND headers.value = '1', page, NULL)) AS headers_pages_count,
-    COUNT(DISTINCT IF(headers.name = 'sec-gpc' AND headers.value = '1', page, NULL)) / COUNT(DISTINCT page) AS headers_pages_pct
+    COUNT(DISTINCT IF(headers.name = 'sec-gpc' AND headers.value = '1', page, NULL)) / COUNT(DISTINCT page) AS pct_pages_headers,
+    COUNT(DISTINCT IF(headers.name = 'sec-gpc' AND headers.value = '1', page, NULL)) AS number_of_pages_headers
   FROM `httparchive.all.requests`,
     UNNEST(response_headers) headers
   WHERE

sql/2024/privacy/number_of_websites_with_iab.sql

@@ -1,4 +1,3 @@
-#standardSQL
 # Counts of pages with IAB Frameworks (Transparency & Consent / US Privacy)
 
 WITH privacy_custom_metrics_data AS (
@@ -12,38 +11,45 @@ WITH privacy_custom_metrics_data AS (
 )
 
 SELECT
-  *,
-  number_of_websites_with_iab_tcf_v1 / number_of_websites AS pct_websites_with_iab_tcf_v1,
-  number_of_websites_with_iab_tcf_v2 / number_of_websites AS pct_websites_with_iab_tcf_v2,
-  number_of_websites_with_iab_usp / number_of_websites AS pct_websites_with_iab_usp,
-  number_of_websites_with_iab_tcf_any / number_of_websites AS pct_websites_with_iab_tcf_any,
-  number_of_websites_with_iab_any / number_of_websites AS pct_websites_with_iab_any,
-  number_of_websites_with_iab_tcf_v1_compliant / number_of_websites_with_iab_tcf_v1 AS pct_websites_with_iab_tcf_v1_compliant,
-  number_of_websites_with_iab_tcf_v2_compliant / number_of_websites_with_iab_tcf_v2 AS pct_websites_with_iab_tcf_v2_compliant
+  client,
+  number_of_pages_with_iab_tcf_v1 / number_of_pages AS pct_pages_with_iab_tcf_v1,
+  number_of_pages_with_iab_tcf_v1,
+  number_of_pages_with_iab_tcf_v2 / number_of_pages AS pct_pages_with_iab_tcf_v2,
+  number_of_pages_with_iab_tcf_v2,
+  number_of_pages_with_iab_usp / number_of_pages AS pct_pages_with_iab_usp,
+  number_of_pages_with_iab_usp,
+  number_of_pages_with_iab_tcf_any / number_of_pages AS pct_pages_with_iab_tcf_any,
+  number_of_pages_with_iab_tcf_any,
+  number_of_pages_with_iab_any / number_of_pages AS pct_pages_with_iab_any,
+  number_of_pages_with_iab_any,
+  number_of_pages_with_iab_tcf_v1_compliant / number_of_pages_with_iab_tcf_v1 AS pct_pages_with_iab_tcf_v1_compliant,
+  number_of_pages_with_iab_tcf_v1_compliant,
+  number_of_pages_with_iab_tcf_v2_compliant / number_of_pages_with_iab_tcf_v2 AS pct_pages_with_iab_tcf_v2_compliant,
+  number_of_pages_with_iab_tcf_v2_compliant
 FROM (
   SELECT
     client,
-    COUNT(0) AS number_of_websites,
-    COUNTIF(JSON_VALUE(metrics, '$.iab_tcf_v1.present') = 'true') AS number_of_websites_with_iab_tcf_v1,
-    COUNTIF(JSON_VALUE(metrics, '$.iab_tcf_v2.present') = 'true') AS number_of_websites_with_iab_tcf_v2,
-    COUNTIF(JSON_VALUE(metrics, '$.iab_usp.present') = 'true') AS number_of_websites_with_iab_usp,
+    COUNT(0) AS number_of_pages,
+    COUNTIF(JSON_VALUE(metrics, '$.iab_tcf_v1.present') = 'true') AS number_of_pages_with_iab_tcf_v1,
+    COUNTIF(JSON_VALUE(metrics, '$.iab_tcf_v2.present') = 'true') AS number_of_pages_with_iab_tcf_v2,
+    COUNTIF(JSON_VALUE(metrics, '$.iab_usp.present') = 'true') AS number_of_pages_with_iab_usp,
     COUNTIF(
       JSON_VALUE(metrics, '$.iab_tcf_v1.present') = 'true' OR
       JSON_VALUE(metrics, '$.iab_tcf_v2.present') = 'true'
-    ) AS number_of_websites_with_iab_tcf_any,
+    ) AS number_of_pages_with_iab_tcf_any,
     COUNTIF(
       JSON_VALUE(metrics, '$.iab_tcf_v1.present') = 'true' OR
       JSON_VALUE(metrics, '$.iab_tcf_v2.present') = 'true' OR
       JSON_VALUE(metrics, '$.iab_usp.present') = 'true'
-    ) AS number_of_websites_with_iab_any,
+    ) AS number_of_pages_with_iab_any,
     COUNTIF(
       JSON_VALUE(metrics, '$.iab_tcf_v1.present') = 'true' AND
       JSON_VALUE(metrics, '$.iab_tcf_v1.compliant_setup') = 'true'
-    ) AS number_of_websites_with_iab_tcf_v1_compliant,
+    ) AS number_of_pages_with_iab_tcf_v1_compliant,
     COUNTIF(
       JSON_VALUE(metrics, '$.iab_tcf_v2.present') = 'true' AND
       JSON_VALUE(metrics, '$.iab_tcf_v2.compliant_setup') = 'true'
-    ) AS number_of_websites_with_iab_tcf_v2_compliant
+    ) AS number_of_pages_with_iab_tcf_v2_compliant
   FROM privacy_custom_metrics_data
   GROUP BY client
 )