report: add table filter for third-party urls (#6351)

Author: wardpeet

lighthouse-core/audits/seo/canonical.js

@@ -35,19 +35,6 @@ const UIStrings = {
 
 const str_ = i18n.createMessageInstanceIdFn(__filename, UIStrings);
 
-/**
- * Returns a primary domain for provided URL (e.g. http://www.example.com -> example.com).
- * Note that it does not take second-level domains into account (.co.uk).
- * @param {URL} url
- * @returns {string}
- */
-function getPrimaryDomain(url) {
-  return url.hostname
-    .split('.')
-    .slice(-2)
-    .join('.');
-}
-
 /**
  * @typedef CanonicalURLData
  * @property {Set<string>} uniqueCanonicalURLs
@@ -173,7 +160,7 @@ class Canonical extends Audit {
 
     // bing and yahoo don't allow canonical URLs pointing to different domains, it's also
     // a common mistake to publish a page with canonical pointing to e.g. a test domain or localhost
-    if (getPrimaryDomain(canonicalURL) !== getPrimaryDomain(baseURL)) {
+    if (!URL.rootDomainsMatch(canonicalURL, baseURL)) {
       return {
         score: 0,
         explanation: str_(UIStrings.explanationDifferentDomain, {url: canonicalURL}),

lighthouse-core/lib/i18n/en-US.json

@@ -1343,6 +1343,10 @@
     "message": "Expand snippet",
     "description": "Label for button that shows all lines of the snippet when clicked"
   },
+  "lighthouse-core/report/html/renderer/util.js | thirdPartyResourcesLabel": {
+    "message": "Show 3rd-party resources",
+    "description": "This label is for a checkbox above a table of items loaded by a web page. The checkbox is used to show or hide third-party (or \"3rd-party\") resources in the table, where \"third-party resources\" refers to items loaded by a web page from URLs that aren't controlled by the owner of the web page."
+  },
   "lighthouse-core/report/html/renderer/util.js | toplevelWarningsMessage": {
     "message": "There were issues affecting this run of Lighthouse:",
     "description": "Label shown preceding any important warnings that may have invalidated the entire report. For example, if the user has Chrome extensions installed, they may add enough performance overhead that Lighthouse's performance metrics are unreliable. If shown, this will be displayed at the top of the report UI."

lighthouse-core/lib/url-shim.js

@@ -9,22 +9,10 @@
  * URL shim so we keep our code DRY
  */
 
-/* global self */
+/* global URL */
 
 const Util = require('../report/html/renderer/util.js');
 
-// Type cast so tsc sees window.URL and require('url').URL as sufficiently equivalent.
-const URL = /** @type {!Window["URL"]} */ (typeof self !== 'undefined' && self.URL) ||
-    require('url').URL;
-
-// 25 most used tld plus one domains (aka public suffixes) from http archive.
-// @see https://github.com/GoogleChrome/lighthouse/pull/5065#discussion_r191926212
-// The canonical list is https://publicsuffix.org/learn/ but we're only using subset to conserve bytes
-const listOfTlds = [
-  'com', 'co', 'gov', 'edu', 'ac', 'org', 'go', 'gob', 'or', 'net', 'in', 'ne', 'nic', 'gouv',
-  'web', 'spb', 'blog', 'jus', 'kiev', 'mil', 'wi', 'qc', 'ca', 'bel', 'on',
-];
-
 const allowedProtocols = [
   'https:', 'http:', 'chrome:', 'chrome-extension:',
 ];
@@ -99,34 +87,18 @@ class URLShim extends URL {
     }
   }
 
-  /**
-   * Gets the tld of a domain
-   *
-   * @param {string} hostname
-   * @return {string} tld
-   */
-  static getTld(hostname) {
-    const tlds = hostname.split('.').slice(-2);
-
-    if (!listOfTlds.includes(tlds[0])) {
-      return `.${tlds[tlds.length - 1]}`;
-    }
-
-    return `.${tlds.join('.')}`;
-  }
-
   /**
    * Check if rootDomains matches
    *
-   * @param {string} urlA
-   * @param {string} urlB
+   * @param {string|URL} urlA
+   * @param {string|URL} urlB
    */
   static rootDomainsMatch(urlA, urlB) {
     let urlAInfo;
     let urlBInfo;
     try {
-      urlAInfo = new URL(urlA);
-      urlBInfo = new URL(urlB);
+      urlAInfo = Util.createOrReturnURL(urlA);
+      urlBInfo = Util.createOrReturnURL(urlB);
     } catch (err) {
       return false;
     }
@@ -135,14 +107,9 @@ class URLShim extends URL {
       return false;
     }
 
-    const tldA = URLShim.getTld(urlAInfo.hostname);
-    const tldB = URLShim.getTld(urlBInfo.hostname);
-
     // get the string before the tld
-    const urlARootDomain = urlAInfo.hostname.replace(new RegExp(`${tldA}$`), '')
-      .split('.').splice(-1)[0];
-    const urlBRootDomain = urlBInfo.hostname.replace(new RegExp(`${tldB}$`), '')
-      .split('.').splice(-1)[0];
+    const urlARootDomain = Util.getRootDomain(urlAInfo);
+    const urlBRootDomain = Util.getRootDomain(urlBInfo);
 
     return urlARootDomain === urlBRootDomain;
   }

lighthouse-core/report/html/renderer/details-renderer.js

@@ -124,7 +124,11 @@ class DetailsRenderer {
       element.appendChild(hostElem);
     }
 
-    if (title) element.title = url;
+    if (title) {
+      element.title = url;
+      // set the url on the element's dataset which we use to check 3rd party origins
+      element.dataset.url = url;
+    }
     return element;
   }
 

lighthouse-core/report/html/renderer/report-ui-features.js

@@ -16,15 +16,25 @@
  */
 'use strict';
 
+/* eslint-env browser */
+
 /**
  * @fileoverview Adds export button, print, and other dynamic functionality to
  * the report.
  */
 
-/* globals self URL Blob CustomEvent getFilenamePrefix window */
+/* globals getFilenamePrefix Util */
 
 /** @typedef {import('./dom.js')} DOM */
 
+/**
+ * @param {HTMLTableElement} tableEl
+ * @return {Array<HTMLTableRowElement>}
+ */
+function getTableRows(tableEl) {
+  return Array.from(tableEl.tBodies[0].rows);
+}
+
 class ReportUIFeatures {
   /**
    * @param {DOM} dom
@@ -74,6 +84,7 @@ class ReportUIFeatures {
     this._setupMediaQueryListeners();
     this._setupSmoothScroll();
     this._setupExportButton();
+    this._setupThirdPartyFilter();
     this._setupStickyHeaderElements();
     this._setUpCollapseDetailsAfterPrinting();
     this._resetUIState();
@@ -131,6 +142,92 @@ class ReportUIFeatures {
     dropdown.addEventListener('click', this.onExport);
   }
 
+  _setupThirdPartyFilter() {
+    // Some audits should not display the third party filter option.
+    const thirdPartyFilterAuditExclusions = [
+      // This audit deals explicitly with third party resources.
+      'uses-rel-preconnect',
+    ];
+
+    // Get all tables with a text url column.
+    /** @type {Array<HTMLTableElement>} */
+    const tables = Array.from(this._document.querySelectorAll('.lh-table'));
+    const tablesWithUrls = tables
+      .filter(el => el.querySelector('td.lh-table-column--url'))
+      .filter(el => {
+        const containingAudit = el.closest('.lh-audit');
+        if (!containingAudit) throw new Error('.lh-table not within audit');
+        return !thirdPartyFilterAuditExclusions.includes(containingAudit.id);
+      });
+
+    tablesWithUrls.forEach((tableEl, index) => {
+      const thirdPartyRows = this._getThirdPartyRows(tableEl, this.json.finalUrl);
+      // No 3rd parties, no checkbox!
+      if (!thirdPartyRows.size) return;
+
+      // create input box
+      const filterTemplate = this._dom.cloneTemplate('#tmpl-lh-3p-filter', this._document);
+      const filterInput = this._dom.find('input', filterTemplate);
+      const id = `lh-3p-filter-label--${index}`;
+
+      filterInput.id = id;
+      filterInput.addEventListener('change', e => {
+        // Remove rows from the dom and keep track of them to readd on uncheck.
+        // Why removing instead of hiding? To keep nth-child(even) background-colors working.
+        if (e.target instanceof HTMLInputElement && !e.target.checked) {
+          for (const row of thirdPartyRows.values()) {
+            row.remove();
+          }
+        } else {
+          // Add row elements back to original positions.
+          for (const [position, row] of thirdPartyRows.entries()) {
+            const childrenArr = getTableRows(tableEl);
+            tableEl.tBodies[0].insertBefore(row, childrenArr[position]);
+          }
+        }
+      });
+
+      this._dom.find('label', filterTemplate).setAttribute('for', id);
+      this._dom.find('.lh-3p-filter-count', filterTemplate).textContent =
+          `${thirdPartyRows.size}`;
+      this._dom.find('.lh-3p-ui-string', filterTemplate).textContent =
+          Util.UIStrings.thirdPartyResourcesLabel;
+
+      // Finally, add checkbox to the DOM.
+      if (!tableEl.parentNode) return; // Keep tsc happy.
+      tableEl.parentNode.insertBefore(filterTemplate, tableEl);
+    });
+  }
+
+  /**
+   * From a table with URL entries, finds the rows containing third-party URLs
+   * and returns a Map of those rows, mapping from row index to row Element.
+   * @param {HTMLTableElement} el
+   * @param {string} finalUrl
+   * @return {Map<number, HTMLTableRowElement>}
+   */
+  _getThirdPartyRows(el, finalUrl) {
+    const urlItems = this._dom.findAll('.lh-text__url', el);
+    const finalUrlRootDomain = Util.getRootDomain(finalUrl);
+
+    /** @type {Map<number, HTMLTableRowElement>} */
+    const thirdPartyRows = new Map();
+    for (const urlItem of urlItems) {
+      const datasetUrl = urlItem.dataset.url;
+      if (!datasetUrl) continue;
+      const isThirdParty = Util.getRootDomain(datasetUrl) !== finalUrlRootDomain;
+      if (!isThirdParty) continue;
+
+      const urlRowEl = urlItem.closest('tr');
+      if (urlRowEl) {
+        const rowPosition = getTableRows(el).indexOf(urlRowEl);
+        thirdPartyRows.set(rowPosition, urlRowEl);
+      }
+    }
+
+    return thirdPartyRows;
+  }
+
   _setupStickyHeaderElements() {
     this.topbarEl = this._dom.find('.lh-topbar', this._document);
     this.scoreScaleEl = this._dom.find('.lh-scorescale', this._document);

lighthouse-core/report/html/renderer/util.js

@@ -16,7 +16,7 @@
  */
 'use strict';
 
-/* globals self URL */
+/* globals self, URL */
 
 const ELLIPSIS = '\u2026';
 const NBSP = '\xa0';
@@ -29,6 +29,14 @@ const RATINGS = {
   ERROR: {label: 'error'},
 };
 
+// 25 most used tld plus one domains (aka public suffixes) from http archive.
+// @see https://github.com/GoogleChrome/lighthouse/pull/5065#discussion_r191926212
+// The canonical list is https://publicsuffix.org/learn/ but we're only using subset to conserve bytes
+const listOfTlds = [
+  'com', 'co', 'gov', 'edu', 'ac', 'org', 'go', 'gob', 'or', 'net', 'in', 'ne', 'nic', 'gouv',
+  'web', 'spb', 'blog', 'jus', 'kiev', 'mil', 'wi', 'qc', 'ca', 'bel', 'on',
+];
+
 class Util {
   static get PASS_THRESHOLD() {
     return PASS_THRESHOLD;
@@ -336,6 +344,51 @@ class Util {
     };
   }
 
+  /**
+   * @param {string|URL} value
+   * @return {URL}
+   */
+  static createOrReturnURL(value) {
+    if (value instanceof URL) {
+      return value;
+    }
+
+    return new URL(value);
+  }
+
+  /**
+   * Gets the tld of a domain
+   *
+   * @param {string} hostname
+   * @return {string} tld
+   */
+  static getTld(hostname) {
+    const tlds = hostname.split('.').slice(-2);
+
+    if (!listOfTlds.includes(tlds[0])) {
+      return `.${tlds[tlds.length - 1]}`;
+    }
+
+    return `.${tlds.join('.')}`;
+  }
+
+  /**
+   * Returns a primary domain for provided hostname (e.g. www.example.com -> example.com).
+   * @param {string|URL} url hostname or URL object
+   * @returns {string}
+   */
+  static getRootDomain(url) {
+    const hostname = Util.createOrReturnURL(url).hostname;
+    const tld = Util.getTld(hostname);
+
+    // tld is .com or .co.uk which means we means that length is 1 to big
+    // .com => 2 & .co.uk => 3
+    const splitTld = tld.split('.');
+
+    // get TLD + root domain
+    return hostname.split('.').slice(-splitTld.length).join('.');
+  }
+
   /**
    * @param {LH.Config.Settings} settings
    * @return {Array<{name: string, description: string}>}
@@ -524,6 +577,9 @@ Util.UIStrings = {
   lsPerformanceCategoryDescription: '[Lighthouse](https://developers.google.com/web/tools/lighthouse/) analysis of the current page on an emulated mobile network. Values are estimated and may vary.',
   /** Title of the lab data section of the Performance category. Within this section are various speed metrics which quantify the pageload performance into values presented in seconds and milliseconds. "Lab" is an abbreviated form of "laboratory", and refers to the fact that the data is from a controlled test of a website, not measurements from real users visiting that site.  */
   labDataTitle: 'Lab Data',
+
+  /** This label is for a checkbox above a table of items loaded by a web page. The checkbox is used to show or hide third-party (or "3rd-party") resources in the table, where "third-party resources" refers to items loaded by a web page from URLs that aren't controlled by the owner of the web page. */
+  thirdPartyResourcesLabel: 'Show 3rd-party resources',
 };
 
 if (typeof module !== 'undefined' && module.exports) {