report(third party): filter out third party urls

lighthouse-core/audits/seo/canonical.js

@@ -35,19 +35,6 @@ const UIStrings = {
 
 const str_ = i18n.createMessageInstanceIdFn(__filename, UIStrings);
 
-/**
- * Returns a primary domain for provided URL (e.g. http://www.example.com -> example.com).
- * Note that it does not take second-level domains into account (.co.uk).
- * @param {URL} url
- * @returns {string}
- */
-function getPrimaryDomain(url) {
-  return url.hostname
-    .split('.')
-    .slice(-2)
-    .join('.');
-}
-
 /**
  * @typedef CanonicalURLData
  * @property {Set<string>} uniqueCanonicalURLs
@@ -173,7 +160,7 @@ class Canonical extends Audit {
 
     // bing and yahoo don't allow canonical URLs pointing to different domains, it's also
     // a common mistake to publish a page with canonical pointing to e.g. a test domain or localhost
-    if (getPrimaryDomain(canonicalURL) !== getPrimaryDomain(baseURL)) {
+    if (!URL.rootDomainsMatch(canonicalURL, baseURL)) {
       return {
         rawValue: false,
         explanation: str_(UIStrings.explanationDifferentDomain, {url: canonicalURL}),

lighthouse-core/lib/i18n/en-US.json

@@ -1319,6 +1319,10 @@
     "message": "Expand snippet",
     "description": "Label for button that shows all lines of the snippet when clicked"
   },
+  "lighthouse-core/report/html/renderer/util.js | thirdPartyResourcesLabel": {
+    "message": "Show 3rd party resources",
+    "description": "This label is shown above a table next to the checkbox to hide 3rd party resources"
+  },
   "lighthouse-core/report/html/renderer/util.js | toplevelWarningsMessage": {
     "message": "There were issues affecting this run of Lighthouse:",
     "description": "Label shown preceding any important warnings that may have invalidated the entire report. For example, if the user has Chrome extensions installed, they may add enough performance overhead that Lighthouse's performance metrics are unreliable. If shown, this will be displayed at the top of the report UI."

lighthouse-core/lib/url-shim.js

@@ -11,20 +11,14 @@
 
 /* global self */
 
+/** @typedef {import('url').URL} OriginalURL */
+
 const Util = require('../report/html/renderer/util.js');
 
 // Type cast so tsc sees window.URL and require('url').URL as sufficiently equivalent.
 const URL = /** @type {!Window["URL"]} */ (typeof self !== 'undefined' && self.URL) ||
     require('url').URL;
 
-// 25 most used tld plus one domains (aka public suffixes) from http archive.
-// @see https://github.com/GoogleChrome/lighthouse/pull/5065#discussion_r191926212
-// The canonical list is https://publicsuffix.org/learn/ but we're only using subset to conserve bytes
-const listOfTlds = [
-  'com', 'co', 'gov', 'edu', 'ac', 'org', 'go', 'gob', 'or', 'net', 'in', 'ne', 'nic', 'gouv',
-  'web', 'spb', 'blog', 'jus', 'kiev', 'mil', 'wi', 'qc', 'ca', 'bel', 'on',
-];
-
 const allowedProtocols = [
   'https:', 'http:', 'chrome:', 'chrome-extension:',
 ];
@@ -106,27 +100,30 @@ class URLShim extends URL {
    * @return {string} tld
    */
   static getTld(hostname) {
-    const tlds = hostname.split('.').slice(-2);
-
-    if (!listOfTlds.includes(tlds[0])) {
-      return `.${tlds[tlds.length - 1]}`;
-    }
+    return Util.getTld(hostname);
+  }
 
-    return `.${tlds.join('.')}`;
+  /**
+   * Returns a primary domain for provided hostname (e.g. www.example.com -> example.com).
+   * @param {string|OriginalURL} url hostname or URL object
+   * @returns {string}
+   */
+  static getRootDomain(url) {
+    return Util.getRootDomain(url);
   }
 
   /**
    * Check if rootDomains matches
    *
-   * @param {string} urlA
-   * @param {string} urlB
+   * @param {string|OriginalURL} urlA
+   * @param {string|OriginalURL} urlB
    */
   static rootDomainsMatch(urlA, urlB) {
     let urlAInfo;
     let urlBInfo;
     try {
-      urlAInfo = new URL(urlA);
-      urlBInfo = new URL(urlB);
+      urlAInfo = Util.createOrReturnURL(urlA);
+      urlBInfo = Util.createOrReturnURL(urlB);
     } catch (err) {
       return false;
     }
@@ -135,14 +132,9 @@ class URLShim extends URL {
       return false;
     }
 
-    const tldA = URLShim.getTld(urlAInfo.hostname);
-    const tldB = URLShim.getTld(urlBInfo.hostname);
-
     // get the string before the tld
-    const urlARootDomain = urlAInfo.hostname.replace(new RegExp(`${tldA}$`), '')
-      .split('.').splice(-1)[0];
-    const urlBRootDomain = urlBInfo.hostname.replace(new RegExp(`${tldB}$`), '')
-      .split('.').splice(-1)[0];
+    const urlARootDomain = URLShim.getRootDomain(urlAInfo);
+    const urlBRootDomain = URLShim.getRootDomain(urlBInfo);
 
     return urlARootDomain === urlBRootDomain;
   }

lighthouse-core/report/html/renderer/details-renderer.js

@@ -124,7 +124,11 @@ class DetailsRenderer {
       element.appendChild(hostElem);
     }
 
-    if (title) element.title = url;
+    if (title) {
+      element.title = url;
+      // set the url on the element's dataset which we use to check 3rd party origins
+      element.dataset.url = url;
+    }
     return element;
   }
 

lighthouse-core/report/html/renderer/report-ui-features.js

@@ -16,15 +16,24 @@
  */
 'use strict';
 
+/* eslint-env browser */
+
 /**
  * @fileoverview Adds export button, print, and other dynamic functionality to
  * the report.
  */
 
-/* globals self URL Blob CustomEvent getFilenamePrefix window */
+/* globals self URL Blob CustomEvent getFilenamePrefix Util window */
 
 /** @typedef {import('./dom.js')} DOM */
 
+/**
+ * @param {HTMLTableElement} tableEl
+ * @return {Array<HTMLTableRowElement>}
+ */
+const getTableRows = tableEl => /** @type {Array<HTMLTableRowElement>} */
+  (Array.from(tableEl.tBodies[0].children));
+
 class ReportUIFeatures {
   /**
    * @param {DOM} dom
@@ -86,6 +95,7 @@ class ReportUIFeatures {
     this.json = report;
     this._setupMediaQueryListeners();
     this._setupExportButton();
+    this._setupThirdPartyFilter();
     this._setUpCollapseDetailsAfterPrinting();
     this._setupHeaderAnimation();
     this._resetUIState();
@@ -128,6 +138,84 @@ class ReportUIFeatures {
     dropdown.addEventListener('click', this.onExport);
   }
 
+  _setupThirdPartyFilter() {
+    const thirdPartyFilterAuditExclusions = ['uses-rel-preconnect'];
+
+    // get all tables with a text url
+    /** @type {Array<HTMLTableElement>} */
+    const tables = Array.from(this._document.querySelectorAll('.lh-table'));
+    const tablesWithUrls = tables
+      .filter(el => el.querySelector('td.lh-table-column--url'))
+      .filter(el => !thirdPartyFilterAuditExclusions.includes(
+        /** @type {Element} */(el.closest('.lh-audit')).id));
+
+    tablesWithUrls.forEach((tableEl, index) => {
+      const thirdPartyRows = this._getThirdPartyRows(tableEl, this.json.finalUrl);
+      // no 3rd parties, no checkbox!
+      if (!thirdPartyRows.size) {
+        return;
+      }
+      // create input box
+      const filterTemplate = this._dom.cloneTemplate('#tmpl-lh-3p-filter', this._document);
+      const filterInput = /** @type {HTMLInputElement} */ (filterTemplate.querySelector('input'));
+      const id = `lh-3p-filter-label--${index}`;
+
+      filterInput.setAttribute('id', id);
+      filterInput.addEventListener('change', e => {
+        // remove elements from the dom and keep track of them to readd on uncheck
+        // why removing instead of hiding? nth-child(even) background-colors keep working
+        if (e.target instanceof HTMLInputElement && e.target.checked) {
+          for (const [position, row] of thirdPartyRows.entries()) {
+            const childrenArr = getTableRows(tableEl);
+            tableEl.tBodies[0].insertBefore(row, childrenArr[position]);
+          }
+        } else {
+          for (const position of thirdPartyRows.keys()) {
+            const row = /** @type {HTMLTableRowElement} */ (thirdPartyRows.get(position));
+            row.remove();
+          }
+        }
+      });
+
+      if (tableEl.parentNode) {
+        /** @type {Element} */ (filterTemplate.querySelector('label')).setAttribute('for', id);
+        /** @type {Element} */ (filterTemplate.querySelector('.lh-3p-filter-count')).textContent =
+          `${thirdPartyRows.size}`;
+        /** @type {Element} */ (filterTemplate.querySelector('.lh-3p-ui-string')).textContent =
+          `${Util.UIStrings.thirdPartyResourcesLabel}`;
+        // Finally, add checkbox to the DOM
+        tableEl.parentNode.insertBefore(filterTemplate, tableEl);
+      }
+    });
+  }
+
+  /**
+   * @param {HTMLTableElement} el
+   * @param {string} finalUrl
+   * @return {Map<number, HTMLTableRowElement>}
+   */
+  _getThirdPartyRows(el, finalUrl) {
+    /** @type {NodeListOf<HTMLElement>} */
+    const urlItems = el.querySelectorAll('.lh-text__url');
+    const rootDomain = Util.getRootDomain(finalUrl);
+    /** @type {Map<number, HTMLTableRowElement>} */
+    const thirdPartyRows = new Map();
+    for (const urlItem of urlItems) {
+      const isThirdParty = !(urlItem.dataset.url || '').includes(`${rootDomain}/`);
+      if (!isThirdParty) {
+        continue;
+      }
+
+      const urlRowEl = urlItem.closest('tr');
+      if (urlRowEl) {
+        const rowPosition = getTableRows(el).indexOf(urlRowEl);
+        thirdPartyRows.set(rowPosition, urlRowEl);
+      }
+    }
+
+    return thirdPartyRows;
+  }
+
   _setupHeaderAnimation() {
     const scoresWrapper = this._dom.find('.lh-scores-wrapper', this._document);
     const computedMarginTop = window.getComputedStyle(scoresWrapper).marginTop;

lighthouse-core/report/html/renderer/util.js

@@ -16,7 +16,13 @@
  */
 'use strict';
 
-/* globals self URL */
+/* globals self */
+
+/** @typedef {import('url').URL} OriginalURL */
+
+// Type cast so tsc sees window.URL and require('url').URL as sufficiently equivalent.
+const URL = /** @type {!Window["URL"]} */ (typeof self !== 'undefined' && self.URL) ||
+    require('url').URL;
 
 const ELLIPSIS = '\u2026';
 const NBSP = '\xa0';
@@ -29,6 +35,14 @@ const RATINGS = {
   ERROR: {label: 'error'},
 };
 
+// 25 most used tld plus one domains (aka public suffixes) from http archive.
+// @see https://github.com/GoogleChrome/lighthouse/pull/5065#discussion_r191926212
+// The canonical list is https://publicsuffix.org/learn/ but we're only using subset to conserve bytes
+const listOfTlds = [
+  'com', 'co', 'gov', 'edu', 'ac', 'org', 'go', 'gob', 'or', 'net', 'in', 'ne', 'nic', 'gouv',
+  'web', 'spb', 'blog', 'jus', 'kiev', 'mil', 'wi', 'qc', 'ca', 'bel', 'on',
+];
+
 class Util {
   static get PASS_THRESHOLD() {
     return PASS_THRESHOLD;
@@ -253,7 +267,7 @@ class Util {
   }
 
   /**
-   * @param {URL} parsedUrl
+   * @param {OriginalURL} parsedUrl
    * @param {{numPathParts?: number, preserveQuery?: boolean, preserveHost?: boolean}=} options
    * @return {string}
    */
@@ -336,6 +350,51 @@ class Util {
     };
   }
 
+  /**
+   * @param {string|OriginalURL} value
+   * @return {OriginalURL}
+   */
+  static createOrReturnURL(value) {
+    if (value instanceof URL) {
+      return value;
+    }
+
+    return new URL(/** @type {string} */ (value));
+  }
+
+  /**
+   * Gets the tld of a domain
+   *
+   * @param {string} hostname
+   * @return {string} tld
+   */
+  static getTld(hostname) {
+    const tlds = hostname.split('.').slice(-2);
+
+    if (!listOfTlds.includes(tlds[0])) {
+      return `.${tlds[tlds.length - 1]}`;
+    }
+
+    return `.${tlds.join('.')}`;
+  }
+
+  /**
+   * Returns a primary domain for provided hostname (e.g. www.example.com -> example.com).
+   * @param {string|OriginalURL} url hostname or URL object
+   * @returns {string}
+   */
+  static getRootDomain(url) {
+    const hostname = Util.createOrReturnURL(url).hostname;
+    const tld = Util.getTld(hostname);
+
+    // tld is .com or .co.uk which means we means that length is 1 to big
+    // .com => 2 & .co.uk => 3
+    const splitTld = tld.split('.');
+
+    // get TLD + root domain
+    return hostname.split('.').slice(-splitTld.length).join('.');
+  }
+
   /**
    * @param {LH.Config.Settings} settings
    * @return {Array<{name: string, description: string}>}
@@ -519,6 +578,9 @@ Util.UIStrings = {
   lsPerformanceCategoryDescription: '[Lighthouse](https://developers.google.com/web/tools/lighthouse/) analysis of the current page on an emulated mobile network. Values are estimated and may vary.',
   /** Title of the lab data section of the Performance category. Within this section are various speed metrics which quantify the pageload performance into values presented in seconds and milliseconds. "Lab" is an abbreviated form of "laboratory", and refers to the fact that the data is from a controlled test of a website, not measurements from real users visiting that site.  */
   labDataTitle: 'Lab Data',
+
+  /** This label is shown above a table next to the checkbox to hide 3rd party resources */
+  thirdPartyResourcesLabel: 'Show 3rd party resources',
 };
 
 if (typeof module !== 'undefined' && module.exports) {