Fine-tune details about when differential update should kick in

Manual update of one or more lists will cause the most recent version
of these lists to be fetched from the "origin" server, and since the
lists from "origin" servers cannot be updated through differential
update, the lists will be subsequently updated according to their
`Expires` directive.

When the lists are auto-updated, the "CDN" servers will be used,
and as a result the lists will start to be updated trhough
differential updates every 6-hour (currently).

Thus it is recommended and optimal to let the lists auto-update,
since you will benefit from a much shorter delay to get up-to-date
lists (i.e. every 6-hour instead of every 6-day).

You can force the auto-updater to fetch all the lists by clicking
"Purge all caches", then restart uBO without clicking "Update".
This will cause uBO to perform an emergency auto-update at restart
time, after which you will have all the lists which are candidates
for differential update.

The "Update now" button in the "Support" pane will also cause lists
to be fetched from their "origin" server.

Author: gorhill

src/js/assets.js

@@ -26,9 +26,10 @@
 import cacheStorage from './cachestorage.js';
 import logger from './logger.js';
 import µb from './background.js';
+import { ubolog } from './console.js';
 import { i18n$ } from './i18n.js';
 import * as sfp from './static-filtering-parser.js';
-import { ubolog } from './console.js';
+import { orphanizeString, } from './text-utils.js';
 
 /******************************************************************************/
 
@@ -47,6 +48,8 @@ let remoteServerFriendly = false;
 
 /******************************************************************************/
 
+const stringIsNotEmpty = s => typeof s === 'string' && s !== '';
+
 const parseExpires = s => {
     const matches = s.match(/(\d+)\s*([dh])?/i);
     if ( matches === null ) { return 0; }
@@ -71,7 +74,7 @@ const extractMetadataFromList = (content, fields) => {
         field = field.toLowerCase().replace(
             /-[a-z]/g, s => s.charAt(1).toUpperCase()
         );
-        out[field] = value;
+        out[field] = value && orphanizeString(value);
     }
     // Pre-process known fields
     if ( out.lastModified ) {
@@ -169,7 +172,44 @@ const isDiffUpdatableAsset = content => {
         /^[^%].*[^%]$/.test(data.diffPath);
 };
 
-const stringIsNotEmpty = s => typeof s === 'string' && s !== '';
+/******************************************************************************/
+
+// favorLocal: avoid making network requests whenever possible
+// favorOrigin: avoid using CDN URLs whenever possible
+
+const getContentURLs = (assetKey, options = {}) => {
+    const contentURLs = [];
+    const entry = assetSourceRegistry[assetKey];
+    if ( entry instanceof Object === false ) { return contentURLs; }
+    if ( typeof entry.contentURL === 'string' ) {
+        contentURLs.push(entry.contentURL);
+    } else if ( Array.isArray(entry.contentURL) ) {
+        contentURLs.push(...entry.contentURL);
+    } else if ( reIsExternalPath.test(assetKey) ) {
+        contentURLs.push(assetKey);
+    }
+    if ( options.favorLocal ) {
+        contentURLs.sort((a, b) => {
+            if ( reIsExternalPath.test(a) ) { return 1; }
+            if ( reIsExternalPath.test(b) ) { return -1; }
+            return 0;
+        });
+    }
+    if ( Array.isArray(entry.cdnURLs) ) {
+        const cdnURLs = entry.cdnURLs.slice();
+        for ( let i = 0, n = cdnURLs.length; i < n; i++ ) {
+            const j = Math.floor(Math.random() * n);
+            if ( j === i ) { continue; }
+            [ cdnURLs[j], cdnURLs[i] ] = [ cdnURLs[i], cdnURLs[j] ];
+        }
+        if ( options.favorLocal || options.favorOrigin ) {
+            contentURLs.push(...cdnURLs);
+        } else {
+            contentURLs.unshift(...cdnURLs);
+        }
+    }
+    return contentURLs;
+};
 
 /******************************************************************************/
 
@@ -917,28 +957,17 @@ assets.get = async function(assetKey, options = {}) {
     }
 
     const assetRegistry = await getAssetSourceRegistry();
+
     assetDetails = assetRegistry[assetKey] || {};
-    const contentURLs = [];
-    if ( typeof assetDetails.contentURL === 'string' ) {
-        contentURLs.push(assetDetails.contentURL);
-    } else if ( Array.isArray(assetDetails.contentURL) ) {
-        contentURLs.push(...assetDetails.contentURL);
-    } else if ( reIsExternalPath.test(assetKey) ) {
+
+    const contentURLs = getContentURLs(assetKey, options);
+    if ( contentURLs.length === 0 && reIsExternalPath.test(assetKey) ) {
         assetDetails.content = 'filters';
         contentURLs.push(assetKey);
     }
 
-    // https://github.com/uBlockOrigin/uBlock-issues/issues/1566#issuecomment-826473517
-    //   Use CDN URLs as fall back URLs.
-    if ( Array.isArray(assetDetails.cdnURLs) ) {
-        contentURLs.push(...assetDetails.cdnURLs);
-    }
-
     let error = 'ENOTFOUND';
     for ( const contentURL of contentURLs ) {
-        if ( reIsExternalPath.test(contentURL) && assetDetails.hasLocalURL ) {
-            continue;
-        }
         const details = assetDetails.content === 'filters'
             ? await assets.fetchFilterList(contentURL)
             : await assets.fetchText(contentURL);
@@ -966,7 +995,7 @@ assets.get = async function(assetKey, options = {}) {
 
 /******************************************************************************/
 
-async function getRemote(assetKey) {
+async function getRemote(assetKey, options = {}) {
     const [
         assetDetails = {},
         cacheDetails = {},
@@ -978,56 +1007,19 @@ async function getRemote(assetKey) {
     let error;
     let stale = false;
 
-    const reportBack = function(content, err) {
-        const details = { assetKey, content };
-        if ( err ) {
+    const reportBack = function(content, url = '', err = '') {
+        const details = { assetKey, content, url };
+        if ( err !== '') {
             details.error = assetDetails.lastError = err;
         } else {
             assetDetails.lastError = undefined;
         }
         return details;
     };
 
-    const contentURLs = [];
-    if ( typeof assetDetails.contentURL === 'string' ) {
-        contentURLs.push(assetDetails.contentURL);
-    } else if ( Array.isArray(assetDetails.contentURL) ) {
-        contentURLs.push(...assetDetails.contentURL);
-    }
-
-    // If asked to be gentle on remote servers, favour using dedicated CDN
-    // servers. If more than one CDN server is present, randomly shuffle the
-    // set of servers so as to spread the bandwidth burden.
-    //
-    // https://github.com/uBlockOrigin/uBlock-issues/issues/1566#issuecomment-826473517
-    //   In case of manual update, use CDNs URLs as fall back URLs.
-    if ( Array.isArray(assetDetails.cdnURLs) ) {
-        const cdnURLs = assetDetails.cdnURLs.slice();
-        for ( let i = 0, n = cdnURLs.length; i < n; i++ ) {
-            const j = Math.floor(Math.random() * n);
-            if ( j === i ) { continue; }
-            [ cdnURLs[j], cdnURLs[i] ] = [ cdnURLs[i], cdnURLs[j] ];
-        }
-        if ( remoteServerFriendly ) {
-            contentURLs.unshift(...cdnURLs);
-        } else {
-            contentURLs.push(...cdnURLs);
-        }
-    }
-
-    for ( let contentURL of contentURLs ) {
+    for ( const contentURL of getContentURLs(assetKey, options) ) {
         if ( reIsExternalPath.test(contentURL) === false ) { continue; }
 
-        // This will force uBO to fetch the proper version according to whether
-        // the dev build is being used. This can be removed when execution of
-        // this code path is widespread for dev build revisions of uBO.
-        if ( assetKey === 'assets.json' ) {
-            contentURL = contentURL.replace(
-                /\/assets\/assets\.json$/,
-                µb.assetsJsonPath
-            );
-        }
-
         const result = assetDetails.content === 'filters'
             ? await assets.fetchFilterList(contentURL)
             : await assets.fetchText(contentURL);
@@ -1066,12 +1058,12 @@ async function getRemote(assetKey) {
         }
 
         registerAssetSource(assetKey, { birthtime: undefined, error: undefined });
-        return reportBack(result.content);
+        return reportBack(result.content, contentURL);
     }
 
     if ( error !== undefined ) {
         registerAssetSource(assetKey, { error: { time: Date.now(), error } });
-        return reportBack('', 'ENOTFOUND');
+        return reportBack('', '', 'ENOTFOUND');
     }
 
     if ( stale ) {
@@ -1194,6 +1186,8 @@ const getAssetDiffDetails = assetKey => {
 };
 
 async function diffUpdater() {
+    if ( updaterAuto === false ) { return; }
+    if ( µb.hiddenSettings.differentialUpdate === false ) { return; }
     const toUpdate = await getUpdateCandidates();
     const now = Date.now();
     const toHardUpdate = [];
@@ -1298,6 +1292,7 @@ async function diffUpdater() {
 
 function updateFirst() {
     ubolog('Updater: cycle start');
+    ubolog('Updater: Fetch from ', updaterAuto ? 'CDNs' : 'origin');
     updaterStatus = 'updating';
     updaterFetched.clear();
     updaterUpdated.length = 0;
@@ -1367,7 +1362,7 @@ async function updateNext() {
 
     let result;
     if ( assetKey !== 'assets.json' || µb.hiddenSettings.debugAssetsJson !== true ) {
-        result = await getRemote(assetKey);
+        result = await getRemote(assetKey, { favorOrigin: updaterAuto === false });
     } else {
         result = await assets.fetchText(µb.assetsJsonPath);
         result.assetKey = 'assets.json';
@@ -1396,6 +1391,7 @@ function updateDone() {
     updaterFetched.clear();
     updaterUpdated.length = 0;
     updaterStatus = undefined;
+    updaterAuto = false;
     updaterAssetDelay = updaterAssetDelayDefault;
     ubolog('Updater: cycle end');
     if ( assetKeys.length ) {

src/js/background.js

@@ -49,9 +49,9 @@ const hiddenSettingsDefault = {
     allowGenericProceduralFilters: false,
     assetFetchTimeout: 30,
     autoCommentFilterTemplate: '{{date}} {{origin}}',
-    autoUpdateAssetFetchPeriod: 60,
+    autoUpdateAssetFetchPeriod: 15,
     autoUpdateDelayAfterLaunch: 105,
-    autoUpdatePeriod: 2,
+    autoUpdatePeriod: 1,
     benchmarkDatasetURL: 'unset',
     blockingProfiles: '11111/#F00 11010/#C0F 11001/#00F 00001',
     cacheStorageAPI: 'unset',
@@ -69,6 +69,7 @@ const hiddenSettingsDefault = {
     debugAssetsJson: false,
     debugScriptlets: false,
     debugScriptletInjector: false,
+    differentialUpdate: true,
     disableWebAssembly: false,
     extensionUpdateForceReload: false,
     filterAuthorMode: false,

src/js/storage.js

@@ -981,7 +981,10 @@ import {
         return { assetKey, content: '' };
     }
 
-    const rawDetails = await io.get(assetKey, { silent: true });
+    const rawDetails = await io.get(assetKey, {
+        favorLocal: this.readyToFilter !== true,
+        silent: true,
+    });
     // Compiling an empty string results in an empty string.
     if ( rawDetails.content === '' ) {
         rawDetails.assetKey = assetKey;