Implement content and session PO tokens (#6931)

* Implement content and session PO tokens

* Remove timeout after BgUtils update

* Trigger GitHub CI

Author: absidue

src/botGuardScript.js

@@ -1,37 +1,90 @@
-import { BG } from 'bgutils-js'
+import { BG, buildURL, GOOG_API_KEY } from 'bgutils-js'
 
 // This script has it's own webpack config, as it gets passed as a string to Electron's evaluateJavaScript function
 // in src/main/poTokenGenerator.js
-export default async function(visitorData) {
+
+/**
+ * Based on: https://github.com/LuanRT/BgUtils/blob/main/examples/node/innertube-challenge-fetcher-example.ts
+ * @param {string} videoId
+ * @param {string} visitorData
+ * @param {import('youtubei.js').Session['context']} context
+ */
+export default async function (videoId, visitorData, context) {
   const requestKey = 'O43z0dpjhgX20SCx4KAo'
 
-  const bgConfig = {
-    fetch: (input, init) => fetch(input, init),
-    requestKey,
-    globalObj: window,
-    identifier: visitorData
+  const challengeResponse = await fetch(
+    'https://www.youtube.com/youtubei/v1/att/get?prettyPrint=false&alt=json',
+    {
+      method: 'POST',
+      headers: {
+        Accept: '*/*',
+        'Content-Type': 'application/json',
+        'X-Goog-Visitor-Id': visitorData,
+        'X-Youtube-Client-Version': context.client.clientVersion,
+        'X-Youtube-Client-Name': '1'
+      },
+      body: JSON.stringify({
+        engagementType: 'ENGAGEMENT_TYPE_UNBOUND',
+        context
+      }),
+    }
+  )
+
+  if (!challengeResponse.ok) {
+    throw new Error(`Request to ${challengeResponse.url} failed with status ${challengeResponse.status}\n${await challengeResponse.text()}`)
   }
 
-  const challenge = await BG.Challenge.create(bgConfig)
+  const challengeData = await challengeResponse.json()
 
-  if (!challenge) {
-    throw new Error('Could not get challenge')
+  if (!challengeData.bgChallenge) {
+    throw new Error('Failed to get BotGuard challenge')
   }
 
-  const interpreterJavascript = challenge.interpreterJavascript.privateDoNotAccessOrElseSafeScriptWrappedValue
+  let interpreterUrl = challengeData.bgChallenge.interpreterUrl.privateDoNotAccessOrElseTrustedResourceUrlWrappedValue
+
+  if (interpreterUrl.startsWith('//')) {
+    interpreterUrl = `https:${interpreterUrl}`
+  }
+
+  const bgScriptResponse = await fetch(interpreterUrl)
+  const interpreterJavascript = await bgScriptResponse.text()
 
   if (interpreterJavascript) {
     // eslint-disable-next-line no-new-func
     new Function(interpreterJavascript)()
   } else {
-    console.warn('Unable to load VM.')
+    throw new Error('Could not load VM.')
   }
 
-  const poTokenResult = await BG.PoToken.generate({
-    program: challenge.program,
-    globalName: challenge.globalName,
-    bgConfig
+  const botGuard = await BG.BotGuardClient.create({
+    program: challengeData.bgChallenge.program,
+    globalName: challengeData.bgChallenge.globalName,
+    globalObj: window
   })
 
-  return poTokenResult.poToken
+  const webPoSignalOutput = []
+  const botGuardResponse = await botGuard.snapshot({ webPoSignalOutput })
+
+  const integrityTokenResponse = await fetch(buildURL('GenerateIT', true), {
+    method: 'POST',
+    headers: {
+      'content-type': 'application/json+protobuf',
+      'x-goog-api-key': GOOG_API_KEY,
+      'x-user-agent': 'grpc-web-javascript/0.1',
+    },
+    body: JSON.stringify([requestKey, botGuardResponse])
+  })
+
+  const response = await integrityTokenResponse.json()
+
+  if (typeof response[0] !== 'string') {
+    throw new Error('Could not get integrity token')
+  }
+
+  const integrityTokenBasedMinter = await BG.WebPoMinter.create({ integrityToken: response[0] }, webPoSignalOutput)
+
+  const contentPoToken = await integrityTokenBasedMinter.mintAsWebsafeString(videoId)
+  const sessionPoToken = await integrityTokenBasedMinter.mintAsWebsafeString(visitorData)
+
+  return { contentPoToken, sessionPoToken }
 }

src/constants.js

@@ -45,7 +45,7 @@ const IpcChannels = {
 
   SET_INVIDIOUS_AUTHORIZATION: 'set-invidious-authorization',
 
-  GENERATE_PO_TOKEN: 'generate-po-token',
+  GENERATE_PO_TOKENS: 'generate-po-tokens',
 
   GET_SCREENSHOT_FALLBACK_FOLDER: 'get-screenshot-fallback-folder',
   CHOOSE_DEFAULT_FOLDER: 'choose-default-folder',

src/main/index.js

@@ -901,8 +901,8 @@ function runApp() {
     })
   })
 
-  ipcMain.handle(IpcChannels.GENERATE_PO_TOKEN, (_, visitorData) => {
-    return generatePoToken(visitorData, proxyUrl)
+  ipcMain.handle(IpcChannels.GENERATE_PO_TOKENS, (_, videoId, visitorData, context) => {
+    return generatePoToken(videoId, visitorData, context, proxyUrl)
   })
 
   ipcMain.on(IpcChannels.ENABLE_PROXY, (_, url) => {

src/main/poTokenGenerator.js

@@ -9,11 +9,13 @@ import { join } from 'path'
  * This is intentionally split out into it's own thing, with it's own temporary in-memory session,
  * as the BotGuard stuff accesses the global `document` and `window` objects and also requires making some requests.
  * So we definitely don't want it running in the same places as the rest of the FreeTube code with the user data.
+ * @param {string} videoId
  * @param {string} visitorData
+ * @param {string} context
  * @param {string|undefined} proxyUrl
- * @returns {Promise<string>}
+ * @returns {Promise<{ contentPoToken: string, sessionPoToken: string }>}
  */
-export async function generatePoToken(visitorData, proxyUrl) {
+export async function generatePoToken(videoId, visitorData, context, proxyUrl) {
   const sessionUuid = crypto.randomUUID()
 
   const theSession = session.fromPartition(`potoken-${sessionUuid}`, { cache: false })
@@ -22,27 +24,44 @@ export async function generatePoToken(visitorData, proxyUrl) {
   // eslint-disable-next-line n/no-callback-literal
   theSession.setPermissionRequestHandler((webContents, permission, callback) => callback(false))
 
-  theSession.setUserAgent(
-    theSession.getUserAgent()
-      .split(' ')
-      .filter(part => !part.includes('Electron'))
-      .join(' ')
-  )
+  theSession.setUserAgent(session.defaultSession.getUserAgent())
 
   if (proxyUrl) {
     await theSession.setProxy({
       proxyRules: proxyUrl
     })
   }
 
+  theSession.webRequest.onBeforeSendHeaders({
+    urls: ['https://www.google.com/js/*', 'https://www.youtube.com/youtubei/*']
+  }, ({ requestHeaders, url }, callback) => {
+    if (url.startsWith('https://www.youtube.com/youtubei/')) {
+      // make InnerTube requests work with the fetch function
+      // InnerTube rejects requests if the referer isn't YouTube or empty
+      requestHeaders.Referer = 'https://www.youtube.com/'
+      requestHeaders.Origin = 'https://www.youtube.com'
+
+      requestHeaders['Sec-Fetch-Site'] = 'same-origin'
+      requestHeaders['Sec-Fetch-Mode'] = 'same-origin'
+      requestHeaders['X-Youtube-Bootstrap-Logged-In'] = 'false'
+    } else {
+      requestHeaders['Sec-Fetch-Dest'] = 'script'
+      requestHeaders['Sec-Fetch-Site'] = 'cross-site'
+      requestHeaders['Accept-Language'] = '*'
+    }
+
+    callback({ requestHeaders })
+  })
+
   const webContentsView = new WebContentsView({
     webPreferences: {
       backgroundThrottling: false,
       safeDialogs: true,
       sandbox: true,
       v8CacheOptions: 'none',
       session: theSession,
-      offscreen: true
+      offscreen: true,
+      webSecurity: false
     }
   })
 
@@ -58,43 +77,8 @@ export async function generatePoToken(visitorData, proxyUrl) {
 
   webContentsView.webContents.debugger.attach()
 
-  await webContentsView.webContents.loadURL('data:text/html,', {
-    baseURLForDataURL: 'https://www.youtube.com'
-  })
-
-  await webContentsView.webContents.debugger.sendCommand('Emulation.setUserAgentOverride', {
-    userAgent: theSession.getUserAgent(),
-    acceptLanguage: 'en-US',
-    platform: 'Win32',
-    userAgentMetadata: {
-      brands: [
-        {
-          brand: 'Not/A)Brand',
-          version: '99'
-        },
-        {
-          brand: 'Chromium',
-          version: process.versions.chrome.split('.')[0]
-        }
-      ],
-      fullVersionList: [
-        {
-          brand: 'Not/A)Brand',
-          version: '99.0.0.0'
-        },
-        {
-          brand: 'Chromium',
-          version: process.versions.chrome
-        }
-      ],
-      platform: 'Windows',
-      platformVersion: '10.0.0',
-      architecture: 'x86',
-      model: '',
-      mobile: false,
-      bitness: '64',
-      wow64: false
-    }
+  await webContentsView.webContents.loadURL('data:text/html,<!DOCTYPE html><html lang="en"><head><title></title></head><body></body></html>', {
+    baseURLForDataURL: 'https://www.youtube.com/'
   })
 
   await webContentsView.webContents.debugger.sendCommand('Emulation.setDeviceMetricsOverride', {
@@ -112,7 +96,7 @@ export async function generatePoToken(visitorData, proxyUrl) {
     }
   })
 
-  const script = await getScript(visitorData)
+  const script = await getScript(videoId, visitorData, context)
 
   const response = await webContentsView.webContents.executeJavaScript(script)
 
@@ -125,9 +109,11 @@ export async function generatePoToken(visitorData, proxyUrl) {
 let cachedScript
 
 /**
+ * @param {string} videoId
  * @param {string} visitorData
+ * @param {string} context
  */
-async function getScript(visitorData) {
+async function getScript(videoId, visitorData, context) {
   if (!cachedScript) {
     const pathToScript = process.env.NODE_ENV === 'development'
       ? join(__dirname, '../../dist/botGuardScript.js')
@@ -140,8 +126,8 @@ async function getScript(visitorData) {
 
     const functionName = match[1]
 
-    cachedScript = content.replace(match[0], `;${functionName}("FT_VISITOR_DATA")`)
+    cachedScript = content.replace(match[0], `;${functionName}(FT_PARAMS)`)
   }
 
-  return cachedScript.replace('FT_VISITOR_DATA', visitorData)
+  return cachedScript.replace('FT_PARAMS', `"${videoId}","${visitorData}",${context}`)
 }

src/renderer/helpers/api/local.js

@@ -198,16 +198,24 @@ export async function getLocalSearchContinuation(continuationData) {
 export async function getLocalVideoInfo(id) {
   const webInnertube = await createInnertube({ withPlayer: true, generateSessionLocally: false })
 
-  let poToken
+  // based on the videoId (added to the body of the /player request)
+  let contentPoToken
+  // based on the visitor data (added to the streaming URLs)
+  let sessionPoToken
 
   if (process.env.IS_ELECTRON) {
     const { ipcRenderer } = require('electron')
 
     try {
-      poToken = await ipcRenderer.invoke(IpcChannels.GENERATE_PO_TOKEN, webInnertube.session.context.client.visitorData)
-
-      webInnertube.session.po_token = poToken
-      webInnertube.session.player.po_token = poToken
+      ({ contentPoToken, sessionPoToken } = await ipcRenderer.invoke(
+        IpcChannels.GENERATE_PO_TOKENS,
+        id,
+        webInnertube.session.context.client.visitorData,
+        JSON.stringify(webInnertube.session.context)
+      ))
+
+      webInnertube.session.po_token = contentPoToken
+      webInnertube.session.player.po_token = sessionPoToken
     } catch (error) {
       console.error('Local API, poToken generation failed', error)
       throw error
@@ -227,8 +235,8 @@ export async function getLocalVideoInfo(id) {
     const webEmbeddedInnertube = await createInnertube({ clientType: ClientType.WEB_EMBEDDED })
     webEmbeddedInnertube.session.context.client.visitorData = webInnertube.session.context.client.visitorData
 
-    if (poToken) {
-      webEmbeddedInnertube.session.po_token = poToken
+    if (contentPoToken) {
+      webEmbeddedInnertube.session.po_token = contentPoToken
     }
 
     const videoId = hasTrailer && trailerIsAgeRestricted ? info.playability_status.error_screen.video_id : id
@@ -283,9 +291,9 @@ export async function getLocalVideoInfo(id) {
       let url = info.streaming_data.dash_manifest_url
 
       if (url.includes('?')) {
-        url += `&pot=${encodeURIComponent(poToken)}&mpd_version=7`
+        url += `&pot=${encodeURIComponent(sessionPoToken)}&mpd_version=7`
       } else {
-        url += `${url.endsWith('/') ? '' : '/'}pot/${encodeURIComponent(poToken)}/mpd_version/7`
+        url += `${url.endsWith('/') ? '' : '/'}pot/${encodeURIComponent(sessionPoToken)}/mpd_version/7`
       }
 
       info.streaming_data.dash_manifest_url = url