From fe390f5374abda7dea7a79a1ca6ff4e99a22e478 Mon Sep 17 00:00:00 2001 From: absidue <48293849+absidue@users.noreply.github.com> Date: Sun, 2 Mar 2025 16:48:59 +0100 Subject: [PATCH 1/3] Implement content and session PO tokens --- src/botGuardScript.js | 91 +++++++++++++++++++++++++------ src/constants.js | 2 +- src/main/index.js | 4 +- src/main/poTokenGenerator.js | 86 ++++++++++++----------------- src/renderer/helpers/api/local.js | 26 ++++++--- 5 files changed, 130 insertions(+), 79 deletions(-) diff --git a/src/botGuardScript.js b/src/botGuardScript.js index 11da8855a6472..a9e95f134ecff 100644 --- a/src/botGuardScript.js +++ b/src/botGuardScript.js @@ -1,37 +1,94 @@ -import { BG } from 'bgutils-js' +import { BG, buildURL, GOOG_API_KEY } from 'bgutils-js' // This script has it's own webpack config, as it gets passed as a string to Electron's evaluateJavaScript function // in src/main/poTokenGenerator.js -export default async function(visitorData) { + +/** + * Based on: https://github.com/LuanRT/BgUtils/blob/main/examples/node/innertube-challenge-fetcher-example.ts + * @param {string} videoId + * @param {string} visitorData + * @param {import('youtubei.js').Session['context']} context + */ +export default async function (videoId, visitorData, context) { const requestKey = 'O43z0dpjhgX20SCx4KAo' - const bgConfig = { - fetch: (input, init) => fetch(input, init), - requestKey, - globalObj: window, - identifier: visitorData + const challengeResponse = await fetch( + 'https://www.youtube.com/youtubei/v1/att/get?prettyPrint=false&alt=json', + { + method: 'POST', + headers: { + Accept: '*/*', + 'Content-Type': 'application/json', + 'X-Goog-Visitor-Id': visitorData, + 'X-Youtube-Client-Version': context.client.clientVersion, + 'X-Youtube-Client-Name': '1' + }, + body: JSON.stringify({ + engagementType: 'ENGAGEMENT_TYPE_UNBOUND', + context + }), + } + ) + + if (!challengeResponse.ok) { + throw new Error(`Request to ${challengeResponse.url} failed with status ${challengeResponse.status}\n${await challengeResponse.text()}`) } - const challenge = await BG.Challenge.create(bgConfig) + const challengeData = await challengeResponse.json() - if (!challenge) { - throw new Error('Could not get challenge') + if (!challengeData.bgChallenge) { + throw new Error('Failed to get BotGuard challenge') } - const interpreterJavascript = challenge.interpreterJavascript.privateDoNotAccessOrElseSafeScriptWrappedValue + let interpreterUrl = challengeData.bgChallenge.interpreterUrl.privateDoNotAccessOrElseTrustedResourceUrlWrappedValue + + if (interpreterUrl.startsWith('//')) { + interpreterUrl = `https:${interpreterUrl}` + } + + const bgScriptResponse = await fetch(interpreterUrl) + const interpreterJavascript = await bgScriptResponse.text() if (interpreterJavascript) { // eslint-disable-next-line no-new-func new Function(interpreterJavascript)() } else { - console.warn('Unable to load VM.') + throw new Error('Could not load VM.') } - const poTokenResult = await BG.PoToken.generate({ - program: challenge.program, - globalName: challenge.globalName, - bgConfig + const botGuard = await BG.BotGuardClient.create({ + program: challengeData.bgChallenge.program, + globalName: challengeData.bgChallenge.globalName, + globalObj: window + }) + + // DO NOT REMOVE + // Without this timeout we get an "Async snapshot function not found" error + await new Promise((resolve) => setTimeout(resolve, 1000)) + + const webPoSignalOutput = [] + const botGuardResponse = await botGuard.snapshot({ webPoSignalOutput }) + + const integrityTokenResponse = await fetch(buildURL('GenerateIT', true), { + method: 'POST', + headers: { + 'content-type': 'application/json+protobuf', + 'x-goog-api-key': GOOG_API_KEY, + 'x-user-agent': 'grpc-web-javascript/0.1', + }, + body: JSON.stringify([requestKey, botGuardResponse]) }) - return poTokenResult.poToken + const response = await integrityTokenResponse.json() + + if (typeof response[0] !== 'string') { + throw new Error('Could not get integrity token') + } + + const integrityTokenBasedMinter = await BG.WebPoMinter.create({ integrityToken: response[0] }, webPoSignalOutput) + + const contentPoToken = await integrityTokenBasedMinter.mintAsWebsafeString(videoId) + const sessionPoToken = await integrityTokenBasedMinter.mintAsWebsafeString(visitorData) + + return { contentPoToken, sessionPoToken } } diff --git a/src/constants.js b/src/constants.js index 355fa8cebe38e..29e6c0f42b408 100644 --- a/src/constants.js +++ b/src/constants.js @@ -46,7 +46,7 @@ const IpcChannels = { SET_INVIDIOUS_AUTHORIZATION: 'set-invidious-authorization', - GENERATE_PO_TOKEN: 'generate-po-token', + GENERATE_PO_TOKENS: 'generate-po-tokens', CHOOSE_DEFAULT_FOLDER: 'choose-default-folder', WRITE_SCREENSHOT: 'write-screenshot', diff --git a/src/main/index.js b/src/main/index.js index 1d14f3450284a..4c9e2f0073d13 100644 --- a/src/main/index.js +++ b/src/main/index.js @@ -901,8 +901,8 @@ function runApp() { }) }) - ipcMain.handle(IpcChannels.GENERATE_PO_TOKEN, (_, visitorData) => { - return generatePoToken(visitorData, proxyUrl) + ipcMain.handle(IpcChannels.GENERATE_PO_TOKENS, (_, videoId, visitorData, context) => { + return generatePoToken(videoId, visitorData, context, proxyUrl) }) ipcMain.on(IpcChannels.ENABLE_PROXY, (_, url) => { diff --git a/src/main/poTokenGenerator.js b/src/main/poTokenGenerator.js index dd8850ed6276b..01211766aa27d 100644 --- a/src/main/poTokenGenerator.js +++ b/src/main/poTokenGenerator.js @@ -9,11 +9,13 @@ import { join } from 'path' * This is intentionally split out into it's own thing, with it's own temporary in-memory session, * as the BotGuard stuff accesses the global `document` and `window` objects and also requires making some requests. * So we definitely don't want it running in the same places as the rest of the FreeTube code with the user data. + * @param {string} videoId * @param {string} visitorData + * @param {string} context * @param {string|undefined} proxyUrl - * @returns {Promise} + * @returns {Promise<{ contentPoToken: string, sessionPoToken: string }>} */ -export async function generatePoToken(visitorData, proxyUrl) { +export async function generatePoToken(videoId, visitorData, context, proxyUrl) { const sessionUuid = crypto.randomUUID() const theSession = session.fromPartition(`potoken-${sessionUuid}`, { cache: false }) @@ -22,12 +24,7 @@ export async function generatePoToken(visitorData, proxyUrl) { // eslint-disable-next-line n/no-callback-literal theSession.setPermissionRequestHandler((webContents, permission, callback) => callback(false)) - theSession.setUserAgent( - theSession.getUserAgent() - .split(' ') - .filter(part => !part.includes('Electron')) - .join(' ') - ) + theSession.setUserAgent(session.defaultSession.getUserAgent()) if (proxyUrl) { await theSession.setProxy({ @@ -35,6 +32,27 @@ export async function generatePoToken(visitorData, proxyUrl) { }) } + theSession.webRequest.onBeforeSendHeaders({ + urls: ['https://www.google.com/js/*', 'https://www.youtube.com/youtubei/*'] + }, ({ requestHeaders, url }, callback) => { + if (url.startsWith('https://www.youtube.com/youtubei/')) { + // make InnerTube requests work with the fetch function + // InnerTube rejects requests if the referer isn't YouTube or empty + requestHeaders.Referer = 'https://www.youtube.com/' + requestHeaders.Origin = 'https://www.youtube.com' + + requestHeaders['Sec-Fetch-Site'] = 'same-origin' + requestHeaders['Sec-Fetch-Mode'] = 'same-origin' + requestHeaders['X-Youtube-Bootstrap-Logged-In'] = 'false' + } else { + requestHeaders['Sec-Fetch-Dest'] = 'script' + requestHeaders['Sec-Fetch-Site'] = 'cross-site' + requestHeaders['Accept-Language'] = '*' + } + + callback({ requestHeaders }) + }) + const webContentsView = new WebContentsView({ webPreferences: { backgroundThrottling: false, @@ -42,7 +60,8 @@ export async function generatePoToken(visitorData, proxyUrl) { sandbox: true, v8CacheOptions: 'none', session: theSession, - offscreen: true + offscreen: true, + webSecurity: false } }) @@ -58,43 +77,8 @@ export async function generatePoToken(visitorData, proxyUrl) { webContentsView.webContents.debugger.attach() - await webContentsView.webContents.loadURL('data:text/html,', { - baseURLForDataURL: 'https://www.youtube.com' - }) - - await webContentsView.webContents.debugger.sendCommand('Emulation.setUserAgentOverride', { - userAgent: theSession.getUserAgent(), - acceptLanguage: 'en-US', - platform: 'Win32', - userAgentMetadata: { - brands: [ - { - brand: 'Not/A)Brand', - version: '99' - }, - { - brand: 'Chromium', - version: process.versions.chrome.split('.')[0] - } - ], - fullVersionList: [ - { - brand: 'Not/A)Brand', - version: '99.0.0.0' - }, - { - brand: 'Chromium', - version: process.versions.chrome - } - ], - platform: 'Windows', - platformVersion: '10.0.0', - architecture: 'x86', - model: '', - mobile: false, - bitness: '64', - wow64: false - } + await webContentsView.webContents.loadURL('data:text/html,', { + baseURLForDataURL: 'https://www.youtube.com/' }) await webContentsView.webContents.debugger.sendCommand('Emulation.setDeviceMetricsOverride', { @@ -112,7 +96,7 @@ export async function generatePoToken(visitorData, proxyUrl) { } }) - const script = await getScript(visitorData) + const script = await getScript(videoId, visitorData, context) const response = await webContentsView.webContents.executeJavaScript(script) @@ -125,9 +109,11 @@ export async function generatePoToken(visitorData, proxyUrl) { let cachedScript /** + * @param {string} videoId * @param {string} visitorData + * @param {string} context */ -async function getScript(visitorData) { +async function getScript(videoId, visitorData, context) { if (!cachedScript) { const pathToScript = process.env.NODE_ENV === 'development' ? join(__dirname, '../../dist/botGuardScript.js') @@ -140,8 +126,8 @@ async function getScript(visitorData) { const functionName = match[1] - cachedScript = content.replace(match[0], `;${functionName}("FT_VISITOR_DATA")`) + cachedScript = content.replace(match[0], `;${functionName}(FT_PARAMS)`) } - return cachedScript.replace('FT_VISITOR_DATA', visitorData) + return cachedScript.replace('FT_PARAMS', `"${videoId}","${visitorData}",${context}`) } diff --git a/src/renderer/helpers/api/local.js b/src/renderer/helpers/api/local.js index 3c75d60d874c7..7f0e40421d110 100644 --- a/src/renderer/helpers/api/local.js +++ b/src/renderer/helpers/api/local.js @@ -197,16 +197,24 @@ export async function getLocalSearchContinuation(continuationData) { export async function getLocalVideoInfo(id) { const webInnertube = await createInnertube({ withPlayer: true, generateSessionLocally: false }) - let poToken + // based on the videoId (added to the body of the /player request) + let contentPoToken + // based on the visitor data (added to the streaming URLs) + let sessionPoToken if (process.env.IS_ELECTRON) { const { ipcRenderer } = require('electron') try { - poToken = await ipcRenderer.invoke(IpcChannels.GENERATE_PO_TOKEN, webInnertube.session.context.client.visitorData) - - webInnertube.session.po_token = poToken - webInnertube.session.player.po_token = poToken + ({ contentPoToken, sessionPoToken } = await ipcRenderer.invoke( + IpcChannels.GENERATE_PO_TOKENS, + id, + webInnertube.session.context.client.visitorData, + JSON.stringify(webInnertube.session.context) + )) + + webInnertube.session.po_token = contentPoToken + webInnertube.session.player.po_token = sessionPoToken } catch (error) { console.error('Local API, poToken generation failed', error) throw error @@ -226,8 +234,8 @@ export async function getLocalVideoInfo(id) { const webEmbeddedInnertube = await createInnertube({ clientType: ClientType.WEB_EMBEDDED }) webEmbeddedInnertube.session.context.client.visitorData = webInnertube.session.context.client.visitorData - if (poToken) { - webEmbeddedInnertube.session.po_token = poToken + if (contentPoToken) { + webEmbeddedInnertube.session.po_token = contentPoToken } const videoId = hasTrailer && trailerIsAgeRestricted ? info.playability_status.error_screen.video_id : id @@ -277,9 +285,9 @@ export async function getLocalVideoInfo(id) { let url = info.streaming_data.dash_manifest_url if (url.includes('?')) { - url += `&pot=${encodeURIComponent(poToken)}&mpd_version=7` + url += `&pot=${encodeURIComponent(sessionPoToken)}&mpd_version=7` } else { - url += `${url.endsWith('/') ? '' : '/'}pot/${encodeURIComponent(poToken)}/mpd_version/7` + url += `${url.endsWith('/') ? '' : '/'}pot/${encodeURIComponent(sessionPoToken)}/mpd_version/7` } info.streaming_data.dash_manifest_url = url From f757fd7cfe54c2782b9fa521e8354a7df3b408dd Mon Sep 17 00:00:00 2001 From: absidue <48293849+absidue@users.noreply.github.com> Date: Mon, 3 Mar 2025 23:16:34 +0100 Subject: [PATCH 2/3] Remove timeout after BgUtils update --- src/botGuardScript.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/botGuardScript.js b/src/botGuardScript.js index a9e95f134ecff..f2f1a9dc52cf9 100644 --- a/src/botGuardScript.js +++ b/src/botGuardScript.js @@ -62,10 +62,6 @@ export default async function (videoId, visitorData, context) { globalObj: window }) - // DO NOT REMOVE - // Without this timeout we get an "Async snapshot function not found" error - await new Promise((resolve) => setTimeout(resolve, 1000)) - const webPoSignalOutput = [] const botGuardResponse = await botGuard.snapshot({ webPoSignalOutput }) From 9c5e98c949bff05252fe1b7e9aa543563185f5ce Mon Sep 17 00:00:00 2001 From: absidue <48293849+absidue@users.noreply.github.com> Date: Mon, 3 Mar 2025 23:32:49 +0100 Subject: [PATCH 3/3] Trigger GitHub CI