diff --git a/package-lock.json b/package-lock.json index 9318920..1f224fc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3360,6 +3360,12 @@ "@types/pg": "*" } }, + "node_modules/@types/proxyquire": { + "version": "1.3.31", + "resolved": "https://registry.npmjs.org/@types/proxyquire/-/proxyquire-1.3.31.tgz", + "integrity": "sha512-uALowNG2TSM1HNPMMOR0AJwv4aPYPhqB0xlEhkeRTMuto5hjoSPZkvgu1nbPUkz3gEPAHv4sy4DmKsurZiEfRQ==", + "dev": true + }, "node_modules/@types/qs": { "version": "6.9.7", "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.7.tgz", @@ -10935,6 +10941,7 @@ "@types/chai": "^4.3.4", "@types/express": "^4.17.17", "@types/node": "^18.15.10", + "@types/proxyquire": "^1.3.31", "@types/sinon": "^10.0.16", "@types/supertest": "^2.0.12", "chai": "^4.3.7", @@ -10952,7 +10959,7 @@ "packages/synthetics-sdk-broken-links/node_modules/@google-cloud/synthetics-sdk-api": { "version": "0.5.1", "resolved": "file:packages/synthetics-sdk-broken-links/google-cloud-synthetics-sdk-api-0.5.1.tgz", - "integrity": "sha512-gNUsbNYcBYt/lLSdCmx8YJWA2Tqb9THrrQrKuUtb1pAnwy1M+pukfYoUztp6rX8jVsdj2/lVgN6iMx9ARLfM0g==", + "integrity": "sha512-kCevsvEiS6DzeRJmcwoY0RvlOm4tvwGxMUGFw0H14ELTUW8QoY6Fj5paLZgSlmC9wNQ5IZZE9g3kNdJnNicm+Q==", "license": "Apache-2.0", "dependencies": { "@google-cloud/opentelemetry-cloud-trace-exporter": "2.1.0", @@ -12275,6 +12282,7 @@ "@types/chai": "^4.3.4", "@types/express": "^4.17.17", "@types/node": "^18.15.10", + "@types/proxyquire": "^1.3.31", "@types/sinon": "^10.0.16", "@types/supertest": "^2.0.12", "chai": "^4.3.7", @@ -12289,7 +12297,7 @@ "dependencies": { "@google-cloud/synthetics-sdk-api": { "version": "file:google-cloud-synthetics-sdk-api-0.5.1.tgz", - "integrity": "sha512-gNUsbNYcBYt/lLSdCmx8YJWA2Tqb9THrrQrKuUtb1pAnwy1M+pukfYoUztp6rX8jVsdj2/lVgN6iMx9ARLfM0g==", + "integrity": "sha512-kCevsvEiS6DzeRJmcwoY0RvlOm4tvwGxMUGFw0H14ELTUW8QoY6Fj5paLZgSlmC9wNQ5IZZE9g3kNdJnNicm+Q==", "requires": { "@google-cloud/opentelemetry-cloud-trace-exporter": "2.1.0", "@opentelemetry/api": "1.6.0", @@ -14437,6 +14445,12 @@ "@types/pg": "*" } }, + "@types/proxyquire": { + "version": "1.3.31", + "resolved": "https://registry.npmjs.org/@types/proxyquire/-/proxyquire-1.3.31.tgz", + "integrity": "sha512-uALowNG2TSM1HNPMMOR0AJwv4aPYPhqB0xlEhkeRTMuto5hjoSPZkvgu1nbPUkz3gEPAHv4sy4DmKsurZiEfRQ==", + "dev": true + }, "@types/qs": { "version": "6.9.7", "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.7.tgz", @@ -19031,6 +19045,7 @@ "@types/chai": "^4.3.4", "@types/express": "^4.17.17", "@types/node": "^18.15.10", + "@types/proxyquire": "^1.3.31", "@types/sinon": "^10.0.16", "@types/supertest": "^2.0.12", "chai": "^4.3.7", @@ -19045,7 +19060,7 @@ "dependencies": { "@google-cloud/synthetics-sdk-api": { "version": "file:google-cloud-synthetics-sdk-api-0.5.1.tgz", - "integrity": "sha512-gNUsbNYcBYt/lLSdCmx8YJWA2Tqb9THrrQrKuUtb1pAnwy1M+pukfYoUztp6rX8jVsdj2/lVgN6iMx9ARLfM0g==", + "integrity": "sha512-kCevsvEiS6DzeRJmcwoY0RvlOm4tvwGxMUGFw0H14ELTUW8QoY6Fj5paLZgSlmC9wNQ5IZZE9g3kNdJnNicm+Q==", "requires": { "@google-cloud/opentelemetry-cloud-trace-exporter": "2.1.0", "@opentelemetry/api": "1.6.0", diff --git a/packages/synthetics-sdk-broken-links/google-cloud-synthetics-sdk-api-0.5.1.tgz b/packages/synthetics-sdk-broken-links/google-cloud-synthetics-sdk-api-0.5.1.tgz index 81d4a1b..8e9d4a8 100644 Binary files a/packages/synthetics-sdk-broken-links/google-cloud-synthetics-sdk-api-0.5.1.tgz and b/packages/synthetics-sdk-broken-links/google-cloud-synthetics-sdk-api-0.5.1.tgz differ diff --git a/packages/synthetics-sdk-broken-links/package.json b/packages/synthetics-sdk-broken-links/package.json index 669700f..ad7a051 100644 --- a/packages/synthetics-sdk-broken-links/package.json +++ b/packages/synthetics-sdk-broken-links/package.json @@ -27,12 +27,14 @@ "@types/chai": "^4.3.4", "@types/express": "^4.17.17", "@types/node": "^18.15.10", + "@types/proxyquire": "^1.3.31", "@types/sinon": "^10.0.16", "@types/supertest": "^2.0.12", "chai": "^4.3.7", "chai-exclude": "^2.1.0", "express": "^4.18.2", "proxyquire": "^2.1.3", + "node-mocks-http": "^1.13.0", "sinon": "^16.1.1", "supertest": "^6.3.3", "synthetics-sdk-broken-links": "file:./" diff --git a/packages/synthetics-sdk-broken-links/src/broken_links.ts b/packages/synthetics-sdk-broken-links/src/broken_links.ts index a99650e..c7a899f 100644 --- a/packages/synthetics-sdk-broken-links/src/broken_links.ts +++ b/packages/synthetics-sdk-broken-links/src/broken_links.ts @@ -40,6 +40,7 @@ import { processOptions } from './options_func'; import { createStorageClientIfStorageSelected, getOrCreateStorageBucket, + StorageParameters, } from './storage_func'; // External Dependencies @@ -100,7 +101,11 @@ try { instantiateMetadata(synthetics_sdk_broken_links_package); export async function runBrokenLinks( - inputOptions: BrokenLinkCheckerOptions + inputOptions: BrokenLinkCheckerOptions, + args: { + executionId: string | undefined; + checkId: string | undefined; + } ): Promise { // init const startTime = new Date().toISOString(); @@ -112,6 +117,11 @@ export async function runBrokenLinks( const adjusted_synthetic_timeout_millis = options.total_synthetic_timeout_millis! - 7000; + // Create Promise and variables used to set and resolve the time limit + // imposed by `adjusted_synthetic_timeout` + const [timeLimitPromise, timeLimitTimeout, timeLimitresolver] = + getTimeLimitPromise(startTime, adjusted_synthetic_timeout_millis); + const errors: BaseError[] = []; // Initialize Storage Client with Error Handling. Set to `null` if @@ -121,18 +131,20 @@ export async function runBrokenLinks( options.screenshot_options!.capture_condition ); - // TODO. Just to show where this will be called. uncommented in next PR - // Bucket Validation - // const bucket: Bucket | null = await getOrCreateStorageBucket( - // storageClient, - // options.screenshot_options!.storage_location, - // errors - // ); + // // Bucket Validation + const bucket: Bucket | null = await getOrCreateStorageBucket( + storageClient, + options.screenshot_options!.storage_location, + errors + ); - // Create Promise and variables used to set and resolve the time limit - // imposed by `adjusted_synthetic_timeout` - const [timeLimitPromise, timeLimitTimeout, timeLimitresolver] = - getTimeLimitPromise(startTime, adjusted_synthetic_timeout_millis); + const storageParams: StorageParameters = { + storageClient: storageClient, + bucket: bucket, + checkId: args.checkId || '_', + executionId: args.executionId || '_', + screenshotNumber: 1, + }; const followed_links: BrokenLinksResultV1_SyntheticLinkResult[] = []; @@ -147,7 +159,8 @@ export async function runBrokenLinks( originPage, options, startTime, - adjusted_synthetic_timeout_millis + adjusted_synthetic_timeout_millis, + storageParams ) ); @@ -169,7 +182,8 @@ export async function runBrokenLinks( linksToFollow, options, startTime, - adjusted_synthetic_timeout_millis + adjusted_synthetic_timeout_millis, + storageParams )) ); return true; @@ -188,6 +202,7 @@ export async function runBrokenLinks( runtime_metadata, options, followed_links, + storageParams, errors ); } catch (err) { @@ -215,7 +230,8 @@ async function checkOriginLink( originPage: Page, options: BrokenLinksResultV1_BrokenLinkCheckerOptions, startTime: string, - adjusted_synthetic_timeout_millis: number + adjusted_synthetic_timeout_millis: number, + storageParams: StorageParameters ): Promise { let originLinkResult: BrokenLinksResultV1_SyntheticLinkResult; @@ -232,6 +248,7 @@ async function checkOriginLink( originPage, { target_uri: options.origin_uri, anchor_text: '', html_element: '' }, options, + storageParams, true ); diff --git a/packages/synthetics-sdk-broken-links/src/handlers.ts b/packages/synthetics-sdk-broken-links/src/handlers.ts index fff255a..b45cacc 100644 --- a/packages/synthetics-sdk-broken-links/src/handlers.ts +++ b/packages/synthetics-sdk-broken-links/src/handlers.ts @@ -18,6 +18,9 @@ import { Request, Response } from 'express'; // Internal Project Files import { runBrokenLinks, BrokenLinkCheckerOptions } from './broken_links'; +const syntheticExecutionIdHeader = 'Synthetic-Execution-Id'; +const checkIdHeader = 'Check-Id'; + /** * Middleware for easy invocation of SyntheticSDK broken links, and may be used to * register a GoogleCloudFunction http function, or express js compatible handler. @@ -29,5 +32,10 @@ import { runBrokenLinks, BrokenLinkCheckerOptions } from './broken_links'; export function runBrokenLinksHandler(options: BrokenLinkCheckerOptions) { // eslint-disable-next-line @typescript-eslint/no-explicit-any return async (req: Request, res: Response): Promise => - res.send(await runBrokenLinks(options)); + res.send( + await runBrokenLinks(options, { + executionId: req.get(syntheticExecutionIdHeader), + checkId: req.get(checkIdHeader), + }) + ); } diff --git a/packages/synthetics-sdk-broken-links/src/link_utils.ts b/packages/synthetics-sdk-broken-links/src/link_utils.ts index 2d7d98e..b90412c 100644 --- a/packages/synthetics-sdk-broken-links/src/link_utils.ts +++ b/packages/synthetics-sdk-broken-links/src/link_utils.ts @@ -12,12 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Standard Libraries +import * as path from 'path'; + // Internal Project Files import { BaseError, BrokenLinksResultV1, BrokenLinksResultV1_BrokenLinkCheckerOptions, BrokenLinksResultV1_BrokenLinkCheckerOptions_LinkOrder, + BrokenLinksResultV1_BrokenLinkCheckerOptions_ScreenshotOptions_CaptureCondition as ApiCaptureCondition, BrokenLinksResultV1_SyntheticLinkResult, GenericResultV1, getRuntimeMetadata, @@ -28,6 +32,7 @@ import { // External Dependencies import { HTTPResponse } from 'puppeteer'; +import { StorageParameters } from './storage_func'; /** * Represents an intermediate link with its properties. @@ -157,7 +162,7 @@ function parseFollowedLinks( options: {} as BrokenLinksResultV1_BrokenLinkCheckerOptions, origin_link_result: {} as BrokenLinksResultV1_SyntheticLinkResult, followed_link_results: [], - execution_data_storage_path: '', // TODO: make sure that when this is set it begins with gs:// + execution_data_storage_path: '', errors: [], }; @@ -223,6 +228,7 @@ export function createSyntheticResult( runtime_metadata: { [key: string]: string }, options: BrokenLinksResultV1_BrokenLinkCheckerOptions, followed_links: BrokenLinksResultV1_SyntheticLinkResult[], + storageParams: StorageParameters, errors: BaseError[] ): SyntheticResult { // Create BrokenLinksResultV1 by parsing followed links and setting options @@ -230,6 +236,12 @@ export function createSyntheticResult( parseFollowedLinks(followed_links); broken_links_result.options = options; broken_links_result.errors = errors; + broken_links_result.execution_data_storage_path = storageParams.bucket + ? 'gs://' + + storageParams.bucket.name + + '/' + + getStoragePathToExecution(storageParams, options) + : ''; // Create SyntheticResult object const synthetic_result: SyntheticResult = { @@ -273,6 +285,26 @@ export function shuffleAndTruncate( } /** + * Determines whether a screenshot should be taken based on screenshot options and link result. + * + * @param options - BrokenLinksResultV1_BrokenLinkCheckerOptions + * @param passed - boolean indicating whether the link navigation succeeded + * @returns true if a screenshot should be taken, false otherwise + */ +export function shouldTakeScreenshot( + options: BrokenLinksResultV1_BrokenLinkCheckerOptions, + passed: boolean +): boolean { + return ( + options.screenshot_options!.capture_condition === ApiCaptureCondition.ALL || + (options.screenshot_options!.capture_condition === + ApiCaptureCondition.FAILING && + !passed) + ); +} + +/** + * Sanitizes an object name string for safe use, ensuring compliance with * naming restrictions. * @@ -303,6 +335,38 @@ export function sanitizeObjectName( .replace(/\s+/g, '_'); // Replace one or more spaces with underscores } +export function getStoragePathToExecution( + storageParams: StorageParameters, + options: BrokenLinksResultV1_BrokenLinkCheckerOptions +) { + try { + const storageLocation = options.screenshot_options!.storage_location; + let writeDestination = ''; + + // extract folder name for a given storage location. If there is no '/' + // present then the storageLocation is just a folder + const firstSlashIndex = storageLocation.indexOf('/'); + if (firstSlashIndex !== -1) { + writeDestination = storageLocation.substring(firstSlashIndex + 1); + } + + // Ensure writeDestination ends with a slash for proper path joining + if (writeDestination && !writeDestination.endsWith('/')) { + writeDestination += '/'; + } + + writeDestination = path.join( + writeDestination, + storageParams.checkId, + storageParams.executionId + ); + + return writeDestination; + } catch (err) { + return ''; + } +} + export function getTimeLimitPromise( startTime: string, totalTimeoutMillis: number, diff --git a/packages/synthetics-sdk-broken-links/src/navigation_func.ts b/packages/synthetics-sdk-broken-links/src/navigation_func.ts index d7d856a..dcca8fd 100644 --- a/packages/synthetics-sdk-broken-links/src/navigation_func.ts +++ b/packages/synthetics-sdk-broken-links/src/navigation_func.ts @@ -19,6 +19,7 @@ import { BrokenLinksResultV1_SyntheticLinkResult, ResponseStatusCode, ResponseStatusCode_StatusClass, + BrokenLinksResultV1_SyntheticLinkResult_ScreenshotOutput as ApiScreenshotOutput, } from '@google-cloud/synthetics-sdk-api'; import { checkStatusPassing, @@ -27,7 +28,9 @@ import { LinkIntermediate, NavigateResponse, getTimeLimitPromise, + shouldTakeScreenshot, } from './link_utils'; +import { StorageParameters, uploadScreenshotToGCS } from './storage_func'; // External Dependencies import { Browser, HTTPResponse, Page } from 'puppeteer'; @@ -104,7 +107,8 @@ export async function checkLinks( links: LinkIntermediate[], options: BrokenLinksResultV1_BrokenLinkCheckerOptions, startTime: string, - total_timeout_millis: number + total_timeout_millis: number, + storageParams: StorageParameters ): Promise { let timeLimitReached = false; const followed_links: BrokenLinksResultV1_SyntheticLinkResult[] = []; @@ -120,7 +124,9 @@ export async function checkLinks( if (timeLimitReached) return false; try { - followed_links.push(await checkLink(page, link, options)); + followed_links.push( + await checkLink(page, link, options, storageParams) + ); /** In the case of a single page app, network requests can hang and cause * timeout issues in following links. To ensure this does not happen we * need to reset the page in between every link checked @@ -165,6 +171,7 @@ export async function checkLink( page: Page, link: LinkIntermediate, options: BrokenLinksResultV1_BrokenLinkCheckerOptions, + storageParams: StorageParameters, isOrigin = false ): Promise { // Determine the expected status code for the link, using per-link setting if @@ -185,6 +192,18 @@ export async function checkLink( linkEndTime, } = await navigate(page, link, options, expectedStatusCode); + let screenshotOutput: ApiScreenshotOutput = { + screenshot_file: '', + screenshot_error: {} as BaseError, + }; + if (shouldTakeScreenshot(options, passed)) { + screenshotOutput = await uploadScreenshotToGCS( + page, + storageParams, + options + ); + } + // Initialize variables for error information let errorType = ''; let errorMessage = ''; @@ -224,10 +243,7 @@ export async function checkLink( link_start_time: linkStartTime, link_end_time: linkEndTime, is_origin: isOrigin, - screenshot_output: { - screenshot_file: '', - screenshot_error: {} as BaseError, - }, // TODO: this is temporary in an effort to make PRs more manageable + screenshot_output: screenshotOutput, }; } diff --git a/packages/synthetics-sdk-broken-links/src/options_func.ts b/packages/synthetics-sdk-broken-links/src/options_func.ts index 1b0a61d..b8b67f9 100644 --- a/packages/synthetics-sdk-broken-links/src/options_func.ts +++ b/packages/synthetics-sdk-broken-links/src/options_func.ts @@ -290,7 +290,7 @@ export function setDefaultOptions( defaultOptions.screenshot_options!.capture_condition; } - if (outputOptions.screenshot_options?.storage_location) { + if (inputOptions.screenshot_options?.storage_location) { outputOptions.screenshot_options.storage_location = inputOptions.screenshot_options!.storage_location!; } else { diff --git a/packages/synthetics-sdk-broken-links/src/storage_func.ts b/packages/synthetics-sdk-broken-links/src/storage_func.ts index a483410..524919b 100644 --- a/packages/synthetics-sdk-broken-links/src/storage_func.ts +++ b/packages/synthetics-sdk-broken-links/src/storage_func.ts @@ -24,16 +24,18 @@ import { getExecutionRegion, resolveProjectId, } from '@google-cloud/synthetics-sdk-api'; -import { sanitizeObjectName } from './link_utils'; +import { getStoragePathToExecution, sanitizeObjectName } from './link_utils'; // External Dependencies import { Storage, Bucket } from '@google-cloud/storage'; +import { Page } from 'puppeteer'; export interface StorageParameters { storageClient: Storage | null; bucket: Bucket | null; - uptimeId: string; + checkId: string; executionId: string; + screenshotNumber: number; } /** @@ -89,12 +91,16 @@ export async function getOrCreateStorageBucket( }); } } catch (err) { - if (err instanceof Error) process.stderr.write(err.message); + const errorType = storageLocation + ? 'StorageValidationError' + : 'BucketCreationError'; + + // Using console.error rather than stderr.write since err type is unknown + console.error(errorType, err); + errors.push({ // General error handling - error_type: storageLocation - ? 'StorageValidationError' - : 'BucketCreationError', + error_type: errorType, error_message: `Failed to ${ storageLocation ? 'validate' : 'create' } bucket ${bucketName}. Please reference server logs for further information.`, @@ -120,7 +126,8 @@ export function createStorageClientIfStorageSelected( try { return new Storage(); } catch (err) { - if (err instanceof Error) process.stderr.write(err.message); + console.error('StorageClientInitializationError', err); + errors.push({ error_type: 'StorageClientInitializationError', error_message: @@ -140,8 +147,7 @@ export function createStorageClientIfStorageSelected( * @returns An ApiScreenshotOutput object indicating success or a screenshot_error. */ export async function uploadScreenshotToGCS( - screenshot: string, - filename: string, + page: Page, storageParams: StorageParameters, options: BrokenLinksResultV1_BrokenLinkCheckerOptions ): Promise { @@ -155,22 +161,14 @@ export async function uploadScreenshotToGCS( return screenshot_output; } - // Construct the destination path within the bucket if given - let writeDestination = options.screenshot_options!.storage_location - ? getFolderNameFromStorageLocation( - options.screenshot_options!.storage_location - ) - : ''; - - // Ensure writeDestination ends with a slash for proper path joining - if (writeDestination && !writeDestination.endsWith('/')) { - writeDestination += '/'; - } + const screenshot: Buffer = await page.screenshot({ + fullPage: true, + encoding: 'binary', + }); + const filename = 'screenshot_' + storageParams.screenshotNumber + '.png'; - writeDestination = path.join( - writeDestination, - storageParams.uptimeId, - storageParams.executionId, + const writeDestination = path.join( + getStoragePathToExecution(storageParams, options), filename ); @@ -179,27 +177,16 @@ export async function uploadScreenshotToGCS( contentType: 'image/png', }); - screenshot_output.screenshot_file = writeDestination; + storageParams.screenshotNumber += 1; + screenshot_output.screenshot_file = filename; } catch (err) { - // Handle upload errors - if (err instanceof Error) process.stderr.write(err.message); + console.error('ScreenshotFileUploadError', err); + screenshot_output.screenshot_error = { - error_type: 'StorageFileUploadError', - error_message: `Failed to upload screenshot for ${filename}. Please reference server logs for further information.`, + error_type: 'ScreenshotFileUploadError', + error_message: `Failed to take and/or upload screenshot for ${await page.url()}. Please reference server logs for further information.`, }; } return screenshot_output; } - -// Helper function to extract folder name for a given storage location. If there -// is no '/' present then the storageLocation is just a folder -export function getFolderNameFromStorageLocation( - storageLocation: string -): string { - const firstSlashIndex = storageLocation.indexOf('/'); - if (firstSlashIndex === -1) { - return ''; - } - return storageLocation.substring(firstSlashIndex + 1); -} diff --git a/packages/synthetics-sdk-broken-links/test/example_html_files/integration_server.js b/packages/synthetics-sdk-broken-links/test/example_html_files/integration_server.js index d63ecd3..5e57389 100644 --- a/packages/synthetics-sdk-broken-links/test/example_html_files/integration_server.js +++ b/packages/synthetics-sdk-broken-links/test/example_html_files/integration_server.js @@ -31,5 +31,8 @@ functions.http('BrokenLinksEmptyPageOk', SyntheticsSdkBrokenLinks.runBrokenLinks origin_uri: `file:${path.join( __dirname, '../example_html_files/200.html' - )}` + )}`, + screenshot_options: { + capture_condition: 'NONE' + } })); diff --git a/packages/synthetics-sdk-broken-links/test/integration/integration.spec.ts b/packages/synthetics-sdk-broken-links/test/integration/integration.spec.ts index aefb918..93a24ff 100644 --- a/packages/synthetics-sdk-broken-links/test/integration/integration.spec.ts +++ b/packages/synthetics-sdk-broken-links/test/integration/integration.spec.ts @@ -36,9 +36,9 @@ describe('CloudFunctionV2 Running Broken Link Synthetics', async () => { const status_class_2xx = { status_class: ResponseStatusCode_StatusClass.STATUS_CLASS_2XX, }; - const defaultScreenshotOptions: BrokenLinksResultV1_BrokenLinkCheckerOptions_ScreenshotOptions = + const noneCaptureScreenshotOptions: BrokenLinksResultV1_BrokenLinkCheckerOptions_ScreenshotOptions = { - capture_condition: ApiCaptureCondition.FAILING, + capture_condition: ApiCaptureCondition.NONE, storage_location: '', }; @@ -96,7 +96,7 @@ describe('CloudFunctionV2 Running Broken Link Synthetics', async () => { wait_for_selector: '', per_link_options: {}, total_synthetic_timeout_millis: 60000, - screenshot_options: defaultScreenshotOptions, + screenshot_options: noneCaptureScreenshotOptions, }); expect(origin_link) diff --git a/packages/synthetics-sdk-broken-links/test/unit/broken_links.spec.ts b/packages/synthetics-sdk-broken-links/test/unit/broken_links.spec.ts index 5ee1593..0599d4f 100644 --- a/packages/synthetics-sdk-broken-links/test/unit/broken_links.spec.ts +++ b/packages/synthetics-sdk-broken-links/test/unit/broken_links.spec.ts @@ -17,11 +17,11 @@ import { expect, use } from 'chai'; import chaiExclude from 'chai-exclude'; use(chaiExclude); const path = require('path'); +import sinon from 'sinon'; // Internal Project Files import { BaseError, - BrokenLinksResultV1_BrokenLinkCheckerOptions, BrokenLinksResultV1_BrokenLinkCheckerOptions_LinkOrder, BrokenLinksResultV1_BrokenLinkCheckerOptions_ScreenshotOptions, BrokenLinksResultV1_BrokenLinkCheckerOptions_ScreenshotOptions_CaptureCondition as ApiCaptureCondition, @@ -33,8 +33,16 @@ import { import { runBrokenLinks, BrokenLinkCheckerOptions, + CaptureCondition, } from '../../src/broken_links'; +// External Dependencies +const proxyquire = require('proxyquire'); +import { Page } from 'puppeteer'; +import { Bucket, Storage } from '@google-cloud/storage'; + +const TEST_BUCKET_NAME = 'gcm-test-project-id-synthetics-test-region'; + describe('runBrokenLinks', async () => { const status_class_2xx: ResponseStatusCode = { status_class: ResponseStatusCode_StatusClass.STATUS_CLASS_2XX, @@ -44,19 +52,59 @@ describe('runBrokenLinks', async () => { capture_condition: ApiCaptureCondition.FAILING, storage_location: '', }; - - const defaultScreenshotOutput: ApiScreenshotOutput = - { - screenshot_file: '', - screenshot_error: {} as BaseError, - }; + const emptyScreenshotOutput: ApiScreenshotOutput = { + screenshot_file: '', + screenshot_error: {} as BaseError, + }; + const successfulScreenshotOuput: ApiScreenshotOutput = { + screenshot_file: 'bucket/folder/file.png', + screenshot_error: {} as BaseError, + }; + const args = { checkId: 'test-check-id', executionId: 'test-execution-id' }; + + const mockedstorageFunc = proxyquire('../../src/storage_func', { + '@google-cloud/synthetics-sdk-api': { + getExecutionRegion: () => 'test-region', + resolveProjectId: () => 'test-project-id', + }, + }); + + const mockedNavigationFunc = proxyquire('../../src/navigation_func', { + './storage_func': { + uploadScreenshotToGCS: () => successfulScreenshotOuput, + }, + }); + + let storageClientStub: sinon.SinonStubbedInstance; + let bucketStub: sinon.SinonStubbedInstance; + let pageStub: sinon.SinonStubbedInstance; + beforeEach(() => { + // Stub a storage bucket + bucketStub = sinon.createStubInstance(Bucket); + bucketStub.name = TEST_BUCKET_NAME; + bucketStub.create.resolves([bucketStub]); + // Simulate default_bucket not existing initially + bucketStub.exists.resolves([false]); // Simulate the bucket not existing initially + + // Stub the storage client + storageClientStub = sinon.createStubInstance(Storage); + storageClientStub.bucket.returns(bucketStub); + + // Stub a puppeteer page to return set Buffer when .screenshot() called + pageStub = sinon.createStubInstance(Page); + pageStub.screenshot.resolves(Buffer.from('screenshot-image-data', 'utf-8')); + }); + + afterEach(() => { + sinon.restore(); + }); it('Exits early when options cannot be parsed', async () => { const inputOptions: BrokenLinkCheckerOptions = { origin_uri: 'uri-does-not-start-with-http', }; - const result = await runBrokenLinks(inputOptions); + const result = await runBrokenLinks(inputOptions, args); const genericResult = result.synthetic_generic_result_v1; @@ -69,6 +117,16 @@ describe('runBrokenLinks', async () => { }).timeout(15000); it('returns broken_links_result with origin link failure when waitForSelector exceeds deadline', async () => { + const mockedBlc = proxyquire('../../src/broken_links', { + './storage_func': { + ...mockedstorageFunc, + createStorageClientIfStorageSelected: () => storageClientStub, + }, + './navigation_func': { + ...mockedNavigationFunc, + }, + }); + const origin_uri = `file:${path.join( __dirname, '../example_html_files/retrieve_links_test.html' @@ -77,9 +135,10 @@ describe('runBrokenLinks', async () => { origin_uri: origin_uri, wait_for_selector: 'not_present', link_timeout_millis: 3001, + screenshot_options: { capture_condition: CaptureCondition.NONE }, }; - const result = await runBrokenLinks(inputOptions); + const result = await mockedBlc.runBrokenLinks(inputOptions, args); const broken_links_result = result?.synthetic_broken_links_result_v1; const origin_link = broken_links_result?.origin_link_result; @@ -93,6 +152,16 @@ describe('runBrokenLinks', async () => { }).timeout(15000); it('Global timeout occurs during checkOriginLink waiting for `wait_for_selector', async () => { + const mockedBlc = proxyquire('../../src/broken_links', { + './storage_func': { + ...mockedstorageFunc, + createStorageClientIfStorageSelected: () => storageClientStub, + }, + './navigation_func': { + ...mockedNavigationFunc, + }, + }); + const origin_uri = `file:${path.join( __dirname, '../example_html_files/retrieve_links_test.html' @@ -104,8 +173,9 @@ describe('runBrokenLinks', async () => { wait_for_selector: 'none existent', link_timeout_millis: 35000, total_synthetic_timeout_millis: 31000, + screenshot_options: { capture_condition: CaptureCondition.NONE }, }; - const result = await runBrokenLinks(inputOptions); + const result = await mockedBlc.runBrokenLinks(inputOptions, args); const broken_links_result = result.synthetic_broken_links_result_v1; const expectedOriginLinkResult: BrokenLinksResultV1_SyntheticLinkResult = { @@ -122,10 +192,7 @@ describe('runBrokenLinks', async () => { link_start_time: 'NA', link_end_time: 'NA', is_origin: true, - screenshot_output: { - screenshot_file: '', - screenshot_error: {} as BaseError, - }, + screenshot_output: emptyScreenshotOutput, }; expect(broken_links_result?.origin_link_result) @@ -134,17 +201,17 @@ describe('runBrokenLinks', async () => { expect(broken_links_result?.followed_link_results.length).to.equal(0); }).timeout(40000); - it('Handles error when trying to visit page that does not exist', async () => { const origin_uri = `file:${path.join( __dirname, '../example_html_files/file_doesnt_exist.html' )}`; - const inputOptions : BrokenLinkCheckerOptions = { - origin_uri: origin_uri + const inputOptions: BrokenLinkCheckerOptions = { + origin_uri: origin_uri, + screenshot_options: { capture_condition: CaptureCondition.NONE }, }; - const result = await runBrokenLinks(inputOptions); + const result = await runBrokenLinks(inputOptions, args); const broken_links_result = result?.synthetic_broken_links_result_v1; const origin_link = broken_links_result?.origin_link_result; @@ -177,14 +244,23 @@ describe('runBrokenLinks', async () => { link_start_time: 'NA', link_end_time: 'NA', is_origin: true, - screenshot_output: defaultScreenshotOutput, + screenshot_output: emptyScreenshotOutput, }); expect(followed_links).to.deep.equal([]); }).timeout(10000); - it('Completes a full failing execution (1 failing link)', async () => { + const mockedBlc = proxyquire('../../src/broken_links', { + './storage_func': { + ...mockedstorageFunc, + createStorageClientIfStorageSelected: () => storageClientStub, + }, + './navigation_func': { + ...mockedNavigationFunc, + }, + }); + const origin_uri = `file:${path.join( __dirname, '../example_html_files/retrieve_links_test.html' @@ -194,9 +270,12 @@ describe('runBrokenLinks', async () => { query_selector_all: 'a[src], img[href]', get_attributes: ['href', 'src'], wait_for_selector: '', + screenshot_options: { + capture_condition: CaptureCondition.FAILING, + }, }; - const result = await runBrokenLinks(inputOptions); + const result = await mockedBlc.runBrokenLinks(inputOptions, args); const broken_links_result = result?.synthetic_broken_links_result_v1; const options = broken_links_result?.options; @@ -245,11 +324,14 @@ describe('runBrokenLinks', async () => { link_start_time: 'NA', link_end_time: 'NA', is_origin: true, - screenshot_output: defaultScreenshotOutput, + screenshot_output: emptyScreenshotOutput, }); - const sorted_followed_links = followed_links?.sort((a, b) => - a.target_uri.localeCompare(b.target_uri) + const sorted_followed_links = followed_links?.sort( + ( + a: BrokenLinksResultV1_SyntheticLinkResult, + b: BrokenLinksResultV1_SyntheticLinkResult + ) => a.target_uri.localeCompare(b.target_uri) ); const fileDoesntExistPath = `file://${path.join( @@ -259,7 +341,7 @@ describe('runBrokenLinks', async () => { .split(' ') .join('%20'); - expect(sorted_followed_links) + expect(sorted_followed_links) .excluding(['target_uri', 'link_start_time', 'link_end_time']) .to.deep.equal([ { @@ -275,7 +357,7 @@ describe('runBrokenLinks', async () => { link_start_time: 'NA', link_end_time: 'NA', is_origin: false, - screenshot_output: defaultScreenshotOutput, + screenshot_output: emptyScreenshotOutput, }, { link_passed: false, @@ -290,7 +372,7 @@ describe('runBrokenLinks', async () => { link_start_time: 'NA', link_end_time: 'NA', is_origin: false, - screenshot_output: defaultScreenshotOutput, + screenshot_output: successfulScreenshotOuput, }, ]); @@ -299,9 +381,11 @@ describe('runBrokenLinks', async () => { '/example_html_files/200.html', '/example_html_files/file_doesnt_exist.html', ]; - broken_links_result?.followed_link_results?.forEach((link, index) => { - expect(link.target_uri.endsWith(expectedTargeturis[index])); - }); + broken_links_result?.followed_link_results?.forEach( + (link: BrokenLinksResultV1_SyntheticLinkResult, index: number) => { + expect(link.target_uri.endsWith(expectedTargeturis[index])); + } + ); }).timeout(150000); it('Completes a full passing execution', async () => { @@ -309,13 +393,14 @@ describe('runBrokenLinks', async () => { __dirname, '../example_html_files/retrieve_links_test.html' )}`; - const inputOptions : BrokenLinkCheckerOptions = { + const inputOptions: BrokenLinkCheckerOptions = { origin_uri: origin_uri, query_selector_all: 'a[src]', - get_attributes: ['src'] + get_attributes: ['src'], + screenshot_options: { capture_condition: CaptureCondition.NONE }, }; - const result = await runBrokenLinks(inputOptions); + const result = await runBrokenLinks(inputOptions, args); const broken_links_result = result?.synthetic_broken_links_result_v1; const options = broken_links_result?.options; @@ -346,7 +431,10 @@ describe('runBrokenLinks', async () => { wait_for_selector: '', per_link_options: {}, total_synthetic_timeout_millis: 60000, - screenshot_options: defaultScreenshotOptions, + screenshot_options: { + capture_condition: ApiCaptureCondition.NONE, + storage_location: '', + }, }); expect(origin_link) @@ -364,7 +452,7 @@ describe('runBrokenLinks', async () => { link_start_time: 'NA', link_end_time: 'NA', is_origin: true, - screenshot_output: defaultScreenshotOutput, + screenshot_output: emptyScreenshotOutput, }); expect(followed_links) @@ -383,7 +471,7 @@ describe('runBrokenLinks', async () => { link_start_time: 'NA', link_end_time: 'NA', is_origin: false, - screenshot_output: defaultScreenshotOutput, + screenshot_output: emptyScreenshotOutput, }, ]); diff --git a/packages/synthetics-sdk-broken-links/test/unit/handlers.spec.ts b/packages/synthetics-sdk-broken-links/test/unit/handlers.spec.ts new file mode 100644 index 0000000..a228db4 --- /dev/null +++ b/packages/synthetics-sdk-broken-links/test/unit/handlers.spec.ts @@ -0,0 +1,60 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Standard Libraries +import { expect } from 'chai'; +import sinon from 'sinon'; + +// External Dependency +import httpMocks from 'node-mocks-http'; +const proxyquire = require('proxyquire'); + +import { runBrokenLinksHandler } from '../../src/handlers'; +import { BrokenLinkCheckerOptions } from '../../src/broken_links'; + +describe('Broken Links Synthetic Handler', async () => { + it('has check id and execution id available', async () => { + // Stub the runBrokenLinks function using Sinon + const mockRunBrokenLinks = sinon.stub().callsFake(async (opts, args) => { + return Promise.resolve({ mocked_response: 'is unimportant' }); + }); + const mockedBrokenLinks = proxyquire('../../src/handlers', { + './broken_links': { runBrokenLinks: mockRunBrokenLinks }, + }); + + // Options for the runBrokenLinksHandler + const options: BrokenLinkCheckerOptions = { + origin_uri: 'https://example.com', + }; + + // Create mock request and response + const req = httpMocks.createRequest({ + headers: { + ['Synthetic-Execution-Id']: 'test-execution-id', + ['Check-Id']: 'test-check-id', + }, + }); + const res = httpMocks.createResponse(); + + // Call the middleware + await mockedBrokenLinks.runBrokenLinksHandler(options)(req, res); + + // Assertions with Sinon and Chai + sinon.assert.calledWith(mockRunBrokenLinks, options, { + executionId: 'test-execution-id', + checkId: 'test-check-id', + }); + expect(res.statusCode).to.equal(200); + }).timeout(5000); +}); diff --git a/packages/synthetics-sdk-broken-links/test/unit/link_utils.spec.ts b/packages/synthetics-sdk-broken-links/test/unit/link_utils.spec.ts index 2c486c5..e59d3f1 100644 --- a/packages/synthetics-sdk-broken-links/test/unit/link_utils.spec.ts +++ b/packages/synthetics-sdk-broken-links/test/unit/link_utils.spec.ts @@ -14,12 +14,14 @@ // Standard Libraries import { expect } from 'chai'; +import sinon from 'sinon'; // Internal Project Files import { BaseError, BrokenLinksResultV1_BrokenLinkCheckerOptions, BrokenLinksResultV1_BrokenLinkCheckerOptions_LinkOrder, + BrokenLinksResultV1_BrokenLinkCheckerOptions_ScreenshotOptions_CaptureCondition as ApiCaptureCondition, BrokenLinksResultV1_SyntheticLinkResult, ResponseStatusCode, ResponseStatusCode_StatusClass, @@ -29,12 +31,19 @@ import { checkStatusPassing, createSyntheticResult, getGenericSyntheticResult, + getStoragePathToExecution, LinkIntermediate, sanitizeObjectName, shuffleAndTruncate, + shouldTakeScreenshot, } from '../../src/link_utils'; import { setDefaultOptions } from '../../src/options_func'; +// External Dependencies +import { Bucket, Storage } from '@google-cloud/storage'; +import { StorageParameters } from '../../src/storage_func'; +import { TEST_BUCKET_NAME } from './storage_func.spec'; + describe('GCM Synthetics Broken Links Utilies', async () => { const status_value_200: ResponseStatusCode = { status_value: 200 }; const status_value_404: ResponseStatusCode = { status_value: 404 }; @@ -56,6 +65,16 @@ describe('GCM Synthetics Broken Links Utilies', async () => { const default_errors: BaseError[] = [ { error_type: 'fake-error-type', error_message: 'fake-error-message' }, ]; + const bucketStub: sinon.SinonStubbedInstance = + sinon.createStubInstance(Bucket); + bucketStub.name = TEST_BUCKET_NAME; + const storageParams = { + storageClient: {} as Storage, + bucket: bucketStub, + checkId: 'uptime123', + executionId: 'exec456', + screenshotNumber: 1, + }; it('checkStatusPassing returns correctly when passed a number as ResponseStatusCode', () => { // expecting success @@ -129,6 +148,7 @@ describe('GCM Synthetics Broken Links Utilies', async () => { runtime_metadata, options, all_links, + storageParams, default_errors ); @@ -147,7 +167,8 @@ describe('GCM Synthetics Broken Links Utilies', async () => { options: options, origin_link_result: origin_link, followed_link_results: followed_links, - execution_data_storage_path: '', + execution_data_storage_path: + 'gs://gcm-test-project-id-synthetics-test-region/uptime123/exec456', errors: default_errors, }); @@ -214,31 +235,124 @@ describe('GCM Synthetics Broken Links Utilies', async () => { describe('sanitizeObjectName', () => { it('should remove invalid characters', () => { - const input = "test/\@#$%^&*()/_+\-=[]{};':\"\|,.<>/?\r\n\t"; - const expectedOutput = "test_@_$%^&_()__+-=__{};'__\_,.______"; + const input = 'test/@#$%^&*()/_+-=[]{};\':"|,.<>/?\r\n\t'; + const expectedOutput = "test_@_$%^&_()__+-=__{};'___,.______"; expect(sanitizeObjectName(input)).to.equal(expectedOutput); }); it('should replace the forbidden prefix', () => { - const input = ".well-known/acme-challenge/test"; - const expectedOutput = "_test"; + const input = '.well-known/acme-challenge/test'; + const expectedOutput = '_test'; expect(sanitizeObjectName(input)).to.equal(expectedOutput); }); it('should handle standalone "." and ".."', () => { - expect(sanitizeObjectName(".")).to.equal("_"); - expect(sanitizeObjectName("..")).to.equal("_"); + expect(sanitizeObjectName('.')).to.equal('_'); + expect(sanitizeObjectName('..')).to.equal('_'); }); it('should handle null and undefined', () => { - expect(sanitizeObjectName(null)).to.equal("_"); - expect(sanitizeObjectName(undefined)).to.equal("_"); - }) + expect(sanitizeObjectName(null)).to.equal('_'); + expect(sanitizeObjectName(undefined)).to.equal('_'); + }); it('should trim leading and trailing whitespace', () => { - const input = " test name "; - const expectedOutput = "test_name"; + const input = ' test name '; + const expectedOutput = 'test_name'; expect(sanitizeObjectName(input)).to.equal(expectedOutput); }); }); + + describe('shouldTakeScreenshot', () => { + describe('screenshot_condition: ALL', () => { + const options = { + screenshot_options: { capture_condition: ApiCaptureCondition.ALL }, + } as BrokenLinksResultV1_BrokenLinkCheckerOptions; + + it('should return true when passed is true', () => { + const result = shouldTakeScreenshot(options, true); + expect(result).to.be.true; + }); + + it('should return true when passed is false', () => { + const result = shouldTakeScreenshot(options, false); + expect(result).to.be.true; + }); + }); + + describe('screenshot_condition: FAILING', () => { + const options = { + screenshot_options: { + capture_condition: ApiCaptureCondition.FAILING, + }, + } as BrokenLinksResultV1_BrokenLinkCheckerOptions; + + it('should return true if passed is false', () => { + const result = shouldTakeScreenshot(options, false); + expect(result).to.be.true; + }); + + it('should return false if passed is true', () => { + const result = shouldTakeScreenshot(options, true); + expect(result).to.be.false; + }); + }); + + describe('screenshot_condition: NONE', () => { + const options = { + screenshot_options: { capture_condition: ApiCaptureCondition.NONE }, + } as BrokenLinksResultV1_BrokenLinkCheckerOptions; + + it('should retrun true if passed is false', () => { + const result = shouldTakeScreenshot(options, false); + expect(result).to.be.false; + }); + + it('should retrun true if passed is true', () => { + const result = shouldTakeScreenshot(options, true); + expect(result).to.be.false; + }); + }); + }); + + describe('getStoragePathToExecution()', () => { + it('returns write_destination when given folder in storage location', () => { + const options = { + screenshot_options: { storage_location: 'bucket/folder1/folder2' }, + } as BrokenLinksResultV1_BrokenLinkCheckerOptions; + + const writeDestination = getStoragePathToExecution( + storageParams, + options + ); + expect(writeDestination).to.equal('folder1/folder2/uptime123/exec456'); + }); + + it('should handle no folder and just bucket in storage_location', () => { + const options = { + screenshot_options: { storage_location: 'bucket' }, + } as BrokenLinksResultV1_BrokenLinkCheckerOptions; + + const result = getStoragePathToExecution(storageParams, options); + expect(result).to.equal('uptime123/exec456'); + }); + + it('should handle error by returning empty string', () => { + const options = { + screenshot_options: { storage_location: 'bucket' }, + } as BrokenLinksResultV1_BrokenLinkCheckerOptions; + + const storageParamsUndefiniedCheckId = { + storageClient: {} as Storage, + bucket: {} as Bucket, + executionId: 'exec456', + } as StorageParameters; + + const result = getStoragePathToExecution( + storageParamsUndefiniedCheckId, + options + ); + expect(result).to.equal(''); + }); + }); }); diff --git a/packages/synthetics-sdk-broken-links/test/unit/navigation_func.spec.ts b/packages/synthetics-sdk-broken-links/test/unit/navigation_func.spec.ts index b023a50..c751827 100644 --- a/packages/synthetics-sdk-broken-links/test/unit/navigation_func.spec.ts +++ b/packages/synthetics-sdk-broken-links/test/unit/navigation_func.spec.ts @@ -23,6 +23,7 @@ import sinon from 'sinon'; import { BaseError, BrokenLinksResultV1_SyntheticLinkResult, + BrokenLinksResultV1_SyntheticLinkResult_ScreenshotOutput as ApiScreenshotOutput, ResponseStatusCode, ResponseStatusCode_StatusClass, } from '@google-cloud/synthetics-sdk-api'; @@ -34,34 +35,62 @@ import { retrieveLinksFromPage, } from '../../src/navigation_func'; import { setDefaultOptions } from '../../src/options_func'; +import * as storageFunc from '../../src/storage_func'; + +// External Dependencies +import { Bucket, Storage } from '@google-cloud/storage'; +const proxyquire = require('proxyquire'); // External Dependencies import puppeteer, { Browser, HTTPResponse, Page } from 'puppeteer'; describe('GCM Synthetics Broken Links Navigation Functionality', async () => { - // constants + // Constants const link: LinkIntermediate = { target_uri: 'https://example.com', anchor_text: '', html_element: '', }; - const input_options: BrokenLinkCheckerOptions = { + const defaultOptions: BrokenLinkCheckerOptions = { origin_uri: 'http://origin.com', max_retries: 2, link_timeout_millis: 5000, }; - const options = setDefaultOptions(input_options); + const options = setDefaultOptions(defaultOptions); const response2xx: Partial = { status: () => 200 }; const response4xx: Partial = { status: () => 404 }; - const status_class_2xx: ResponseStatusCode = { + const statusClass2xx: ResponseStatusCode = { status_class: ResponseStatusCode_StatusClass.STATUS_CLASS_2XX, }; - const status_class_4xx: ResponseStatusCode = { + const statusClass4xx: ResponseStatusCode = { status_class: ResponseStatusCode_StatusClass.STATUS_CLASS_4XX, }; + const emptyScreenshotOutput: ApiScreenshotOutput = { + screenshot_file: '', + screenshot_error: {} as BaseError, + }; + const successfulScreenshotOuput: ApiScreenshotOutput = { + screenshot_file: 'bucket/folder/file.png', + screenshot_error: {} as BaseError, + }; - // Puppeteer constants + const storageParams: storageFunc.StorageParameters = { + storageClient: sinon.createStubInstance(Storage), + bucket: sinon.createStubInstance(Bucket), + checkId: '', + executionId: '', + screenshotNumber: 1, + }; + + const navigStorageUploadSuccMocked = proxyquire('../../src/navigation_func', { + './storage_func': { + ...storageFunc, + uploadScreenshotToGCS: () => successfulScreenshotOuput, + }, + }); + + // Puppeteer Setup let browser: Browser; let page: Page; before(async () => { @@ -71,21 +100,35 @@ describe('GCM Synthetics Broken Links Navigation Functionality', async () => { }); beforeEach(async () => { - // Create a new page for each test page = await browser.newPage(); page.setCacheEnabled(false); + + sinon + .stub(page, 'screenshot') + .resolves(Buffer.from('encoded-image-data', 'utf-8')); + }); + + afterEach(() => { + sinon.restore(); }); after(async () => { - // Close the browser after all tests browser && (await browser.close()); }); describe('navigate', async () => { - it('should pass after retries', async () => { - const pageStub = sinon.createStubInstance(Page); + let pageStub: sinon.SinonStubbedInstance; + + beforeEach(() => { + pageStub = sinon.createStubInstance(Page); pageStub.url.returns('fake-current-uri'); + }); + + afterEach(() => { + sinon.restore(); + }); + it('should pass after retries', async () => { // Configure the stub to simulate a failed navigation on the first call // and a successful one on the second pageStub.goto @@ -102,9 +145,6 @@ describe('GCM Synthetics Broken Links Navigation Functionality', async () => { }); it('should fail after maximum retries', async () => { - const pageStub = sinon.createStubInstance(Page); - pageStub.url.returns('fake-current-uri'); - // Configure the stub to simulate a failed navigation on three // consecutive calls pageStub.goto @@ -125,11 +165,11 @@ describe('GCM Synthetics Broken Links Navigation Functionality', async () => { describe('checkLink', async () => { it('passes when navigating to real uri', async () => { - const synLinkResult = await checkLink(page, link, options); + const synLinkResult = await checkLink(page, link, options, storageParams); const expectations: BrokenLinksResultV1_SyntheticLinkResult = { link_passed: true, - expected_status_code: status_class_2xx, + expected_status_code: statusClass2xx, source_uri: 'http://origin.com', target_uri: 'https://example.com', html_element: '', @@ -140,10 +180,7 @@ describe('GCM Synthetics Broken Links Navigation Functionality', async () => { link_start_time: 'NA', link_end_time: 'NA', is_origin: false, - screenshot_output: { - screenshot_file: '', - screenshot_error: {} as BaseError, - }, + screenshot_output: emptyScreenshotOutput, }; expect(synLinkResult) @@ -165,11 +202,16 @@ describe('GCM Synthetics Broken Links Navigation Functionality', async () => { anchor_text: '', html_element: '', }; - const synLinkResult = await checkLink(page, json_link, options); + const synLinkResult = await checkLink( + page, + json_link, + options, + storageParams + ); const expectations: BrokenLinksResultV1_SyntheticLinkResult = { link_passed: true, - expected_status_code: status_class_2xx, + expected_status_code: statusClass2xx, source_uri: 'http://origin.com', target_uri: `file:${path.join( __dirname, @@ -183,10 +225,7 @@ describe('GCM Synthetics Broken Links Navigation Functionality', async () => { link_start_time: 'NA', link_end_time: 'NA', is_origin: false, - screenshot_output: { - screenshot_file: '', - screenshot_error: {} as BaseError, - }, + screenshot_output: emptyScreenshotOutput, }; expect(synLinkResult) @@ -210,15 +249,16 @@ describe('GCM Synthetics Broken Links Navigation Functionality', async () => { html_element: 'img', }; - const synLinkResult = await checkLink( + const synLinkResult = await navigStorageUploadSuccMocked.checkLink( page, timeout_link, - options_with_timeout + options_with_timeout, + storageParams ); const expectations: BrokenLinksResultV1_SyntheticLinkResult = { link_passed: false, - expected_status_code: status_class_2xx, + expected_status_code: statusClass2xx, source_uri: 'http://origin.com', target_uri: target_uri, html_element: 'img', @@ -229,10 +269,7 @@ describe('GCM Synthetics Broken Links Navigation Functionality', async () => { link_start_time: 'NA', link_end_time: 'NA', is_origin: false, - screenshot_output: { - screenshot_file: '', - screenshot_error: {} as BaseError, - }, + screenshot_output: successfulScreenshotOuput, }; expect(synLinkResult) @@ -241,14 +278,14 @@ describe('GCM Synthetics Broken Links Navigation Functionality', async () => { }).timeout(5000); it('returns error when the actual response code does not match the expected', async () => { - // add expected 404 status to options of broken link checker - const optionsExp404 = Object.assign({}, options); const per_link_expected_404 = { expected_status_code: { status_class: ResponseStatusCode_StatusClass.STATUS_CLASS_4XX, }, link_timeout_millis: options.link_timeout_millis, }; + + const optionsExp404 = Object.assign({}, options); optionsExp404.per_link_options['https://expecting404.com'] = per_link_expected_404; @@ -268,15 +305,16 @@ describe('GCM Synthetics Broken Links Navigation Functionality', async () => { .onThirdCall() .resolves(response2xx as HTTPResponse); - const synLinkResult = await checkLink( + const synLinkResult = await navigStorageUploadSuccMocked.checkLink( pageStub, timeoutLink, - optionsExp404 + optionsExp404, + storageParams ); const expectations: BrokenLinksResultV1_SyntheticLinkResult = { link_passed: false, - expected_status_code: status_class_4xx, + expected_status_code: statusClass4xx, source_uri: 'http://origin.com', target_uri: 'https://expecting404.com', html_element: 'a', @@ -288,10 +326,7 @@ describe('GCM Synthetics Broken Links Navigation Functionality', async () => { link_start_time: 'NA', link_end_time: 'NA', is_origin: false, - screenshot_output: { - screenshot_file: '', - screenshot_error: {} as BaseError, - }, + screenshot_output: successfulScreenshotOuput, }; expect(synLinkResult) @@ -305,13 +340,11 @@ describe('retrieveLinksFromPage', async () => { // Puppeteer constants let browser: Browser; let page: Page; - let pageuriStub: sinon.SinonStub<[], string>; before(async () => { browser = await puppeteer.launch({ headless: 'new' }); }); beforeEach(async () => { - // Create a new page for each test page = await browser.newPage(); await page.goto( `file:${path.join( @@ -320,11 +353,10 @@ describe('retrieveLinksFromPage', async () => { )}` ); // Mock page.uri() to return a custom uri - pageuriStub = sinon.stub(page, 'url').returns('https://mocked.com'); + sinon.stub(page, 'url').returns('https://mocked.com'); }); after(async () => { - // Close the browser after all tests browser && (await browser.close()); }); @@ -378,7 +410,7 @@ describe('retrieveLinksFromPage', async () => { // note: does not return `mailto:...` link expect(results).to.deep.equal(expectations); - }); + }).timeout(5000); it('handles complicated query_selector_all', async () => { const query_selector_all = 'img[href], a[src]'; diff --git a/packages/synthetics-sdk-broken-links/test/unit/storage_func.spec.ts b/packages/synthetics-sdk-broken-links/test/unit/storage_func.spec.ts index b7e583a..9a29019 100644 --- a/packages/synthetics-sdk-broken-links/test/unit/storage_func.spec.ts +++ b/packages/synthetics-sdk-broken-links/test/unit/storage_func.spec.ts @@ -24,8 +24,6 @@ import { } from '@google-cloud/synthetics-sdk-api'; import { createStorageClientIfStorageSelected, - getFolderNameFromStorageLocation, - getOrCreateStorageBucket, StorageParameters, uploadScreenshotToGCS, } from '../../src/storage_func'; @@ -33,9 +31,10 @@ import { // External Dependencies import { Bucket, File, Storage } from '@google-cloud/storage'; const proxyquire = require('proxyquire'); +import { Page } from 'puppeteer'; // global test vars -const TEST_BUCKET_NAME = 'gcm-test-project-id-synthetics-test-region'; +export const TEST_BUCKET_NAME = 'gcm-test-project-id-synthetics-test-region'; describe('GCM Synthetics Broken Links storage_func suite testing', () => { let storageClientStub: sinon.SinonStubbedInstance; @@ -45,7 +44,6 @@ describe('GCM Synthetics Broken Links storage_func suite testing', () => { '@google-cloud/synthetics-sdk-api': { getExecutionRegion: () => 'test-region', resolveProjectId: () => 'test-project-id', - getOrCreateStorageBucket: () => getOrCreateStorageBucket, }, }); @@ -163,13 +161,14 @@ describe('GCM Synthetics Broken Links storage_func suite testing', () => { describe('uploadScreenshotToGCS', () => { let storageClientStub: sinon.SinonStubbedInstance; let bucketStub: sinon.SinonStubbedInstance; - - const screenshotData = 'encoded-image-data'; - const filename = 'test-screenshot.png'; + let pageStub: sinon.SinonStubbedInstance; beforeEach(() => { storageClientStub = sinon.createStubInstance(Storage); bucketStub = sinon.createStubInstance(Bucket); + pageStub = sinon.createStubInstance(Page); + pageStub.url.resolves('https://fake-url'); + storageClientStub.bucket.returns(bucketStub); }); @@ -177,82 +176,97 @@ describe('GCM Synthetics Broken Links storage_func suite testing', () => { sinon.restore(); }); - it('should upload the screenshot and return the write_destination', async () => { - const storageParams = { - storageClient: storageClientStub, - bucket: bucketStub, - uptimeId: 'uptime123', - executionId: 'exec456', - }; - const options = { - screenshot_options: { storage_location: 'bucket/folder1/folder2' }, - } as BrokenLinksResultV1_BrokenLinkCheckerOptions; - const expectedWriteDestination = - 'folder1/folder2/uptime123/exec456/test-screenshot.png'; - - const successPartialFileMock: Partial = { - save: sinon.stub().resolves(), - }; - bucketStub.file.returns(successPartialFileMock as File); - - const result = await uploadScreenshotToGCS( - screenshotData, - filename, - storageParams, - options - ); + describe('Valid Storage Configuration', () => { + it('should upload the screenshots and return updated write_destination', async () => { + const storageParams = { + storageClient: storageClientStub, + bucket: bucketStub, + checkId: 'uptime123', + executionId: 'exec456', + screenshotNumber: 1, + }; + const options = { + screenshot_options: { storage_location: 'bucket/folder1/folder2' }, + } as BrokenLinksResultV1_BrokenLinkCheckerOptions; - expect(result.screenshot_file).to.equal(expectedWriteDestination); - expect(result.screenshot_error).to.deep.equal({}); - }); + const successPartialFileMock: Partial = { + save: sinon.stub().resolves(), + }; + bucketStub.file.returns(successPartialFileMock as File); - it('should handle GCS upload errors', async () => { - const storageParams: StorageParameters = { - storageClient: storageClientStub, - bucket: bucketStub, - uptimeId: '', - executionId: '', - }; - const options = { - screenshot_options: {}, - } as BrokenLinksResultV1_BrokenLinkCheckerOptions; - - const gcsError = new Error('Simulated GCS upload error'); - const failingPartialFileMock: Partial = { - save: sinon.stub().throws(gcsError), - }; - bucketStub.file.returns(failingPartialFileMock as File); - - const result = await uploadScreenshotToGCS( - screenshotData, - filename, - storageParams, - options - ); + const result = await uploadScreenshotToGCS( + pageStub, + storageParams, + options + ); + + expect(result.screenshot_file).to.equal('screenshot_1.png'); + expect(result.screenshot_error).to.deep.equal({}); + + const result2 = await uploadScreenshotToGCS( + pageStub, + storageParams, + options + ); + + expect(result2.screenshot_file).to.equal('screenshot_2.png'); + expect(result2.screenshot_error).to.deep.equal({}); + }); + + it('should handle GCS upload errors', async () => { + const storageParams: StorageParameters = { + storageClient: storageClientStub, + bucket: bucketStub, + checkId: '', + executionId: '', + screenshotNumber: 1, + }; + const options = { + screenshot_options: {}, + } as BrokenLinksResultV1_BrokenLinkCheckerOptions; + + const gcsError = new Error('Simulated GCS upload error'); + const failingPartialFileMock: Partial = { + save: sinon.stub().throws(gcsError), + }; + bucketStub.file.returns(failingPartialFileMock as File); + + const result = await uploadScreenshotToGCS( + pageStub, + storageParams, + options + ); - expect(result.screenshot_file).to.equal(''); - expect(result.screenshot_error).to.deep.equal({ - error_type: 'StorageFileUploadError', - error_message: `Failed to upload screenshot for ${filename}. Please reference server logs for further information.`, + expect(result.screenshot_file).to.equal(''); + expect(result.screenshot_error).to.deep.equal({ + error_type: 'ScreenshotFileUploadError', + error_message: + 'Failed to take and/or upload screenshot for https://fake-url. Please reference server logs for further information.', + }); }); }); describe('Invalid Storage Configuration', () => { - const emptyScreenshotData = ''; - const emptyFilename = ''; const emptyOptions = {} as BrokenLinksResultV1_BrokenLinkCheckerOptions; + + beforeEach(() => { + pageStub.screenshot.resolves( + Buffer.from('encoded-image-data', 'utf-8') + ); + }); + it('should return an empty result if storageClient is null', async () => { // Missing storageClient const storageParams = { storageClient: null, bucket: bucketStub, - uptimeId: '', + checkId: '', executionId: '', + screenshotNumber: 1, }; const result = await uploadScreenshotToGCS( - emptyScreenshotData, - emptyFilename, + pageStub, storageParams, emptyOptions ); @@ -268,13 +282,13 @@ describe('GCM Synthetics Broken Links storage_func suite testing', () => { const storageParams = { storageClient: storageClientStub, bucket: null, - uptimeId: '', + checkId: '', executionId: '', - }; + screenshotNumber: 1, + } as StorageParameters; const result = await uploadScreenshotToGCS( - emptyScreenshotData, - emptyFilename, + pageStub, storageParams, emptyOptions ); @@ -286,30 +300,4 @@ describe('GCM Synthetics Broken Links storage_func suite testing', () => { }); }); }); - - describe('getFolderNameFromStorageLocation', () => { - it('should extract folder name when storage location has a slash', () => { - const storageLocation = 'some-bucket/folder1/folder2'; - const expectedFolderName = 'folder1/folder2'; - - const result = getFolderNameFromStorageLocation(storageLocation); - expect(result).to.equal(expectedFolderName); - }); - - it('should return an empty string if storage location has no slash', () => { - const storageLocation = 'my-bucket'; - const expectedFolderName = ''; - - const result = getFolderNameFromStorageLocation(storageLocation); - expect(result).to.equal(expectedFolderName); - }); - - it('should return an empty string if given an empty string', () => { - const storageLocation = ''; - const expectedFolderName = ''; - - const result = getFolderNameFromStorageLocation(storageLocation); - expect(result).to.equal(expectedFolderName); - }); - }); });