DLP sample: Add autoPopulateTimespan option for creating job triggers. (#64)

Author: micheldavid

dlp/.gitignore

@@ -0,0 +1,3 @@
+# Test outputs
+*.actual.png
+*.actual.csv

dlp/package.json

@@ -21,6 +21,8 @@
   "devDependencies": {
     "@google-cloud/nodejs-repo-tools": "^2.3.0",
     "ava": "^0.25.0",
+    "pixelmatch": "^4.0.2",
+    "pngjs": "^3.3.3",
     "uuid": "^3.3.2"
   }
 }

dlp/system-test/deid.test.js

@@ -123,7 +123,7 @@ test(`should handle FPE decryption errors`, async t => {
 
 // deidentify_date_shift
 test(`should date-shift a CSV file`, async t => {
-  const outputCsvFile = path.join(__dirname, 'dates.result.csv');
+  const outputCsvFile = 'dates.actual.csv';
   const output = await tools.runAsync(
     `${cmd} deidDateShift "${csvFile}" "${outputCsvFile}" ${dateShiftAmount} ${dateShiftAmount} ${dateFields}`,
     cwd
@@ -138,9 +138,9 @@ test(`should date-shift a CSV file`, async t => {
 });
 
 test(`should date-shift a CSV file using a context field`, async t => {
-  const outputCsvFile = path.join(__dirname, 'dates-context.result.csv');
-  const correctResultFile =
-    'system-test/resources/date-shift-context.correct.csv';
+  const outputCsvFile = 'dates-context.actual.csv';
+  const expectedCsvFile =
+    'system-test/resources/date-shift-context.expected.csv';
   const output = await tools.runAsync(
     `${cmd} deidDateShift "${csvFile}" "${outputCsvFile}" ${dateShiftAmount} ${dateShiftAmount} ${dateFields} -f ${csvContextField} -n ${keyName} -w ${wrappedKey}`,
     cwd
@@ -150,7 +150,7 @@ test(`should date-shift a CSV file using a context field`, async t => {
   );
   t.is(
     fs.readFileSync(outputCsvFile).toString(),
-    fs.readFileSync(correctResultFile).toString()
+    fs.readFileSync(expectedCsvFile).toString()
   );
 });
 

dlp/system-test/redact.test.js

@@ -19,6 +19,8 @@ const path = require('path');
 const test = require('ava');
 const fs = require('fs');
 const tools = require('@google-cloud/nodejs-repo-tools');
+const PNG = require('pngjs').PNG;
+const pixelmatch = require('pixelmatch');
 
 const cmd = 'node redact.js';
 const cwd = path.join(__dirname, `..`);
@@ -28,6 +30,33 @@ const testResourcePath = 'system-test/resources';
 
 test.before(tools.checkCredentials);
 
+function readImage(filePath) {
+  return new Promise((resolve, reject) => {
+    fs.createReadStream(filePath)
+      .pipe(new PNG())
+      .on('error', reject)
+      .on('parsed', function() {
+        resolve(this);
+      });
+  });
+}
+
+async function getImageDiffPercentage(image1Path, image2Path) {
+  let image1 = await readImage(image1Path);
+  let image2 = await readImage(image2Path);
+  let diff = new PNG({width: image1.width, height: image1.height});
+
+  const diffPixels = pixelmatch(
+    image1.data,
+    image2.data,
+    diff.data,
+    image1.width,
+    image1.height
+  );
+  return diffPixels / (diff.width * diff.height);
+}
+
+// redact_text
 test(`should redact a single sensitive data type from a string`, async t => {
   const output = await tools.runAsync(
     `${cmd} string "My email is [email protected]" -t EMAIL_ADDRESS`,
@@ -56,33 +85,33 @@ test(`should handle string with no sensitive data`, async t => {
 test(`should redact a single sensitive data type from an image`, async t => {
   const testName = `redact-single-type`;
   const output = await tools.runAsync(
-    `${cmd} image ${testImage} ${testName}.result.png -t PHONE_NUMBER`,
+    `${cmd} image ${testImage} ${testName}.actual.png -t PHONE_NUMBER`,
     cwd
   );
 
   t.regex(output, /Saved image redaction results to path/);
 
-  const correct = fs.readFileSync(
-    `${testResourcePath}/${testName}.correct.png`
+  const difference = await getImageDiffPercentage(
+    `${testName}.actual.png`,
+    `${testResourcePath}/${testName}.expected.png`
   );
-  const result = fs.readFileSync(`${testName}.result.png`);
-  t.deepEqual(correct, result);
+  t.true(difference < 0.03);
 });
 
 test(`should redact multiple sensitive data types from an image`, async t => {
   const testName = `redact-multiple-types`;
   const output = await tools.runAsync(
-    `${cmd} image ${testImage} ${testName}.result.png -t PHONE_NUMBER EMAIL_ADDRESS`,
+    `${cmd} image ${testImage} ${testName}.actual.png -t PHONE_NUMBER EMAIL_ADDRESS`,
     cwd
   );
 
   t.regex(output, /Saved image redaction results to path/);
 
-  const correct = fs.readFileSync(
-    `${testResourcePath}/${testName}.correct.png`
+  const difference = await getImageDiffPercentage(
+    `${testName}.actual.png`,
+    `${testResourcePath}/${testName}.expected.png`
   );
-  const result = fs.readFileSync(`${testName}.result.png`);
-  t.deepEqual(correct, result);
+  t.true(difference < 0.03);
 });
 
 test(`should report info type errors`, async t => {
@@ -95,7 +124,7 @@ test(`should report info type errors`, async t => {
 
 test(`should report image redaction handling errors`, async t => {
   const output = await tools.runAsync(
-    `${cmd} image ${testImage} nonexistent.result.png -t BAD_TYPE`,
+    `${cmd} image ${testImage} output.png -t BAD_TYPE`,
     cwd
   );
   t.regex(output, /Error in redactImage/);

dlp/system-test/resources/date-shift-context.correct.csv renamed to dlp/system-test/resources/date-shift-context.expected.csv

@@ -29,7 +29,7 @@ const uniqueField = 'Name';
 const repeatedField = 'Mystery';
 const numericField = 'Age';
 const stringBooleanField = 'Gender';
-const testProjectId = 'nodejs-docs-samples';
+const testProjectId = process.env.GCLOUD_PROJECT;
 
 test.before(tools.checkCredentials);
 

dlp/system-test/resources/redact-multiple-types.correct.png renamed to dlp/system-test/resources/redact-multiple-types.expected.png

@@ -42,7 +42,7 @@ test.serial(`should create template`, async t => {
 });
 
 test(`should handle template creation errors`, async t => {
-  const output = await tools.runAsync(`${cmd} create -t BAD_INFOTYPE`);
+  const output = await tools.runAsync(`${cmd} create -i invalid_template#id`);
   t.regex(output, /Error in createInspectTemplate/);
 });
 

dlp/system-test/resources/redact-single-type.correct.png renamed to dlp/system-test/resources/redact-single-type.expected.png

@@ -33,7 +33,8 @@ const bucketName = process.env.BUCKET_NAME;
 
 test.serial(`should create a trigger`, async t => {
   const output = await tools.runAsync(
-    `${cmd} create ${bucketName} 1 -n ${triggerName} -m ${minLikelihood} -t ${infoType} -f ${maxFindings} -d "${triggerDisplayName}" -s "${triggerDescription}"`
+    `${cmd} create ${bucketName} 1 -n ${triggerName} --autoPopulateTimespan \
+    -m ${minLikelihood} -t ${infoType} -f ${maxFindings} -d "${triggerDisplayName}" -s "${triggerDescription}"`
   );
   t.true(output.includes(`Successfully created trigger ${fullTriggerName}`));
 });

dlp/system-test/resources/redact-single-type.output.png

@@ -21,6 +21,7 @@ function createTrigger(
   displayName,
   description,
   bucketName,
+  autoPopulateTimespan,
   scanPeriod,
   infoTypes,
   minLikelihood,
@@ -48,6 +49,9 @@ function createTrigger(
   // The name of the bucket to scan.
   // const bucketName = 'YOUR-BUCKET';
 
+  // Limit scan to new content only.
+  // const autoPopulateTimespan = true;
+
   // How often to wait between scans, in days (minimum = 1 day)
   // const scanPeriod = 1;
 
@@ -65,6 +69,9 @@ function createTrigger(
     cloudStorageOptions: {
       fileSet: {url: `gs://${bucketName}/*`},
     },
+    timeSpanConfig: {
+      enableAutoPopulationOfTimespanConfig: autoPopulateTimespan,
+    },
   };
 
   // Construct job to be triggered
@@ -221,6 +228,10 @@ const cli = require(`yargs`) // eslint-disable-line
         default: '',
         type: 'string',
       },
+      autoPopulateTimespan: {
+        default: false,
+        type: 'boolean',
+      },
       minLikelihood: {
         alias: 'm',
         default: 'LIKELIHOOD_UNSPECIFIED',
@@ -249,6 +260,7 @@ const cli = require(`yargs`) // eslint-disable-line
         opts.displayName,
         opts.description,
         opts.bucketName,
+        opts.autoPopulateTimespan,
         opts.scanPeriod,
         opts.infoTypes,
         opts.minLikelihood,