Skip to content

Commit b455497

Browse files
rontalxrontalx
committed
feat: iac experimental tf support
1 parent 7dfd3ea commit b455497

File tree

14 files changed

+242
-73
lines changed

14 files changed

+242
-73
lines changed

‎package.json

+1
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,7 @@
7777
"configstore": "^5.0.1",
7878
"debug": "^4.1.1",
7979
"diff": "^4.0.1",
80+
"hcl-to-json": "^0.1.1",
8081
"lodash.assign": "^4.2.0",
8182
"lodash.camelcase": "^4.3.0",
8283
"lodash.clonedeep": "^4.5.0",

‎src/cli/commands/test/iac-local-execution/index.ts

+28-32
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,27 @@
11
import * as fs from 'fs';
2-
import * as YAML from 'js-yaml';
32
import { isLocalFolder } from '../../../../lib/detect';
43
import { getFileType } from '../../../../lib/iac/iac-parser';
54
import * as util from 'util';
65
import { IacFileTypes } from '../../../../lib/iac/constants';
76
import { IacFileScanResult, IacFileMetadata, IacFileData } from './types';
8-
import { buildPolicyEngine } from './policy-engine';
7+
import { getPolicyEngine } from './policy-engine';
98
import { formatResults } from './results-formatter';
9+
import { tryParseIacFile } from './parsers';
10+
import { isLocalCacheExists, REQUIRED_LOCAL_CACHE_FILES } from './local-cache';
1011

1112
const readFileContentsAsync = util.promisify(fs.readFile);
12-
const REQUIRED_K8S_FIELDS = ['apiVersion', 'kind', 'metadata'];
1313

1414
// this method executes the local processing engine and then formats the results to adapt with the CLI output.
1515
// the current version is dependent on files to be present locally which are not part of the source code.
1616
// without these files this method would fail.
1717
// if you're interested in trying out the experimental local execution model for IaC scanning, please reach-out.
1818
export async function test(pathToScan: string, options) {
19+
if (!isLocalCacheExists())
20+
throw Error(
21+
`Missing IaC local cache data, please validate you have: \n${REQUIRED_LOCAL_CACHE_FILES.join(
22+
'\n',
23+
)}`,
24+
);
1925
// TODO: add support for proper typing of old TestResult interface.
2026
const results = await localProcessing(pathToScan);
2127
const formattedResults = formatResults(results, options);
@@ -27,12 +33,9 @@ export async function test(pathToScan: string, options) {
2733
async function localProcessing(
2834
pathToScan: string,
2935
): Promise<IacFileScanResult[]> {
30-
const policyEngine = await buildPolicyEngine();
3136
const filePathsToScan = await getFilePathsToScan(pathToScan);
32-
const fileDataToScan = await parseFileContentsForPolicyEngine(
33-
filePathsToScan,
34-
);
35-
const scanResults = await policyEngine.scanFiles(fileDataToScan);
37+
const fileDataToScan = await parseFilesForScan(filePathsToScan);
38+
const scanResults = await scanFilesForIssues(fileDataToScan);
3639
return scanResults;
3740
}
3841

@@ -42,18 +45,13 @@ async function getFilePathsToScan(pathToScan): Promise<IacFileMetadata[]> {
4245
'IaC Experimental version does not support directory scan yet.',
4346
);
4447
}
45-
if (getFileType(pathToScan) === 'tf') {
46-
throw new Error(
47-
'IaC Experimental version does not support Terraform scan yet.',
48-
);
49-
}
5048

5149
return [
5250
{ filePath: pathToScan, fileType: getFileType(pathToScan) as IacFileTypes },
5351
];
5452
}
5553

56-
async function parseFileContentsForPolicyEngine(
54+
async function parseFilesForScan(
5755
filesMetadata: IacFileMetadata[],
5856
): Promise<IacFileData[]> {
5957
const parsedFileData: Array<IacFileData> = [];
@@ -62,25 +60,23 @@ async function parseFileContentsForPolicyEngine(
6260
fileMetadata.filePath,
6361
'utf-8',
6462
);
65-
const yamlDocuments = YAML.safeLoadAll(fileContent);
66-
67-
yamlDocuments.forEach((parsedYamlDocument, docId) => {
68-
if (
69-
REQUIRED_K8S_FIELDS.every((requiredField) =>
70-
parsedYamlDocument.hasOwnProperty(requiredField),
71-
)
72-
) {
73-
parsedFileData.push({
74-
...fileMetadata,
75-
fileContent: fileContent,
76-
jsonContent: parsedYamlDocument,
77-
docId,
78-
});
79-
} else {
80-
throw new Error('Invalid K8s File!');
81-
}
82-
});
63+
const parsedFiles = tryParseIacFile(fileMetadata, fileContent);
64+
parsedFileData.push(...parsedFiles);
8365
}
8466

8567
return parsedFileData;
8668
}
69+
70+
async function scanFilesForIssues(
71+
parsedFiles: Array<IacFileData>,
72+
): Promise<IacFileScanResult[]> {
73+
// TODO: when adding dir support move implementation to queue.
74+
// TODO: when adding dir support gracefully handle failed scans
75+
return Promise.all(
76+
parsedFiles.map(async (file) => {
77+
const policyEngine = await getPolicyEngine(file.engineType);
78+
const scanResults = policyEngine.scanFile(file);
79+
return scanResults;
80+
}),
81+
);
82+
}

‎src/cli/commands/test/iac-local-execution/local-cache.ts

+48
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
import * as path from 'path';
2+
import * as fs from 'fs';
3+
import { EngineType } from './types';
4+
5+
export const LOCAL_POLICY_ENGINE_DIR = `.iac-data`;
6+
7+
const KUBERNETES_POLICY_ENGINE_WASM_PATH = path.join(
8+
LOCAL_POLICY_ENGINE_DIR,
9+
'k8s_policy.wasm',
10+
);
11+
const KUBERNETES_POLICY_ENGINE_DATA_PATH = path.join(
12+
LOCAL_POLICY_ENGINE_DIR,
13+
'k8s_data.json',
14+
);
15+
const TERRAFORM_POLICY_ENGINE_WASM_PATH = path.join(
16+
LOCAL_POLICY_ENGINE_DIR,
17+
'tf_policy.wasm',
18+
);
19+
const TERRAFORM_POLICY_ENGINE_DATA_PATH = path.join(
20+
LOCAL_POLICY_ENGINE_DIR,
21+
'tf_data.json',
22+
);
23+
24+
export const REQUIRED_LOCAL_CACHE_FILES = [
25+
KUBERNETES_POLICY_ENGINE_WASM_PATH,
26+
KUBERNETES_POLICY_ENGINE_DATA_PATH,
27+
TERRAFORM_POLICY_ENGINE_WASM_PATH,
28+
TERRAFORM_POLICY_ENGINE_DATA_PATH,
29+
];
30+
31+
export function isLocalCacheExists(): boolean {
32+
return REQUIRED_LOCAL_CACHE_FILES.every(fs.existsSync);
33+
}
34+
35+
export function getLocalCachePath(engineType: EngineType) {
36+
switch (engineType) {
37+
case EngineType.Kubernetes:
38+
return [
39+
`${process.cwd()}/${KUBERNETES_POLICY_ENGINE_WASM_PATH}`,
40+
`${process.cwd()}/${KUBERNETES_POLICY_ENGINE_DATA_PATH}`,
41+
];
42+
case EngineType.Terraform:
43+
return [
44+
`${process.cwd()}/${TERRAFORM_POLICY_ENGINE_WASM_PATH}`,
45+
`${process.cwd()}/${TERRAFORM_POLICY_ENGINE_DATA_PATH}`,
46+
];
47+
}
48+
}

‎src/cli/commands/test/iac-local-execution/parsers.ts

+65
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
import * as hclToJson from 'hcl-to-json';
2+
import * as YAML from 'js-yaml';
3+
import { EngineType, IacFileData, IacFileMetadata } from './types';
4+
5+
const REQUIRED_K8S_FIELDS = ['apiVersion', 'kind', 'metadata'];
6+
7+
export function tryParseIacFile(
8+
fileMetadata: IacFileMetadata,
9+
fileContent: string,
10+
): Array<IacFileData> {
11+
switch (fileMetadata.fileType) {
12+
case 'yaml':
13+
case 'yml':
14+
case 'json':
15+
return tryParsingKubernetesFile(fileContent, fileMetadata);
16+
case 'tf':
17+
return [tryParsingTerraformFile(fileContent, fileMetadata)];
18+
default:
19+
throw new Error('Invalid IaC file');
20+
}
21+
}
22+
23+
function tryParsingKubernetesFile(
24+
fileContent: string,
25+
fileMetadata: IacFileMetadata,
26+
): IacFileData[] {
27+
const yamlDocuments = YAML.safeLoadAll(fileContent);
28+
29+
return yamlDocuments.map((parsedYamlDocument, docId) => {
30+
if (
31+
REQUIRED_K8S_FIELDS.every((requiredField) =>
32+
parsedYamlDocument.hasOwnProperty(requiredField),
33+
)
34+
) {
35+
return {
36+
...fileMetadata,
37+
fileContent: fileContent,
38+
jsonContent: parsedYamlDocument,
39+
engineType: EngineType.Kubernetes,
40+
docId,
41+
};
42+
} else {
43+
throw new Error('Invalid K8s File!');
44+
}
45+
});
46+
}
47+
48+
function tryParsingTerraformFile(
49+
fileContent: string,
50+
fileMetadata: IacFileMetadata,
51+
): IacFileData {
52+
try {
53+
// TODO: This parser does not fail on inavlid Terraform files! it is here temporarily.
54+
// cloud-config team will replace it to a valid parser for the beta release.
55+
const parsedData = hclToJson(fileContent);
56+
return {
57+
...fileMetadata,
58+
fileContent: fileContent,
59+
jsonContent: parsedData,
60+
engineType: EngineType.Terraform,
61+
};
62+
} catch (err) {
63+
throw new Error('Invalid Terraform File!');
64+
}
65+
}

‎src/cli/commands/test/iac-local-execution/policy-engine.ts

+31-17
Original file line numberDiff line numberDiff line change
@@ -3,18 +3,36 @@ import {
33
IacFileData,
44
IacFileScanResult,
55
PolicyMetadata,
6+
EngineType,
67
} from './types';
78
import { loadPolicy } from '@open-policy-agent/opa-wasm';
89
import * as fs from 'fs';
9-
import * as path from 'path';
10+
import { getLocalCachePath, LOCAL_POLICY_ENGINE_DIR } from './local-cache';
1011

11-
const LOCAL_POLICY_ENGINE_DIR = `.iac-data`;
12-
const LOCAL_POLICY_ENGINE_WASM_PATH = `${LOCAL_POLICY_ENGINE_DIR}${path.sep}policy.wasm`;
13-
const LOCAL_POLICY_ENGINE_DATA_PATH = `${LOCAL_POLICY_ENGINE_DIR}${path.sep}data.json`;
12+
export async function getPolicyEngine(
13+
engineType: EngineType,
14+
): Promise<PolicyEngine> {
15+
if (policyEngineCache[engineType]) {
16+
return policyEngineCache[engineType]!;
17+
}
18+
19+
policyEngineCache[engineType] = await buildPolicyEngine(engineType);
20+
return policyEngineCache[engineType]!;
21+
}
22+
23+
const policyEngineCache: { [key in EngineType]: PolicyEngine | null } = {
24+
[EngineType.Kubernetes]: null,
25+
[EngineType.Terraform]: null,
26+
};
27+
28+
async function buildPolicyEngine(
29+
engineType: EngineType,
30+
): Promise<PolicyEngine> {
31+
const [
32+
policyEngineCoreDataPath,
33+
policyEngineMetaDataPath,
34+
] = getLocalCachePath(engineType);
1435

15-
export async function buildPolicyEngine(): Promise<PolicyEngine> {
16-
const policyEngineCoreDataPath = `${process.cwd()}/${LOCAL_POLICY_ENGINE_WASM_PATH}`;
17-
const policyEngineMetaDataPath = `${process.cwd()}/${LOCAL_POLICY_ENGINE_DATA_PATH}`;
1836
try {
1937
const wasmFile = fs.readFileSync(policyEngineCoreDataPath);
2038
const policyMetaData = fs.readFileSync(policyEngineMetaDataPath);
@@ -44,17 +62,13 @@ class PolicyEngine {
4462
return this.opaWasmInstance.evaluate(data)[0].result;
4563
}
4664

47-
public async scanFiles(
48-
filesToScan: IacFileData[],
49-
): Promise<IacFileScanResult[]> {
65+
public scanFile(iacFile: IacFileData): IacFileScanResult {
5066
try {
51-
return filesToScan.map((iacFile: IacFileData) => {
52-
const violatedPolicies = this.evaluate(iacFile.jsonContent);
53-
return {
54-
...iacFile,
55-
violatedPolicies,
56-
};
57-
});
67+
const violatedPolicies = this.evaluate(iacFile.jsonContent);
68+
return {
69+
...iacFile,
70+
violatedPolicies,
71+
};
5872
} catch (err) {
5973
// TODO: to distinguish between different failure reasons
6074
throw new Error(`Failed to run policy engine: ${err}`);

‎src/cli/commands/test/iac-local-execution/results-formatter.ts

+12-5
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
1-
import { IacFileScanResult, PolicyMetadata } from './types';
1+
import { EngineType, IacFileScanResult, PolicyMetadata } from './types';
22
import { SEVERITY } from '../../../../lib/snyk-test/common';
3+
import { IacProjectType } from '../../../../lib/iac/constants';
34
// import {
45
// issuesToLineNumbers,
56
// CloudConfigFileTypes,
@@ -34,16 +35,22 @@ export function formatResults(
3435
// }
3536
// }
3637

38+
const engineTypeToProjectType = {
39+
[EngineType.Kubernetes]: IacProjectType.K8S,
40+
[EngineType.Terraform]: IacProjectType.TERRAFORM,
41+
};
42+
3743
function iacLocalFileScanToFormattedResult(
3844
iacFileScanResult: IacFileScanResult,
3945
severityThreshold?: SEVERITY,
4046
) {
4147
const formattedIssues = iacFileScanResult.violatedPolicies.map((policy) => {
4248
// TODO: make sure we handle this issue with annotations:
4349
// https://github.com/snyk/registry/pull/17277
44-
const cloudConfigPath = [`[DocId:${iacFileScanResult.docId}]`].concat(
45-
policy.msg.split('.'),
46-
);
50+
const cloudConfigPath =
51+
iacFileScanResult.docId !== undefined
52+
? [`[DocId:${iacFileScanResult.docId}]`].concat(policy.msg.split('.'))
53+
: policy.msg.split('.');
4754
const lineNumber = -1;
4855
// TODO: once package becomes public, restore the commented out code for having the issue-to-line-number functionality
4956
// try {
@@ -80,7 +87,7 @@ function iacLocalFileScanToFormattedResult(
8087
),
8188
},
8289
isPrivate: true,
83-
packageManager: 'k8sconfig',
90+
packageManager: engineTypeToProjectType[iacFileScanResult.engineType],
8491
targetFile: iacFileScanResult.filePath,
8592
};
8693
}

‎src/cli/commands/test/iac-local-execution/types.ts

+5
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ export type IacFileMetadata = IacFileInDirectory;
55
export interface IacFileData extends IacFileMetadata {
66
jsonContent: Record<string, any>;
77
fileContent: string;
8+
engineType: EngineType;
89
docId?: number;
910
}
1011
export interface IacFileScanResult extends IacFileData {
@@ -16,6 +17,10 @@ export interface OpaWasmInstance {
1617
setData: (data: Record<string, any>) => void;
1718
}
1819

20+
export enum EngineType {
21+
Kubernetes,
22+
Terraform,
23+
}
1924
export interface PolicyMetadata {
2025
id: string;
2126
publicId: string;

‎src/cli/commands/test/iac-output.ts

+1-13
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,6 @@ function formatIacIssue(
3030
introducedBy = `\n introduced by ${pathStr}`;
3131
}
3232

33-
const description = extractOverview(issue.description).trim();
34-
const descriptionLine = `\n ${description}\n`;
3533
const severityColor = getSeveritiesColour(issue.severity);
3634

3735
return (
@@ -43,20 +41,10 @@ function formatIacIssue(
4341
` [${issue.id}]` +
4442
name +
4543
introducedBy +
46-
descriptionLine
44+
'\n'
4745
);
4846
}
4947

50-
function extractOverview(description: string): string {
51-
if (!description) {
52-
return '';
53-
}
54-
55-
const overviewRegExp = /## Overview([\s\S]*?)(?=##|(# Details))/m;
56-
const overviewMatches = overviewRegExp.exec(description);
57-
return (overviewMatches && overviewMatches[1]) || '';
58-
}
59-
6048
export function getIacDisplayedOutput(
6149
iacTest: IacTestResponse,
6250
testedInfoText: string,

0 commit comments

Comments
 (0)