Skip to content

Commit 3e4e755

Browse files
eirsepeirsep
committed
fix null query filter conversion from sigma to query string query (opensearch-project#722)
* fix null query filter conversion from sigma to query string query Signed-off-by: Surya Sashank Nistala <[email protected]> * fix rule to query conversion tests for null filter Signed-off-by: Surya Sashank Nistala <[email protected]> * enhance test to verify non null doc doesnt match null query Signed-off-by: Surya Sashank Nistala <[email protected]> --------- Signed-off-by: Surya Sashank Nistala <[email protected]>
1 parent a941222 commit 3e4e755

File tree

3 files changed

+162
-3
lines changed

3 files changed

+162
-3
lines changed

‎src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -131,7 +131,7 @@ public OSQueryBackend(Map<String, String> fieldMappings, boolean collectErrors,
131131
this.reEscapeChar = "\\";
132132
this.reExpression = "%s: /%s/";
133133
this.cidrExpression = "%s: \"%s\"";
134-
this.fieldNullExpression = "%s: null";
134+
this.fieldNullExpression = "%s: (NOT [* TO *])";
135135
this.unboundValueStrExpression = "%s: \"%s\"";
136136
this.unboundValueNumExpression = "%s: %s";
137137
this.unboundWildcardExpression = "%s: %s";

‎src/test/java/org/opensearch/securityanalytics/TestHelpers.java

Lines changed: 159 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -209,6 +209,65 @@ public static String randomRule() {
209209
"level: high";
210210
}
211211

212+
public static String randomNullRule() {
213+
return "title: null field\n" +
214+
"id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" +
215+
"description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" +
216+
"references:\n" +
217+
" - https://attack.mitre.org/tactics/TA0008/\n" +
218+
" - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" +
219+
" - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" +
220+
" - https://github.com/zeronetworks/rpcfirewall\n" +
221+
" - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" +
222+
"tags:\n" +
223+
" - attack.defense_evasion\n" +
224+
"status: experimental\n" +
225+
"author: Sagie Dulce, Dekel Paz\n" +
226+
"date: 2022/01/01\n" +
227+
"modified: 2022/01/01\n" +
228+
"logsource:\n" +
229+
" product: rpc_firewall\n" +
230+
" category: application\n" +
231+
" definition: 'Requirements: install and apply the RPC Firew all to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" +
232+
"detection:\n" +
233+
" selection:\n" +
234+
" EventID: 22\n" +
235+
" RecordNumber: null\n" +
236+
" condition: selection\n" +
237+
"falsepositives:\n" +
238+
" - Legitimate usage of remote file encryption\n" +
239+
"level: high";
240+
}
241+
242+
public static String randomRuleForMappingView(String field) {
243+
return "title: Remote Encrypting File System Abuse\n" +
244+
"id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" +
245+
"description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" +
246+
"references:\n" +
247+
" - https://attack.mitre.org/tactics/TA0008/\n" +
248+
" - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" +
249+
" - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" +
250+
" - https://github.com/zeronetworks/rpcfirewall\n" +
251+
" - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" +
252+
"tags:\n" +
253+
" - attack.defense_evasion\n" +
254+
"status: experimental\n" +
255+
"author: Sagie Dulce, Dekel Paz\n" +
256+
"date: 2022/01/01\n" +
257+
"modified: 2022/01/01\n" +
258+
"logsource:\n" +
259+
" product: rpc_firewall\n" +
260+
" category: application\n" +
261+
" definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" +
262+
"detection:\n" +
263+
" selection:\n" +
264+
" "+ field + ": 'ACL'\n" +
265+
" condition: selection\n" +
266+
"falsepositives:\n" +
267+
" - Legitimate usage of remote file encryption\n" +
268+
"level: high";
269+
}
270+
212271
public static String randomRuleForCustomLogType() {
213272
return "title: Remote Encrypting File System Abuse\n" +
214273
"id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" +
@@ -1296,6 +1355,106 @@ public static String randomDoc(int severity, int version, String opCode) {
12961355

12971356
}
12981357

1358+
public static String randomDocOnlyNumericAndDate(int severity, int version, String opCode) {
1359+
String doc = "{\n" +
1360+
"\"EventTime\":\"2020-02-04T14:59:39.343541+00:00\",\n" +
1361+
"\"ExecutionProcessID\":2001,\n" +
1362+
"\"ExecutionThreadID\":2616,\n" +
1363+
"\"EventID\": 1234,\n" +
1364+
"\"TaskValue\":22\n" +
1365+
"}";
1366+
return String.format(Locale.ROOT, doc, severity, version, opCode);
1367+
}
1368+
1369+
public static String randomDocOnlyNumericAndText(int severity, int version, String opCode) {
1370+
String doc = "{\n" +
1371+
"\"TaskName\":\"SYSTEM\",\n" +
1372+
"\"ExecutionProcessID\":2001,\n" +
1373+
"\"ExecutionThreadID\":2616,\n" +
1374+
"\"EventID\": 1234,\n" +
1375+
"\"TaskValue\":22\n" +
1376+
"}";
1377+
return String.format(Locale.ROOT, doc, severity, version, opCode);
1378+
}
1379+
1380+
//Add IPs in HostName field.
1381+
public static String randomDocWithIpIoc(int severity, int version, String ioc) {
1382+
String doc = "{\n" +
1383+
"\"EventTime\":\"2020-02-04T14:59:39.343541+00:00\",\n" +
1384+
"\"HostName\":\"%s\",\n" +
1385+
"\"Keywords\":\"9223372036854775808\",\n" +
1386+
"\"SeverityValue\":%s,\n" +
1387+
"\"Severity\":\"INFO\",\n" +
1388+
"\"EventID\":22,\n" +
1389+
"\"SourceName\":\"Microsoft-Windows-Sysmon\",\n" +
1390+
"\"ProviderGuid\":\"{5770385F-C22A-43E0-BF4C-06F5698FFBD9}\",\n" +
1391+
"\"Version\":%s,\n" +
1392+
"\"TaskValue\":22,\n" +
1393+
"\"OpcodeValue\":0,\n" +
1394+
"\"RecordNumber\":9532,\n" +
1395+
"\"ExecutionProcessID\":1996,\n" +
1396+
"\"ExecutionThreadID\":2616,\n" +
1397+
"\"Channel\":\"Microsoft-Windows-Sysmon/Operational\",\n" +
1398+
"\"Domain\":\"NT AUTHORITY\",\n" +
1399+
"\"AccountName\":\"SYSTEM\",\n" +
1400+
"\"UserID\":\"S-1-5-18\",\n" +
1401+
"\"AccountType\":\"User\",\n" +
1402+
"\"Message\":\"Dns query:\\r\\nRuleName: \\r\\nUtcTime: 2020-02-04 14:59:38.349\\r\\nProcessGuid: {b3c285a4-3cda-5dc0-0000-001077270b00}\\r\\nProcessId: 1904\\r\\nQueryName: EC2AMAZ-EPO7HKA\\r\\nQueryStatus: 0\\r\\nQueryResults: 172.31.46.38;\\r\\nImage: C:\\\\Program Files\\\\nxlog\\\\nxlog.exe\",\n" +
1403+
"\"Category\":\"Dns query (rule: DnsQuery)\",\n" +
1404+
"\"Opcode\":\"blahblah\",\n" +
1405+
"\"UtcTime\":\"2020-02-04 14:59:38.349\",\n" +
1406+
"\"ProcessGuid\":\"{b3c285a4-3cda-5dc0-0000-001077270b00}\",\n" +
1407+
"\"ProcessId\":\"1904\",\"QueryName\":\"EC2AMAZ-EPO7HKA\",\"QueryStatus\":\"0\",\n" +
1408+
"\"QueryResults\":\"172.31.46.38;\",\n" +
1409+
"\"Image\":\"C:\\\\Program Files\\\\nxlog\\\\regsvr32.exe\",\n" +
1410+
"\"EventReceivedTime\":\"2020-02-04T14:59:40.780905+00:00\",\n" +
1411+
"\"SourceModuleName\":\"in\",\n" +
1412+
"\"SourceModuleType\":\"im_msvistalog\",\n" +
1413+
"\"CommandLine\": \"eachtest\",\n" +
1414+
"\"Initiated\": \"true\"\n" +
1415+
"}";
1416+
return String.format(Locale.ROOT, doc, ioc, severity, version);
1417+
1418+
}
1419+
1420+
public static String randomDocWithNullField() {
1421+
return "{\n" +
1422+
"\"@timestamp\":\"2020-02-04T14:59:39.343541+00:00\",\n" +
1423+
"\"EventTime\":\"2020-02-04T14:59:39.343541+00:00\",\n" +
1424+
"\"HostName\":\"EC2AMAZ-EPO7HKA\",\n" +
1425+
"\"Keywords\":\"9223372036854775808\",\n" +
1426+
"\"SeverityValue\":2,\n" +
1427+
"\"Severity\":\"INFO\",\n" +
1428+
"\"EventID\":22,\n" +
1429+
"\"SourceName\":\"Microsoft-Windows-Sysmon\",\n" +
1430+
"\"ProviderGuid\":\"{5770385F-C22A-43E0-BF4C-06F5698FFBD9}\",\n" +
1431+
"\"Version\":5,\n" +
1432+
"\"TaskValue\":22,\n" +
1433+
"\"OpcodeValue\":0,\n" +
1434+
"\"RecordNumber\":null,\n" +
1435+
"\"ExecutionProcessID\":1996,\n" +
1436+
"\"ExecutionThreadID\":2616,\n" +
1437+
"\"Channel\":\"Microsoft-Windows-Sysmon/Operational\",\n" +
1438+
"\"Domain\":\"NTAUTHORITY\",\n" +
1439+
"\"AccountName\":\"SYSTEM\",\n" +
1440+
"\"UserID\":\"S-1-5-18\",\n" +
1441+
"\"AccountType\":\"User\",\n" +
1442+
"\"Message\":\"Dns query:\\r\\nRuleName: \\r\\nUtcTime: 2020-02-04 14:59:38.349\\r\\nProcessGuid: {b3c285a4-3cda-5dc0-0000-001077270b00}\\r\\nProcessId: 1904\\r\\nQueryName: EC2AMAZ-EPO7HKA\\r\\nQueryStatus: 0\\r\\nQueryResults: 172.31.46.38;\\r\\nImage: C:\\\\Program Files\\\\nxlog\\\\nxlog.exe\",\n" +
1443+
"\"Category\":\"Dns query (rule: DnsQuery)\",\n" +
1444+
"\"Opcode\":\"Info\",\n" +
1445+
"\"UtcTime\":\"2020-02-04 14:59:38.349\",\n" +
1446+
"\"ProcessGuid\":\"{b3c285a4-3cda-5dc0-0000-001077270b00}\",\n" +
1447+
"\"ProcessId\":\"1904\",\"QueryName\":\"EC2AMAZ-EPO7HKA\",\"QueryStatus\":\"0\",\n" +
1448+
"\"QueryResults\":\"172.31.46.38;\",\n" +
1449+
"\"Image\":\"C:\\\\Program Files\\\\nxlog\\\\regsvr32.exe\",\n" +
1450+
"\"EventReceivedTime\":\"2020-02-04T14:59:40.780905+00:00\",\n" +
1451+
"\"SourceModuleName\":\"in\",\n" +
1452+
"\"SourceModuleType\":\"im_msvistalog\",\n" +
1453+
"\"CommandLine\": \"eachtest\",\n" +
1454+
"\"Initiated\": \"true\"\n" +
1455+
"}";
1456+
}
1457+
12991458
public static String randomDoc() {
13001459
return "{\n" +
13011460
"\"@timestamp\":\"2020-02-04T14:59:39.343541+00:00\",\n" +

‎src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -288,7 +288,7 @@ public void testConvertValueNull() throws IOException, SigmaError {
288288
" sel:\n" +
289289
" fieldA1: null\n" +
290290
" condition: sel", false));
291-
Assert.assertEquals("mappedA: null", queries.get(0).toString());
291+
Assert.assertEquals("mappedA: (NOT [* TO *])", queries.get(0).toString());
292292
}
293293

294294
public void testConvertValueRegex() throws IOException, SigmaError {
@@ -531,7 +531,7 @@ public void testConvertOrInUnallowedValueType() throws IOException, SigmaError {
531531
" - value2\n" +
532532
" - null\n" +
533533
" condition: sel", false));
534-
Assert.assertEquals("(mappedA: \"value1\") OR (mappedA: \"value2\") OR (mappedA: null)", queries.get(0).toString());
534+
Assert.assertEquals("(mappedA: \"value1\") OR (mappedA: \"value2\") OR (mappedA: (NOT [* TO *]))", queries.get(0).toString());
535535
}
536536

537537
public void testConvertOrInListNumbers() throws IOException, SigmaError {

0 commit comments

Comments
 (0)