Skip to content

Commit 61909b2

Browse files
eirsepeirsep
authored
[2.9] Backport [#722] fix null query filter conversion from sigma to query string query (#743)
* fix null query filter conversion from sigma to query string query (#722) * fix null query filter conversion from sigma to query string query Signed-off-by: Surya Sashank Nistala <[email protected]> * fix rule to query conversion tests for null filter Signed-off-by: Surya Sashank Nistala <[email protected]> * enhance test to verify non null doc doesnt match null query Signed-off-by: Surya Sashank Nistala <[email protected]> --------- Signed-off-by: Surya Sashank Nistala <[email protected]> * add test to verify null filter in sigma rules Signed-off-by: Surya Sashank Nistala <[email protected]> --------- Signed-off-by: Surya Sashank Nistala <[email protected]>
1 parent a941222 commit 61909b2

File tree

4 files changed

+250
-3
lines changed

4 files changed

+250
-3
lines changed

‎src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -131,7 +131,7 @@ public OSQueryBackend(Map<String, String> fieldMappings, boolean collectErrors,
131131
this.reEscapeChar = "\\";
132132
this.reExpression = "%s: /%s/";
133133
this.cidrExpression = "%s: \"%s\"";
134-
this.fieldNullExpression = "%s: null";
134+
this.fieldNullExpression = "%s: (NOT [* TO *])";
135135
this.unboundValueStrExpression = "%s: \"%s\"";
136136
this.unboundValueNumExpression = "%s: %s";
137137
this.unboundWildcardExpression = "%s: %s";

‎src/test/java/org/opensearch/securityanalytics/TestHelpers.java

Lines changed: 159 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -209,6 +209,65 @@ public static String randomRule() {
209209
"level: high";
210210
}
211211

212+
public static String randomNullRule() {
213+
return "title: null field\n" +
214+
"id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" +
215+
"description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" +
216+
"references:\n" +
217+
" - https://attack.mitre.org/tactics/TA0008/\n" +
218+
" - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" +
219+
" - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" +
220+
" - https://github.com/zeronetworks/rpcfirewall\n" +
221+
" - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" +
222+
"tags:\n" +
223+
" - attack.defense_evasion\n" +
224+
"status: experimental\n" +
225+
"author: Sagie Dulce, Dekel Paz\n" +
226+
"date: 2022/01/01\n" +
227+
"modified: 2022/01/01\n" +
228+
"logsource:\n" +
229+
" product: rpc_firewall\n" +
230+
" category: application\n" +
231+
" definition: 'Requirements: install and apply the RPC Firew all to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" +
232+
"detection:\n" +
233+
" selection:\n" +
234+
" EventID: 22\n" +
235+
" RecordNumber: null\n" +
236+
" condition: selection\n" +
237+
"falsepositives:\n" +
238+
" - Legitimate usage of remote file encryption\n" +
239+
"level: high";
240+
}
241+
242+
public static String randomRuleForMappingView(String field) {
243+
return "title: Remote Encrypting File System Abuse\n" +
244+
"id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" +
245+
"description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" +
246+
"references:\n" +
247+
" - https://attack.mitre.org/tactics/TA0008/\n" +
248+
" - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" +
249+
" - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" +
250+
" - https://github.com/zeronetworks/rpcfirewall\n" +
251+
" - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" +
252+
"tags:\n" +
253+
" - attack.defense_evasion\n" +
254+
"status: experimental\n" +
255+
"author: Sagie Dulce, Dekel Paz\n" +
256+
"date: 2022/01/01\n" +
257+
"modified: 2022/01/01\n" +
258+
"logsource:\n" +
259+
" product: rpc_firewall\n" +
260+
" category: application\n" +
261+
" definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" +
262+
"detection:\n" +
263+
" selection:\n" +
264+
" "+ field + ": 'ACL'\n" +
265+
" condition: selection\n" +
266+
"falsepositives:\n" +
267+
" - Legitimate usage of remote file encryption\n" +
268+
"level: high";
269+
}
270+
212271
public static String randomRuleForCustomLogType() {
213272
return "title: Remote Encrypting File System Abuse\n" +
214273
"id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" +
@@ -1296,6 +1355,106 @@ public static String randomDoc(int severity, int version, String opCode) {
12961355

12971356
}
12981357

1358+
public static String randomDocOnlyNumericAndDate(int severity, int version, String opCode) {
1359+
String doc = "{\n" +
1360+
"\"EventTime\":\"2020-02-04T14:59:39.343541+00:00\",\n" +
1361+
"\"ExecutionProcessID\":2001,\n" +
1362+
"\"ExecutionThreadID\":2616,\n" +
1363+
"\"EventID\": 1234,\n" +
1364+
"\"TaskValue\":22\n" +
1365+
"}";
1366+
return String.format(Locale.ROOT, doc, severity, version, opCode);
1367+
}
1368+
1369+
public static String randomDocOnlyNumericAndText(int severity, int version, String opCode) {
1370+
String doc = "{\n" +
1371+
"\"TaskName\":\"SYSTEM\",\n" +
1372+
"\"ExecutionProcessID\":2001,\n" +
1373+
"\"ExecutionThreadID\":2616,\n" +
1374+
"\"EventID\": 1234,\n" +
1375+
"\"TaskValue\":22\n" +
1376+
"}";
1377+
return String.format(Locale.ROOT, doc, severity, version, opCode);
1378+
}
1379+
1380+
//Add IPs in HostName field.
1381+
public static String randomDocWithIpIoc(int severity, int version, String ioc) {
1382+
String doc = "{\n" +
1383+
"\"EventTime\":\"2020-02-04T14:59:39.343541+00:00\",\n" +
1384+
"\"HostName\":\"%s\",\n" +
1385+
"\"Keywords\":\"9223372036854775808\",\n" +
1386+
"\"SeverityValue\":%s,\n" +
1387+
"\"Severity\":\"INFO\",\n" +
1388+
"\"EventID\":22,\n" +
1389+
"\"SourceName\":\"Microsoft-Windows-Sysmon\",\n" +
1390+
"\"ProviderGuid\":\"{5770385F-C22A-43E0-BF4C-06F5698FFBD9}\",\n" +
1391+
"\"Version\":%s,\n" +
1392+
"\"TaskValue\":22,\n" +
1393+
"\"OpcodeValue\":0,\n" +
1394+
"\"RecordNumber\":9532,\n" +
1395+
"\"ExecutionProcessID\":1996,\n" +
1396+
"\"ExecutionThreadID\":2616,\n" +
1397+
"\"Channel\":\"Microsoft-Windows-Sysmon/Operational\",\n" +
1398+
"\"Domain\":\"NT AUTHORITY\",\n" +
1399+
"\"AccountName\":\"SYSTEM\",\n" +
1400+
"\"UserID\":\"S-1-5-18\",\n" +
1401+
"\"AccountType\":\"User\",\n" +
1402+
"\"Message\":\"Dns query:\\r\\nRuleName: \\r\\nUtcTime: 2020-02-04 14:59:38.349\\r\\nProcessGuid: {b3c285a4-3cda-5dc0-0000-001077270b00}\\r\\nProcessId: 1904\\r\\nQueryName: EC2AMAZ-EPO7HKA\\r\\nQueryStatus: 0\\r\\nQueryResults: 172.31.46.38;\\r\\nImage: C:\\\\Program Files\\\\nxlog\\\\nxlog.exe\",\n" +
1403+
"\"Category\":\"Dns query (rule: DnsQuery)\",\n" +
1404+
"\"Opcode\":\"blahblah\",\n" +
1405+
"\"UtcTime\":\"2020-02-04 14:59:38.349\",\n" +
1406+
"\"ProcessGuid\":\"{b3c285a4-3cda-5dc0-0000-001077270b00}\",\n" +
1407+
"\"ProcessId\":\"1904\",\"QueryName\":\"EC2AMAZ-EPO7HKA\",\"QueryStatus\":\"0\",\n" +
1408+
"\"QueryResults\":\"172.31.46.38;\",\n" +
1409+
"\"Image\":\"C:\\\\Program Files\\\\nxlog\\\\regsvr32.exe\",\n" +
1410+
"\"EventReceivedTime\":\"2020-02-04T14:59:40.780905+00:00\",\n" +
1411+
"\"SourceModuleName\":\"in\",\n" +
1412+
"\"SourceModuleType\":\"im_msvistalog\",\n" +
1413+
"\"CommandLine\": \"eachtest\",\n" +
1414+
"\"Initiated\": \"true\"\n" +
1415+
"}";
1416+
return String.format(Locale.ROOT, doc, ioc, severity, version);
1417+
1418+
}
1419+
1420+
public static String randomDocWithNullField() {
1421+
return "{\n" +
1422+
"\"@timestamp\":\"2020-02-04T14:59:39.343541+00:00\",\n" +
1423+
"\"EventTime\":\"2020-02-04T14:59:39.343541+00:00\",\n" +
1424+
"\"HostName\":\"EC2AMAZ-EPO7HKA\",\n" +
1425+
"\"Keywords\":\"9223372036854775808\",\n" +
1426+
"\"SeverityValue\":2,\n" +
1427+
"\"Severity\":\"INFO\",\n" +
1428+
"\"EventID\":22,\n" +
1429+
"\"SourceName\":\"Microsoft-Windows-Sysmon\",\n" +
1430+
"\"ProviderGuid\":\"{5770385F-C22A-43E0-BF4C-06F5698FFBD9}\",\n" +
1431+
"\"Version\":5,\n" +
1432+
"\"TaskValue\":22,\n" +
1433+
"\"OpcodeValue\":0,\n" +
1434+
"\"RecordNumber\":null,\n" +
1435+
"\"ExecutionProcessID\":1996,\n" +
1436+
"\"ExecutionThreadID\":2616,\n" +
1437+
"\"Channel\":\"Microsoft-Windows-Sysmon/Operational\",\n" +
1438+
"\"Domain\":\"NTAUTHORITY\",\n" +
1439+
"\"AccountName\":\"SYSTEM\",\n" +
1440+
"\"UserID\":\"S-1-5-18\",\n" +
1441+
"\"AccountType\":\"User\",\n" +
1442+
"\"Message\":\"Dns query:\\r\\nRuleName: \\r\\nUtcTime: 2020-02-04 14:59:38.349\\r\\nProcessGuid: {b3c285a4-3cda-5dc0-0000-001077270b00}\\r\\nProcessId: 1904\\r\\nQueryName: EC2AMAZ-EPO7HKA\\r\\nQueryStatus: 0\\r\\nQueryResults: 172.31.46.38;\\r\\nImage: C:\\\\Program Files\\\\nxlog\\\\nxlog.exe\",\n" +
1443+
"\"Category\":\"Dns query (rule: DnsQuery)\",\n" +
1444+
"\"Opcode\":\"Info\",\n" +
1445+
"\"UtcTime\":\"2020-02-04 14:59:38.349\",\n" +
1446+
"\"ProcessGuid\":\"{b3c285a4-3cda-5dc0-0000-001077270b00}\",\n" +
1447+
"\"ProcessId\":\"1904\",\"QueryName\":\"EC2AMAZ-EPO7HKA\",\"QueryStatus\":\"0\",\n" +
1448+
"\"QueryResults\":\"172.31.46.38;\",\n" +
1449+
"\"Image\":\"C:\\\\Program Files\\\\nxlog\\\\regsvr32.exe\",\n" +
1450+
"\"EventReceivedTime\":\"2020-02-04T14:59:40.780905+00:00\",\n" +
1451+
"\"SourceModuleName\":\"in\",\n" +
1452+
"\"SourceModuleType\":\"im_msvistalog\",\n" +
1453+
"\"CommandLine\": \"eachtest\",\n" +
1454+
"\"Initiated\": \"true\"\n" +
1455+
"}";
1456+
}
1457+
12991458
public static String randomDoc() {
13001459
return "{\n" +
13011460
"\"@timestamp\":\"2020-02-04T14:59:39.343541+00:00\",\n" +

‎src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,15 @@
44
*/
55
package org.opensearch.securityanalytics.resthandler;
66

7+
import static java.util.Collections.emptyList;
78
import static org.opensearch.securityanalytics.TestHelpers.randomAggregationRule;
89
import static org.opensearch.securityanalytics.TestHelpers.randomDetector;
910
import static org.opensearch.securityanalytics.TestHelpers.randomDetectorType;
1011
import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithInputs;
1112
import static org.opensearch.securityanalytics.TestHelpers.randomDoc;
13+
import static org.opensearch.securityanalytics.TestHelpers.randomDocWithNullField;
1214
import static org.opensearch.securityanalytics.TestHelpers.randomIndex;
15+
import static org.opensearch.securityanalytics.TestHelpers.randomNullRule;
1316
import static org.opensearch.securityanalytics.TestHelpers.randomRule;
1417
import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping;
1518

@@ -37,6 +40,7 @@
3740
import org.opensearch.securityanalytics.model.Detector;
3841
import org.opensearch.securityanalytics.model.DetectorInput;
3942
import org.opensearch.securityanalytics.model.DetectorRule;
43+
import org.opensearch.securityanalytics.model.DetectorTrigger;
4044
import org.opensearch.securityanalytics.model.Rule;
4145

4246
public class DetectorMonitorRestApiIT extends SecurityAnalyticsRestTestCase {
@@ -974,6 +978,90 @@ else if (ruleId == minRuleId) {
974978
assertTrue(Arrays.asList("1", "2", "3", "4", "5", "6", "7", "8").containsAll(docLevelFinding));
975979
}
976980

981+
public void testCreateDetectorForSigmaRuleWithNullCondition() throws IOException {
982+
983+
String index = createTestIndex(randomIndex(), windowsIndexMapping());
984+
985+
// Execute CreateMappingsAction to add alias mapping for index
986+
Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI);
987+
// both req params and req body are supported
988+
createMappingRequest.setJsonEntity(
989+
"{ \"index_name\":\"" + index + "\"," +
990+
" \"rule_topic\":\"" + randomDetectorType() + "\", " +
991+
" \"partial\":true" +
992+
"}"
993+
);
994+
995+
Response createMappingResponse = client().performRequest(createMappingRequest);
996+
997+
assertEquals(HttpStatus.SC_OK, createMappingResponse.getStatusLine().getStatusCode());
998+
999+
String testOpCode = "Test";
1000+
1001+
String randomDocRuleId = createRule(randomNullRule());
1002+
List<DetectorRule> detectorRules = List.of(new DetectorRule(randomDocRuleId));
1003+
DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), detectorRules,
1004+
emptyList());
1005+
Detector detector = randomDetectorWithInputs(List.of(input));
1006+
Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector));
1007+
1008+
String request = "{\n" +
1009+
" \"query\" : {\n" +
1010+
" \"match_all\":{\n" +
1011+
" }\n" +
1012+
" }\n" +
1013+
"}";
1014+
1015+
assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse));
1016+
Map<String, Object> responseBody = asMap(createResponse);
1017+
1018+
String detectorId = responseBody.get("_id").toString();
1019+
request = "{\n" +
1020+
" \"query\" : {\n" +
1021+
" \"match\":{\n" +
1022+
" \"_id\": \"" + detectorId + "\"\n" +
1023+
" }\n" +
1024+
" }\n" +
1025+
"}";
1026+
List<SearchHit> hits = executeSearch(Detector.DETECTORS_INDEX, request);
1027+
SearchHit hit = hits.get(0);
1028+
Map<String, Object> detectorMap = (HashMap<String, Object>) (hit.getSourceAsMap().get("detector"));
1029+
String monitorId = ((List<String>) (detectorMap).get("monitor_id")).get(0);
1030+
List inputArr = (List) detectorMap.get("inputs");
1031+
1032+
1033+
List<String> monitorIds = ((List<String>) (detectorMap).get("monitor_id"));
1034+
assertEquals(1, monitorIds.size());
1035+
1036+
indexDoc(index, "1", randomDocWithNullField());
1037+
indexDoc(index, "2", randomDoc());
1038+
1039+
Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap());
1040+
Map<String, Object> executeResults = entityAsMap(executeResponse);
1041+
int noOfSigmaRuleMatches = ((List<Map<String, Object>>) ((Map<String, Object>) executeResults.get("input_results")).get("results")).get(0).size();
1042+
assertEquals(1, noOfSigmaRuleMatches);
1043+
1044+
Map<String, String> params = new HashMap<>();
1045+
params.put("detector_id", detectorId);
1046+
Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null);
1047+
Map<String, Object> getFindingsBody = entityAsMap(getFindingsResponse);
1048+
1049+
assertNotNull(getFindingsBody);
1050+
// When doc level monitor is being applied one finding is generated per document
1051+
assertEquals(1, getFindingsBody.get("total_findings"));
1052+
1053+
1054+
1055+
List<Map<String, Object>> findings = (List)getFindingsBody.get("findings");
1056+
List<String> foundDocIds = new ArrayList<>();
1057+
for(Map<String, Object> finding : findings) {
1058+
List<String> findingDocs = (List<String>)finding.get("related_doc_ids");
1059+
Assert.assertEquals(1, findingDocs.size());
1060+
foundDocIds.addAll(findingDocs);
1061+
}
1062+
assertTrue(Arrays.asList("1").containsAll(foundDocIds));
1063+
}
1064+
9771065
private static void assertRuleMonitorFinding(Map<String, Object> executeResults, String ruleId, int expectedDocCount, List<String> expectedTriggerResult) {
9781066
List<Map<String, Object>> buckets = ((List<Map<String, Object>>)(((Map<String, Object>)((Map<String, Object>)((Map<String, Object>)((List<Object>)((Map<String, Object>) executeResults.get("input_results")).get("results")).get(0)).get("aggregations")).get("result_agg")).get("buckets")));
9791067
Integer docCount = buckets.stream().mapToInt(it -> (Integer)it.get("doc_count")).sum();

‎src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -288,7 +288,7 @@ public void testConvertValueNull() throws IOException, SigmaError {
288288
" sel:\n" +
289289
" fieldA1: null\n" +
290290
" condition: sel", false));
291-
Assert.assertEquals("mappedA: null", queries.get(0).toString());
291+
Assert.assertEquals("mappedA: (NOT [* TO *])", queries.get(0).toString());
292292
}
293293

294294
public void testConvertValueRegex() throws IOException, SigmaError {
@@ -531,7 +531,7 @@ public void testConvertOrInUnallowedValueType() throws IOException, SigmaError {
531531
" - value2\n" +
532532
" - null\n" +
533533
" condition: sel", false));
534-
Assert.assertEquals("(mappedA: \"value1\") OR (mappedA: \"value2\") OR (mappedA: null)", queries.get(0).toString());
534+
Assert.assertEquals("(mappedA: \"value1\") OR (mappedA: \"value2\") OR (mappedA: (NOT [* TO *]))", queries.get(0).toString());
535535
}
536536

537537
public void testConvertOrInListNumbers() throws IOException, SigmaError {

0 commit comments

Comments
 (0)