Skip to content

Commit 9e47ea3

Browse files
eirsepgithub-actions[bot]
eirsep
authored and
github-actions[bot]
committed
fix null query filter conversion from sigma to query string query (#722)
* fix null query filter conversion from sigma to query string query Signed-off-by: Surya Sashank Nistala <[email protected]> * fix rule to query conversion tests for null filter Signed-off-by: Surya Sashank Nistala <[email protected]> * enhance test to verify non null doc doesnt match null query Signed-off-by: Surya Sashank Nistala <[email protected]> --------- Signed-off-by: Surya Sashank Nistala <[email protected]> (cherry picked from commit a59a014)
1 parent 8990aed commit 9e47ea3

File tree

4 files changed

+168
-7
lines changed

4 files changed

+168
-7
lines changed

‎src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -131,7 +131,7 @@ public OSQueryBackend(Map<String, String> fieldMappings, boolean collectErrors,
131131
this.reEscapeChar = "\\";
132132
this.reExpression = "%s: /%s/";
133133
this.cidrExpression = "%s: \"%s\"";
134-
this.fieldNullExpression = "%s: null";
134+
this.fieldNullExpression = "%s: (NOT [* TO *])";
135135
this.unboundValueStrExpression = "\"%s\"";
136136
this.unboundValueNumExpression = "\"%s\"";
137137
this.unboundWildcardExpression = "%s";

‎src/test/java/org/opensearch/securityanalytics/DetectorThreatIntelIT.java

Lines changed: 95 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,11 +26,14 @@
2626

2727
import static java.util.Collections.emptyList;
2828
import static org.opensearch.securityanalytics.TestHelpers.randomDetectorType;
29+
import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithInputs;
2930
import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithInputsAndThreatIntel;
3031
import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithInputsAndThreatIntelAndTriggers;
3132
import static org.opensearch.securityanalytics.TestHelpers.randomDoc;
3233
import static org.opensearch.securityanalytics.TestHelpers.randomDocWithIpIoc;
34+
import static org.opensearch.securityanalytics.TestHelpers.randomDocWithNullField;
3335
import static org.opensearch.securityanalytics.TestHelpers.randomIndex;
36+
import static org.opensearch.securityanalytics.TestHelpers.randomNullRule;
3437
import static org.opensearch.securityanalytics.TestHelpers.randomRule;
3538
import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping;
3639
import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ENABLE_WORKFLOW_USAGE;
@@ -150,6 +153,94 @@ public void testCreateDetectorWithThreatIntelEnabled_updateDetectorWithThreatInt
150153
assertEquals(1, noOfSigmaRuleMatches);
151154
}
152155

156+
public void testCreateDetectorForSigmaRuleWithNullCondition() throws IOException {
157+
158+
updateClusterSetting(ENABLE_WORKFLOW_USAGE.getKey(), "true");
159+
String index = createTestIndex(randomIndex(), windowsIndexMapping());
160+
161+
// Execute CreateMappingsAction to add alias mapping for index
162+
Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI);
163+
// both req params and req body are supported
164+
createMappingRequest.setJsonEntity(
165+
"{ \"index_name\":\"" + index + "\"," +
166+
" \"rule_topic\":\"" + randomDetectorType() + "\", " +
167+
" \"partial\":true" +
168+
"}"
169+
);
170+
171+
Response createMappingResponse = client().performRequest(createMappingRequest);
172+
173+
assertEquals(HttpStatus.SC_OK, createMappingResponse.getStatusLine().getStatusCode());
174+
175+
String testOpCode = "Test";
176+
177+
String randomDocRuleId = createRule(randomNullRule());
178+
List<DetectorRule> detectorRules = List.of(new DetectorRule(randomDocRuleId));
179+
DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), detectorRules,
180+
emptyList());
181+
DetectorTrigger trigger = new DetectorTrigger("all", "all", "high", List.of(randomDetectorType()), emptyList(), emptyList(), List.of(), emptyList(), List.of(DetectorTrigger.RULES_DETECTION_TYPE, DetectorTrigger.THREAT_INTEL_DETECTION_TYPE));
182+
Detector detector = randomDetectorWithInputs(List.of(input));
183+
Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector));
184+
185+
String request = "{\n" +
186+
" \"query\" : {\n" +
187+
" \"match_all\":{\n" +
188+
" }\n" +
189+
" }\n" +
190+
"}";
191+
192+
assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse));
193+
Map<String, Object> responseBody = asMap(createResponse);
194+
195+
String detectorId = responseBody.get("_id").toString();
196+
request = "{\n" +
197+
" \"query\" : {\n" +
198+
" \"match\":{\n" +
199+
" \"_id\": \"" + detectorId + "\"\n" +
200+
" }\n" +
201+
" }\n" +
202+
"}";
203+
List<SearchHit> hits = executeSearch(Detector.DETECTORS_INDEX, request);
204+
SearchHit hit = hits.get(0);
205+
Map<String, Object> detectorMap = (HashMap<String, Object>) (hit.getSourceAsMap().get("detector"));
206+
List inputArr = (List) detectorMap.get("inputs");
207+
208+
209+
List<String> monitorIds = ((List<String>) (detectorMap).get("monitor_id"));
210+
assertEquals(1, monitorIds.size());
211+
212+
Response getMonitorResponse = getAlertingMonitor(client(), monitorIds.get(0));
213+
Map<String, Object> alertingMonitor = asMap(getMonitorResponse);
214+
assertNotNull(alertingMonitor);
215+
String workflowId = ((List<String>) detectorMap.get("workflow_ids")).get(0);
216+
217+
indexDoc(index, "1", randomDocWithNullField());
218+
indexDoc(index, "2", randomDoc());
219+
220+
Response executeResponse = executeAlertingWorkflow(workflowId, Collections.emptyMap());
221+
222+
List<Map<String, Object>> monitorRunResults = (List<Map<String, Object>>) entityAsMap(executeResponse).get("monitor_run_results");
223+
assertEquals(1, monitorRunResults.size());
224+
225+
Map<String, Object> docLevelQueryResults = ((List<Map<String, Object>>) ((Map<String, Object>) monitorRunResults.get(0).get("input_results")).get("results")).get(0);
226+
int noOfSigmaRuleMatches = docLevelQueryResults.size();
227+
assertEquals(1, noOfSigmaRuleMatches);
228+
String queryId = docLevelQueryResults.keySet().stream().findAny().get();
229+
ArrayList<String> docs = (ArrayList<String>) docLevelQueryResults.get(queryId);
230+
assertEquals(docs.size(), 1);
231+
232+
indexDoc(index, "3", randomDoc());
233+
Response executeResponse1 = executeAlertingWorkflow(workflowId, Collections.emptyMap());
234+
235+
List<Map<String, Object>> monitorRunResults1 = (List<Map<String, Object>>) entityAsMap(executeResponse1).get("monitor_run_results");
236+
assertEquals(1, monitorRunResults1.size());
237+
238+
Map<String, Object> docLevelQueryResults1 = ((List<Map<String, Object>>) ((Map<String, Object>) monitorRunResults1.get(0).get("input_results")).get("results")).get(0);
239+
int noOfSigmaRuleMatches1 = docLevelQueryResults1.size();
240+
assertEquals(0, noOfSigmaRuleMatches1);
241+
242+
}
243+
153244
public void testCreateDetectorWithThreatIntelDisabled_updateDetectorWithThreatIntelEnabled() throws IOException {
154245

155246
updateClusterSetting(ENABLE_WORKFLOW_USAGE.getKey(), "true");
@@ -594,8 +685,8 @@ public void testCreateDetectorWithThreatIntelDisabled_triggerWithThreatIntelDete
594685
verifyWorkflow(detectorMap, monitorIds, 1);
595686

596687
int i = 1;
597-
while (i<4) {
598-
indexDoc(index, i + "", randomDocWithIpIoc(5, 3, i+""));
688+
while (i < 4) {
689+
indexDoc(index, i + "", randomDocWithIpIoc(5, 3, i + ""));
599690
i++;
600691
}
601692
String workflowId = ((List<String>) detectorMap.get("workflow_ids")).get(0);
@@ -686,8 +777,8 @@ public void testCreateDetectorWithThreatIntelDisabled_triggerWithRulesDetectionT
686777
verifyWorkflow(detectorMap, monitorIds, 1);
687778

688779
int i = 1;
689-
while (i<4) {
690-
indexDoc(index, i + "", randomDocWithIpIoc(5, 3, i+""));
780+
while (i < 4) {
781+
indexDoc(index, i + "", randomDocWithIpIoc(5, 3, i + ""));
691782
i++;
692783
}
693784
String workflowId = ((List<String>) detectorMap.get("workflow_ids")).get(0);

‎src/test/java/org/opensearch/securityanalytics/TestHelpers.java

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -259,6 +259,38 @@ public static String randomRule() {
259259
"level: high";
260260
}
261261

262+
263+
264+
public static String randomNullRule() {
265+
return "title: null field\n" +
266+
"id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" +
267+
"description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" +
268+
"references:\n" +
269+
" - https://attack.mitre.org/tactics/TA0008/\n" +
270+
" - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" +
271+
" - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" +
272+
" - https://github.com/zeronetworks/rpcfirewall\n" +
273+
" - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" +
274+
"tags:\n" +
275+
" - attack.defense_evasion\n" +
276+
"status: experimental\n" +
277+
"author: Sagie Dulce, Dekel Paz\n" +
278+
"date: 2022/01/01\n" +
279+
"modified: 2022/01/01\n" +
280+
"logsource:\n" +
281+
" product: rpc_firewall\n" +
282+
" category: application\n" +
283+
" definition: 'Requirements: install and apply the RPC Firew all to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" +
284+
"detection:\n" +
285+
" selection:\n" +
286+
" EventID: 22\n" +
287+
" RecordNumber: null\n" +
288+
" condition: selection\n" +
289+
"falsepositives:\n" +
290+
" - Legitimate usage of remote file encryption\n" +
291+
"level: high";
292+
}
293+
262294
public static String randomRuleForMappingView(String field) {
263295
return "title: Remote Encrypting File System Abuse\n" +
264296
"id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" +
@@ -1604,6 +1636,44 @@ public static String randomDocWithIpIoc(int severity, int version, String ioc)
16041636

16051637
}
16061638

1639+
public static String randomDocWithNullField() {
1640+
return "{\n" +
1641+
"\"@timestamp\":\"2020-02-04T14:59:39.343541+00:00\",\n" +
1642+
"\"EventTime\":\"2020-02-04T14:59:39.343541+00:00\",\n" +
1643+
"\"HostName\":\"EC2AMAZ-EPO7HKA\",\n" +
1644+
"\"Keywords\":\"9223372036854775808\",\n" +
1645+
"\"SeverityValue\":2,\n" +
1646+
"\"Severity\":\"INFO\",\n" +
1647+
"\"EventID\":22,\n" +
1648+
"\"SourceName\":\"Microsoft-Windows-Sysmon\",\n" +
1649+
"\"ProviderGuid\":\"{5770385F-C22A-43E0-BF4C-06F5698FFBD9}\",\n" +
1650+
"\"Version\":5,\n" +
1651+
"\"TaskValue\":22,\n" +
1652+
"\"OpcodeValue\":0,\n" +
1653+
"\"RecordNumber\":null,\n" +
1654+
"\"ExecutionProcessID\":1996,\n" +
1655+
"\"ExecutionThreadID\":2616,\n" +
1656+
"\"Channel\":\"Microsoft-Windows-Sysmon/Operational\",\n" +
1657+
"\"Domain\":\"NTAUTHORITY\",\n" +
1658+
"\"AccountName\":\"SYSTEM\",\n" +
1659+
"\"UserID\":\"S-1-5-18\",\n" +
1660+
"\"AccountType\":\"User\",\n" +
1661+
"\"Message\":\"Dns query:\\r\\nRuleName: \\r\\nUtcTime: 2020-02-04 14:59:38.349\\r\\nProcessGuid: {b3c285a4-3cda-5dc0-0000-001077270b00}\\r\\nProcessId: 1904\\r\\nQueryName: EC2AMAZ-EPO7HKA\\r\\nQueryStatus: 0\\r\\nQueryResults: 172.31.46.38;\\r\\nImage: C:\\\\Program Files\\\\nxlog\\\\nxlog.exe\",\n" +
1662+
"\"Category\":\"Dns query (rule: DnsQuery)\",\n" +
1663+
"\"Opcode\":\"Info\",\n" +
1664+
"\"UtcTime\":\"2020-02-04 14:59:38.349\",\n" +
1665+
"\"ProcessGuid\":\"{b3c285a4-3cda-5dc0-0000-001077270b00}\",\n" +
1666+
"\"ProcessId\":\"1904\",\"QueryName\":\"EC2AMAZ-EPO7HKA\",\"QueryStatus\":\"0\",\n" +
1667+
"\"QueryResults\":\"172.31.46.38;\",\n" +
1668+
"\"Image\":\"C:\\\\Program Files\\\\nxlog\\\\regsvr32.exe\",\n" +
1669+
"\"EventReceivedTime\":\"2020-02-04T14:59:40.780905+00:00\",\n" +
1670+
"\"SourceModuleName\":\"in\",\n" +
1671+
"\"SourceModuleType\":\"im_msvistalog\",\n" +
1672+
"\"CommandLine\": \"eachtest\",\n" +
1673+
"\"Initiated\": \"true\"\n" +
1674+
"}";
1675+
}
1676+
16071677
public static String randomDoc() {
16081678
return "{\n" +
16091679
"\"@timestamp\":\"2020-02-04T14:59:39.343541+00:00\",\n" +

‎src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -288,7 +288,7 @@ public void testConvertValueNull() throws IOException, SigmaError {
288288
" sel:\n" +
289289
" fieldA1: null\n" +
290290
" condition: sel", false));
291-
Assert.assertEquals("mappedA: null", queries.get(0).toString());
291+
Assert.assertEquals("mappedA: (NOT [* TO *])", queries.get(0).toString());
292292
}
293293

294294
public void testConvertValueRegex() throws IOException, SigmaError {
@@ -531,7 +531,7 @@ public void testConvertOrInUnallowedValueType() throws IOException, SigmaError {
531531
" - value2\n" +
532532
" - null\n" +
533533
" condition: sel", false));
534-
Assert.assertEquals("(mappedA: \"value1\") OR (mappedA: \"value2\") OR (mappedA: null)", queries.get(0).toString());
534+
Assert.assertEquals("(mappedA: \"value1\") OR (mappedA: \"value2\") OR (mappedA: (NOT [* TO *]))", queries.get(0).toString());
535535
}
536536

537537
public void testConvertOrInListNumbers() throws IOException, SigmaError {

0 commit comments

Comments
 (0)