Skip to content

Commit 865ad34

Browse files
jowg-amazonjowg-amazon
committed
Fix get mappings view API incorrectly returning ecs path (opensearch-project#867)
* add logic and integ tests to not add duplicate to log-types-config index Signed-off-by: Joanne Wang <[email protected]> * change naming for existingFieldMapping and change contains to equals Signed-off-by: Joanne Wang <[email protected]> --------- Signed-off-by: Joanne Wang <[email protected]>
1 parent 99931b4 commit 865ad34

File tree

3 files changed

+139
-12
lines changed

3 files changed

+139
-12
lines changed

‎src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -348,15 +348,26 @@ private List<FieldMappingDoc> mergeFieldMappings(List<FieldMappingDoc> existingF
348348
List<FieldMappingDoc> newFieldMappings = new ArrayList<>();
349349
fieldMappingDocs.forEach( newFieldMapping -> {
350350
Optional<FieldMappingDoc> foundFieldMappingDoc = Optional.empty();
351-
for (FieldMappingDoc e: existingFieldMappings) {
352-
if (e.getRawField().equals(newFieldMapping.getRawField())) {
351+
for (FieldMappingDoc existingFieldMapping: existingFieldMappings) {
352+
if (existingFieldMapping.getRawField().equals(newFieldMapping.getRawField())) {
353353
if ((
354-
e.get(defaultSchemaField) != null && newFieldMapping.get(defaultSchemaField) != null &&
355-
e.get(defaultSchemaField).equals(newFieldMapping.get(defaultSchemaField))
354+
existingFieldMapping.get(defaultSchemaField) != null && newFieldMapping.get(defaultSchemaField) != null &&
355+
existingFieldMapping.get(defaultSchemaField).equals(newFieldMapping.get(defaultSchemaField))
356356
) || (
357-
e.get(defaultSchemaField) == null && newFieldMapping.get(defaultSchemaField) == null
357+
existingFieldMapping.get(defaultSchemaField) == null && newFieldMapping.get(defaultSchemaField) == null
358358
)) {
359-
foundFieldMappingDoc = Optional.of(e);
359+
foundFieldMappingDoc = Optional.of(existingFieldMapping);
360+
}
361+
// Grabs the right side of the ID with "|" as the delimiter if present representing the ecs field from predefined mappings
362+
// Additional check to see if raw field path + log type combination is already in existingFieldMappings so a new one is not indexed
363+
} else {
364+
String id = existingFieldMapping.getId();
365+
int indexOfPipe = id.indexOf("|");
366+
if (indexOfPipe != -1) {
367+
String ecsIdField = id.substring(indexOfPipe + 1);
368+
if (ecsIdField.equals(newFieldMapping.getRawField()) && existingFieldMapping.getLogTypes().containsAll(newFieldMapping.getLogTypes())) {
369+
foundFieldMappingDoc = Optional.of(existingFieldMapping);
370+
}
360371
}
361372
}
362373
}

‎src/test/java/org/opensearch/securityanalytics/TestHelpers.java

Lines changed: 76 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,19 +6,19 @@
66

77
import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
88
import org.apache.lucene.tests.util.LuceneTestCase;
9+
import org.opensearch.core.common.bytes.BytesReference;
910
import org.opensearch.common.xcontent.LoggingDeprecationHandler;
11+
import org.opensearch.core.xcontent.NamedXContentRegistry;
12+
import org.opensearch.core.xcontent.ToXContent;
1013
import org.opensearch.common.xcontent.XContentFactory;
14+
import org.opensearch.core.xcontent.XContentBuilder;
15+
import org.opensearch.core.xcontent.XContentParser;
1116
import org.opensearch.common.xcontent.XContentType;
1217
import org.opensearch.commons.alerting.model.IntervalSchedule;
1318
import org.opensearch.commons.alerting.model.Schedule;
1419
import org.opensearch.commons.alerting.model.action.Action;
1520
import org.opensearch.commons.alerting.model.action.Throttle;
1621
import org.opensearch.commons.authuser.User;
17-
import org.opensearch.core.common.bytes.BytesReference;
18-
import org.opensearch.core.xcontent.NamedXContentRegistry;
19-
import org.opensearch.core.xcontent.ToXContent;
20-
import org.opensearch.core.xcontent.XContentBuilder;
21-
import org.opensearch.core.xcontent.XContentParser;
2222
import org.opensearch.script.Script;
2323
import org.opensearch.script.ScriptType;
2424
import org.opensearch.securityanalytics.model.CorrelationQuery;
@@ -239,6 +239,35 @@ public static String randomRule() {
239239
"level: high";
240240
}
241241

242+
public static String randomRuleWithRawField() {
243+
return "title: Remote Encrypting File System Abuse\n" +
244+
"id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" +
245+
"description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" +
246+
"references:\n" +
247+
" - https://attack.mitre.org/tactics/TA0008/\n" +
248+
" - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" +
249+
" - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" +
250+
" - https://github.com/zeronetworks/rpcfirewall\n" +
251+
" - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" +
252+
"tags:\n" +
253+
" - attack.defense_evasion\n" +
254+
"status: experimental\n" +
255+
"author: Sagie Dulce, Dekel Paz\n" +
256+
"date: 2022/01/01\n" +
257+
"modified: 2022/01/01\n" +
258+
"logsource:\n" +
259+
" product: rpc_firewall\n" +
260+
" category: application\n" +
261+
" definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" +
262+
"detection:\n" +
263+
" selection:\n" +
264+
" eventName: testinghere\n" +
265+
" condition: selection\n" +
266+
"falsepositives:\n" +
267+
" - Legitimate usage of remote file encryption\n" +
268+
"level: high";
269+
}
270+
242271
public static String randomNullRule() {
243272
return "title: null field\n" +
244273
"id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" +
@@ -1774,6 +1803,46 @@ public static String randomDoc() {
17741803
"}";
17751804
}
17761805

1806+
public static String randomNetworkDoc() {
1807+
return "{\n" +
1808+
"\"@timestamp\":\"2020-02-04T14:59:39.343541+00:00\",\n" +
1809+
"\"EventTime\":\"2020-02-04T14:59:39.343541+00:00\",\n" +
1810+
"\"HostName\":\"EC2AMAZ-EPO7HKA\",\n" +
1811+
"\"Keywords\":\"9223372036854775808\",\n" +
1812+
"\"SeverityValue\":2,\n" +
1813+
"\"Severity\":\"INFO\",\n" +
1814+
"\"EventID\":22,\n" +
1815+
"\"SourceName\":\"Microsoft-Windows-Sysmon\",\n" +
1816+
"\"SourceIp\":\"1.2.3.4\",\n" +
1817+
"\"ProviderGuid\":\"{5770385F-C22A-43E0-BF4C-06F5698FFBD9}\",\n" +
1818+
"\"Version\":5,\n" +
1819+
"\"TaskValue\":22,\n" +
1820+
"\"OpcodeValue\":0,\n" +
1821+
"\"RecordNumber\":9532,\n" +
1822+
"\"ExecutionProcessID\":1996,\n" +
1823+
"\"ExecutionThreadID\":2616,\n" +
1824+
"\"Channel\":\"Microsoft-Windows-Sysmon/Operational\",\n" +
1825+
"\"Domain\":\"NTAUTHORITY\",\n" +
1826+
"\"AccountName\":\"SYSTEM\",\n" +
1827+
"\"UserID\":\"S-1-5-18\",\n" +
1828+
"\"AccountType\":\"User\",\n" +
1829+
"\"Message\":\"Dns query:\\r\\nRuleName: \\r\\nUtcTime: 2020-02-04 14:59:38.349\\r\\nProcessGuid: {b3c285a4-3cda-5dc0-0000-001077270b00}\\r\\nProcessId: 1904\\r\\nQueryName: EC2AMAZ-EPO7HKA\\r\\nQueryStatus: 0\\r\\nQueryResults: 172.31.46.38;\\r\\nImage: C:\\\\Program Files\\\\nxlog\\\\nxlog.exe\",\n" +
1830+
"\"Category\":\"Dns query (rule: DnsQuery)\",\n" +
1831+
"\"Opcode\":\"Info\",\n" +
1832+
"\"UtcTime\":\"2020-02-04 14:59:38.349\",\n" +
1833+
"\"ProcessGuid\":\"{b3c285a4-3cda-5dc0-0000-001077270b00}\",\n" +
1834+
"\"ProcessId\":\"1904\",\"QueryName\":\"EC2AMAZ-EPO7HKA\",\"QueryStatus\":\"0\",\n" +
1835+
"\"QueryResults\":\"172.31.46.38;\",\n" +
1836+
"\"Image\":\"C:\\\\Program Files\\\\nxlog\\\\regsvr32.exe\",\n" +
1837+
"\"EventReceivedTime\":\"2020-02-04T14:59:40.780905+00:00\",\n" +
1838+
"\"SourceModuleName\":\"in\",\n" +
1839+
"\"SourceModuleType\":\"im_msvistalog\",\n" +
1840+
"\"CommandLine\": \"eachtest\",\n" +
1841+
"\"id.orig_h\": \"123.12.123.12\",\n" +
1842+
"\"Initiated\": \"true\"\n" +
1843+
"}";
1844+
}
1845+
17771846
public static String randomCloudtrailAggrDoc(String eventType, String accountId) {
17781847
return "{\n" +
17791848
" \"AccountName\": \"" + accountId + "\",\n" +
@@ -1791,6 +1860,7 @@ public static String randomVpcFlowDoc() {
17911860
" \"srcport\": 9000,\n" +
17921861
" \"dstport\": 8000,\n" +
17931862
" \"severity_id\": \"-1\",\n" +
1863+
" \"id.orig_h\": \"1.2.3.4\",\n" +
17941864
" \"class_name\": \"Network Activity\"\n" +
17951865
"}";
17961866
}
@@ -2366,7 +2436,7 @@ public static List<String> randomLowerCaseStringList() {
23662436
stringList.add(randomLowerCaseString());
23672437
return stringList;
23682438
}
2369-
2439+
23702440
public static XContentParser parser(String xc) throws IOException {
23712441
XContentParser parser = XContentType.JSON.xContent().createParser(xContentRegistry(), LoggingDeprecationHandler.INSTANCE, xc);
23722442
parser.nextToken();

‎src/test/java/org/opensearch/securityanalytics/resthandler/OCSFDetectorRestApiIT.java

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -439,6 +439,52 @@ public void testOCSFCloudtrailGetMappingsViewApi() throws IOException {
439439
assertEquals(24, unmappedFieldAliases.size());
440440
}
441441

442+
@SuppressWarnings("unchecked")
443+
public void testOCSFCloudtrailGetMappingsViewApiWithCustomRule() throws IOException {
444+
String index = createTestIndex("cloudtrail", ocsfCloudtrailMappings());
445+
446+
Request request = new Request("GET", SecurityAnalyticsPlugin.MAPPINGS_VIEW_BASE_URI);
447+
// both req params and req body are supported
448+
request.addParameter("index_name", index);
449+
request.addParameter("rule_topic", "cloudtrail");
450+
Response response = client().performRequest(request);
451+
assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode());
452+
Map<String, Object> respMap = responseAsMap(response);
453+
// Verify alias mappings
454+
Map<String, Object> props = (Map<String, Object>) respMap.get("properties");
455+
Assert.assertEquals(18, props.size());
456+
// Verify unmapped index fields
457+
List<String> unmappedIndexFields = (List<String>) respMap.get("unmapped_index_fields");
458+
assertEquals(20, unmappedIndexFields.size());
459+
// Verify unmapped field aliases
460+
List<String> unmappedFieldAliases = (List<String>) respMap.get("unmapped_field_aliases");
461+
assertEquals(25, unmappedFieldAliases.size());
462+
463+
// create a cloudtrail rule with a raw field
464+
String rule = randomRuleWithRawField();
465+
Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", "cloudtrail"),
466+
new StringEntity(rule), new BasicHeader("Content-Type", "application/json"));
467+
Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse));
468+
469+
// check the mapping view API again to ensure it's the same after rule is created
470+
Response response2 = client().performRequest(request);
471+
assertEquals(HttpStatus.SC_OK, response2.getStatusLine().getStatusCode());
472+
Map<String, Object> respMap2 = responseAsMap(response2);
473+
// Verify alias mappings
474+
Map<String, Object> props2 = (Map<String, Object>) respMap2.get("properties");
475+
Assert.assertEquals(18, props2.size());
476+
// Verify unmapped index fields
477+
List<String> unmappedIndexFields2 = (List<String>) respMap2.get("unmapped_index_fields");
478+
assertEquals(20, unmappedIndexFields2.size());
479+
// Verify unmapped field aliases
480+
List<String> unmappedFieldAliases2 = (List<String>) respMap2.get("unmapped_field_aliases");
481+
assertEquals(25, unmappedFieldAliases2.size());
482+
// Verify that first response and second response are the same after rule was indexed
483+
assertEquals(props, props2);
484+
assertEquals(unmappedIndexFields, unmappedIndexFields2);
485+
assertEquals(unmappedFieldAliases, unmappedFieldAliases2);
486+
}
487+
442488
@SuppressWarnings("unchecked")
443489
public void testOCSFVpcflowGetMappingsViewApi() throws IOException {
444490
String index = createTestIndex("vpcflow", ocsfVpcflowMappings());

0 commit comments

Comments
 (0)