Skip to content

Commit 2d10915

Browse files
engechaspetardzeht16eirsep
authored
Backports for 2.5 (#927)
* query_field_names bugfix (#335) Signed-off-by: Petar Dzepina <[email protected]> * Reduce log level for informative message (#203) Signed-off-by: Enrico Tröger <[email protected]> * fix detector writeTo() method missing fields (#695) * fix detector writeTo() method missing fields Signed-off-by: Surya Sashank Nistala <[email protected]> * fix test Signed-off-by: Surya Sashank Nistala <[email protected]> --------- Signed-off-by: Surya Sashank Nistala <[email protected]> * fix null query filter conversion from sigma to query string query (#722) * fix null query filter conversion from sigma to query string query Signed-off-by: Surya Sashank Nistala <[email protected]> * fix rule to query conversion tests for null filter Signed-off-by: Surya Sashank Nistala <[email protected]> * enhance test to verify non null doc doesnt match null query Signed-off-by: Surya Sashank Nistala <[email protected]> --------- Signed-off-by: Surya Sashank Nistala <[email protected]> * Bump version 2.5.1, fix build Signed-off-by: Chase Engelbrecht <[email protected]> * Manual cherry-pick of #873 Signed-off-by: Chase Engelbrecht <[email protected]> * Fix miss from manual cherry-pick Signed-off-by: Chase Engelbrecht <[email protected]> * Undo exceptional case not originally present Signed-off-by: Chase Engelbrecht <[email protected]> * Revert test from previous commit reversion Signed-off-by: Chase Engelbrecht <[email protected]> * Fix another miss from manual cherry-pick Signed-off-by: Chase Engelbrecht <[email protected]> * Revert behavior for no rules in detector Signed-off-by: Chase Engelbrecht <[email protected]> --------- Signed-off-by: Petar Dzepina <[email protected]> Signed-off-by: Enrico Tröger <[email protected]> Signed-off-by: Surya Sashank Nistala <[email protected]> Signed-off-by: Chase Engelbrecht <[email protected]> Co-authored-by: Petar Dzepina <[email protected]> Co-authored-by: Enrico Tröger <[email protected]> Co-authored-by: Surya Sashank Nistala <[email protected]>
1 parent 0025b7a commit 2d10915

File tree

12 files changed

+535
-207
lines changed

12 files changed

+535
-207
lines changed

‎build.gradle

Lines changed: 18 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import org.opensearch.gradle.test.RestIntegTestTask
66

77
buildscript {
88
ext {
9-
opensearch_version = System.getProperty("opensearch.version", "2.5.0-SNAPSHOT")
9+
opensearch_version = System.getProperty("opensearch.version", "2.5.1-SNAPSHOT")
1010
isSnapshot = "true" == System.getProperty("build.snapshot", "true")
1111
buildVersionQualifier = System.getProperty("build.version_qualifier", "")
1212
version_tokens = opensearch_version.tokenize('-')
@@ -125,6 +125,9 @@ repositories {
125125
}
126126

127127
sourceSets.main.java.srcDirs = ['src/main/generated','src/main/java']
128+
configurations {
129+
zipArchive
130+
}
128131

129132
dependencies {
130133
javaRestTestImplementation project.sourceSets.main.runtimeClasspath
@@ -134,6 +137,11 @@ dependencies {
134137
api "org.opensearch:common-utils:${common_utils_version}"
135138
api "org.opensearch.client:opensearch-rest-client:${opensearch_version}"
136139
implementation "org.jetbrains.kotlin:kotlin-stdlib:${kotlin_version}"
140+
141+
// Needed for integ tests
142+
zipArchive group: 'org.opensearch.plugin', name:'alerting', version: "${opensearch_build}"
143+
zipArchive group: 'org.opensearch.plugin', name:'opensearch-notifications-core', version: "${opensearch_build}"
144+
zipArchive group: 'org.opensearch.plugin', name:'notifications', version: "${opensearch_build}"
137145
}
138146

139147
// RPM & Debian build
@@ -209,15 +217,6 @@ integTest.getClusters().forEach{c -> {
209217
c.plugin(project.getObjects().fileProperty().value(bundle.getArchiveFile()))
210218
}}
211219

212-
String alertingFilePath = "src/test/resources/alerting"
213-
String alertingPlugin = "opensearch-alerting-" + plugin_no_snapshot + ".zip"
214-
String alertingRemoteFile = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/" + opensearch_no_snapshot + "/latest/linux/x64/tar/builds/opensearch/plugins/" + alertingPlugin
215-
String notificationsFilePath = "src/test/resources/notifications"
216-
String notificationsCoreFilePath = "src/test/resources/notifications-core"
217-
String notificationsPlugin = "opensearch-notifications-" + plugin_no_snapshot + ".zip"
218-
String notificationsCorePlugin = "opensearch-notifications-core-" + plugin_no_snapshot + ".zip"
219-
String notificationsRemoteFile = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/" + opensearch_no_snapshot + "/latest/linux/x64/tar/builds/opensearch/plugins/" + notificationsPlugin
220-
String notificationsCoreRemoteFile = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/" + opensearch_no_snapshot + "/latest/linux/x64/tar/builds/opensearch/plugins/" + notificationsCorePlugin
221220
testClusters.integTest {
222221
testDistribution = 'ARCHIVE'
223222

@@ -233,58 +232,33 @@ testClusters.integTest {
233232
debugPort += 1
234233
}
235234
}
236-
setting 'path.repo', repo.absolutePath
237235
plugin(provider({
238236
new RegularFile() {
239237
@Override
240238
File getAsFile() {
241-
File dir = new File(rootDir.path + "/" + alertingFilePath)
242-
243-
if (!dir.exists()) {
244-
dir.mkdirs()
245-
}
246-
247-
File f = new File(dir, alertingPlugin)
248-
if (!f.exists()) {
249-
new URL(alertingRemoteFile).withInputStream{ ins -> f.withOutputStream{ it << ins }}
250-
}
251-
fileTree(alertingFilePath).getSingleFile()
239+
return configurations.zipArchive.asFileTree.matching {
240+
include '**/opensearch-notifications-core*'
241+
}.singleFile
252242
}
253243
}
254244
}))
255245
plugin(provider({
256246
new RegularFile() {
257247
@Override
258248
File getAsFile() {
259-
File dir = new File(rootDir.path + "/" + notificationsCoreFilePath)
260-
261-
if (!dir.exists()) {
262-
dir.mkdirs()
263-
}
264-
265-
File f = new File(dir, notificationsCorePlugin)
266-
if (!f.exists()) {
267-
new URL(notificationsCoreRemoteFile).withInputStream{ ins -> f.withOutputStream{ it << ins }}
268-
}
269-
fileTree(notificationsCoreFilePath).getSingleFile()
249+
return configurations.zipArchive.asFileTree.matching {
250+
include '**/notifications*'
251+
}.singleFile
270252
}
271253
}
272254
}))
273255
plugin(provider({
274256
new RegularFile() {
275257
@Override
276258
File getAsFile() {
277-
File dir = new File(rootDir.path + "/" + notificationsFilePath)
278-
279-
if (!dir.exists()) {
280-
dir.mkdirs()
281-
}
282-
283-
File f = new File(dir, notificationsPlugin)
284-
if (!f.exists()) {
285-
new URL(notificationsRemoteFile).withInputStream{ ins -> f.withOutputStream{ it << ins }}
286-
}
287-
fileTree(notificationsFilePath).getSingleFile()
259+
return configurations.zipArchive.asFileTree.matching {
260+
include '**/alerting*'
261+
}.singleFile
288262
}
289263
}
290264
}))

‎src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -233,7 +233,7 @@ private String executorName() {
233233
}
234234

235235
private void deleteOldIndices(String tag, String... indices) {
236-
logger.error("info deleteOldIndices");
236+
logger.info("info deleteOldIndices");
237237
ClusterStateRequest clusterStateRequest = new ClusterStateRequest()
238238
.clear()
239239
.indices(indices)

‎src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,9 +71,11 @@ public void createMappingAction(String indexName, String ruleTopic, String alias
7171
// since you can't update documents in non-write indices
7272
String index = indexName;
7373
boolean shouldUpsertIndexTemplate = IndexUtils.isConcreteIndex(indexName, this.clusterService.state()) == false;
74-
if (IndexUtils.isDataStream(indexName, this.clusterService.state())) {
74+
if (IndexUtils.isDataStream(indexName, this.clusterService.state()) || IndexUtils.isAlias(indexName, this.clusterService.state())) {
75+
log.debug("{} is an alias or datastream. Fetching write index for create mapping action.", indexName);
7576
String writeIndex = IndexUtils.getWriteIndex(indexName, this.clusterService.state());
7677
if (writeIndex != null) {
78+
log.debug("Write index for {} is {}", indexName, writeIndex);
7779
index = writeIndex;
7880
}
7981
}
@@ -85,6 +87,7 @@ public void onResponse(GetMappingsResponse getMappingsResponse) {
8587
applyAliasMappings(getMappingsResponse.getMappings(), ruleTopic, aliasMappings, partial, new ActionListener<>() {
8688
@Override
8789
public void onResponse(Collection<CreateMappingResult> createMappingResponse) {
90+
log.debug("Completed create mappings for {}", indexName);
8891
// We will return ack==false if one of the requests returned that
8992
// else return ack==true
9093
Optional<AcknowledgedResponse> notAckd = createMappingResponse.stream()
@@ -103,6 +106,7 @@ public void onResponse(Collection<CreateMappingResult> createMappingResponse) {
103106

104107
@Override
105108
public void onFailure(Exception e) {
109+
log.debug("Failed to create mappings for {}", indexName );
106110
actionListener.onFailure(e);
107111
}
108112
});

‎src/main/java/org/opensearch/securityanalytics/model/Detector.java

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -159,12 +159,12 @@ public Detector(StreamInput sin) throws IOException {
159159
sin.readList(DetectorInput::readFrom),
160160
sin.readList(DetectorTrigger::readFrom),
161161
sin.readStringList(),
162-
sin.readString(),
163-
sin.readString(),
164-
sin.readString(),
165-
sin.readString(),
166-
sin.readString(),
167-
sin.readString(),
162+
sin.readOptionalString(),
163+
sin.readOptionalString(),
164+
sin.readOptionalString(),
165+
sin.readOptionalString(),
166+
sin.readOptionalString(),
167+
sin.readOptionalString(),
168168
sin.readMap(StreamInput::readString, StreamInput::readString)
169169
);
170170
}
@@ -197,8 +197,12 @@ public void writeTo(StreamOutput out) throws IOException {
197197
it.writeTo(out);
198198
}
199199
out.writeStringCollection(monitorIds);
200-
out.writeString(ruleIndex);
201-
200+
out.writeOptionalString(ruleIndex);
201+
out.writeOptionalString(alertsIndex);
202+
out.writeOptionalString(alertsHistoryIndex);
203+
out.writeOptionalString(alertsHistoryIndexPattern);
204+
out.writeOptionalString(findingsIndex);
205+
out.writeOptionalString(findingsIndexPattern);
202206
out.writeMap(ruleIdMonitorIdMap, StreamOutput::writeString, StreamOutput::writeString);
203207
}
204208

‎src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -131,7 +131,7 @@ public OSQueryBackend(String ruleCategory, boolean collectErrors, boolean enable
131131
this.reEscapeChar = "\\";
132132
this.reExpression = "%s: /%s/";
133133
this.cidrExpression = "%s: \"%s\"";
134-
this.fieldNullExpression = "%s: null";
134+
this.fieldNullExpression = "%s: (NOT [* TO *])";
135135
this.unboundValueStrExpression = "\"%s\"";
136136
this.unboundValueNumExpression = "\"%s\"";
137137
this.unboundWildcardExpression = "%s";
@@ -331,6 +331,8 @@ public Object convertConditionFieldEqValQueryExpr(ConditionFieldEqualsValueExpre
331331

332332
@Override
333333
public Object convertConditionValStr(ConditionValueExpression condition) throws SigmaValueError {
334+
String field = getFinalValueField();
335+
ruleQueryFields.put(field, Map.of("type", "text", "analyzer", "rule_analyzer"));
334336
SigmaString value = (SigmaString) condition.getValue();
335337
boolean containsWildcard = value.containsWildcard();
336338
return String.format(Locale.getDefault(), (containsWildcard? this.unboundWildcardExpression: this.unboundValueStrExpression), this.convertValueStr((SigmaString) condition.getValue()));

‎src/main/java/org/opensearch/securityanalytics/rules/backend/QueryBackend.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -180,6 +180,10 @@ public Map<String, Object> getQueryFields() {
180180
return queryFields;
181181
}
182182

183+
public void resetQueryFields() {
184+
queryFields.clear();
185+
}
186+
183187
public abstract Object convertConditionAsInExpression(Either<ConditionAND, ConditionOR> condition);
184188

185189
public abstract Object convertConditionAnd(ConditionAND condition);

0 commit comments

Comments
 (0)