Skip to content

Commit a82c4f1

Browse files
eirsepsbcd90goyameghopensearch-trigger-bot[bot]jowg-amazon
authored
[Backport 2.9] Backport #1302, #1355, #1432, #1435, #1445, #1403, #1430, #1441 to 2.9 (#1469)
* optimize doc-level monitor execution workflow for datastreams (#1302) * optimize doc-level monitor execution for datastreams Signed-off-by: Subhobrata Dey <[email protected]> * add more tests to address comments Signed-off-by: Subhobrata Dey <[email protected]> * add integTest for multiple datastreams inside a single index pattern * add integTest for multiple datastreams inside a single index pattern Signed-off-by: Subhobrata Dey <[email protected]> --------- Signed-off-by: Subhobrata Dey <[email protected]> * Bulk index findings and sequentially invoke auto-correlations (#1355) * Bulk index findings and sequentially invoke auto-correlations Signed-off-by: Megha Goyal <[email protected]> * Bulk index findings in batches of 10000 and make it configurable Signed-off-by: Megha Goyal <[email protected]> * Addressing review comments Signed-off-by: Megha Goyal <[email protected]> * Add integ tests to test bulk index findings Signed-off-by: Megha Goyal <[email protected]> * Fix ktlint formatting Signed-off-by: Megha Goyal <[email protected]> --------- Signed-off-by: Megha Goyal <[email protected]> * fix for MapperException[the [enabled] parameter can't be updated for the object mapping [metadata.source_to_query_index_mapping] (#1432) (#1434) Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com> * Add jvm aware setting and max num docs settings for batching docs for percolate queries (#1435) * add jvm aware and max docs settings for batching docs for percolate queries Signed-off-by: Surya Sashank Nistala <[email protected]> * fix stats logging Signed-off-by: Surya Sashank Nistala <[email protected]> * add queryfieldnames field in findings mapping Signed-off-by: Surya Sashank Nistala <[email protected]> --------- Signed-off-by: Surya Sashank Nistala <[email protected]> * clean up doc level queries on dry run (#1430) Signed-off-by: Joanne Wang <[email protected]> * optimize to fetch only fields relevant to doc level queries in doc level monitor instead of entire _source for each doc (#1441) * optimize to fetch only fields relevant to doc level queries in doc level monitor Signed-off-by: Surya Sashank Nistala <[email protected]> * fix test for settings check Signed-off-by: Surya Sashank Nistala <[email protected]> * fix ktlint Signed-off-by: Surya Sashank Nistala <[email protected]> --------- Signed-off-by: Surya Sashank Nistala <[email protected]> * optimize sequence number calculation and reduce search requests in doc level monitor execution (#1445) * optimize sequence number calculation and reduce search requests by n where n is number of shards being queried in the executino Signed-off-by: Surya Sashank Nistala <[email protected]> * fix tests Signed-off-by: Surya Sashank Nistala <[email protected]> * optimize check indices and execute to query only write index of aliases and datastreams during monitor creation Signed-off-by: Surya Sashank Nistala <[email protected]> * fix test Signed-off-by: Surya Sashank Nistala <[email protected]> * add javadoc Signed-off-by: Surya Sashank Nistala <[email protected]> * add tests to verify seq_no calculation Signed-off-by: Surya Sashank Nistala <[email protected]> --------- Signed-off-by: Surya Sashank Nistala <[email protected]> * add distributed locking to jobs in alerting (#1403) Signed-off-by: Subhobrata Dey <[email protected]> * 2.9.1 version bump Signed-off-by: Surya Sashank Nistala <[email protected]> * fix compilation issues Signed-off-by: Surya Sashank Nistala <[email protected]> * dummy commit Signed-off-by: Surya Sashank Nistala <[email protected]> * fix findings index schema version tsts Signed-off-by: Surya Sashank Nistala <[email protected]> --------- Signed-off-by: Subhobrata Dey <[email protected]> Signed-off-by: Megha Goyal <[email protected]> Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com> Signed-off-by: Surya Sashank Nistala <[email protected]> Signed-off-by: Joanne Wang <[email protected]> Co-authored-by: Subhobrata Dey <[email protected]> Co-authored-by: Megha Goyal <[email protected]> Co-authored-by: opensearch-trigger-bot[bot] <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Co-authored-by: Joanne Wang <[email protected]>
1 parent ccfec17 commit a82c4f1

25 files changed

+2460
-281
lines changed

‎alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import org.opensearch.alerting.core.JobSweeper
2121
import org.opensearch.alerting.core.ScheduledJobIndices
2222
import org.opensearch.alerting.core.action.node.ScheduledJobsStatsAction
2323
import org.opensearch.alerting.core.action.node.ScheduledJobsStatsTransportAction
24+
import org.opensearch.alerting.core.lock.LockService
2425
import org.opensearch.alerting.core.resthandler.RestScheduledJobStatsHandler
2526
import org.opensearch.alerting.core.schedule.JobScheduler
2627
import org.opensearch.alerting.core.settings.LegacyOpenDistroScheduledJobSettings
@@ -47,6 +48,7 @@ import org.opensearch.alerting.resthandler.RestSearchMonitorAction
4748
import org.opensearch.alerting.script.TriggerScript
4849
import org.opensearch.alerting.service.DeleteMonitorService
4950
import org.opensearch.alerting.settings.AlertingSettings
51+
import org.opensearch.alerting.settings.AlertingSettings.Companion.DOC_LEVEL_MONITOR_SHARD_FETCH_SIZE
5052
import org.opensearch.alerting.settings.DestinationSettings
5153
import org.opensearch.alerting.settings.LegacyOpenDistroAlertingSettings
5254
import org.opensearch.alerting.settings.LegacyOpenDistroDestinationSettings
@@ -99,6 +101,7 @@ import org.opensearch.core.xcontent.XContentParser
99101
import org.opensearch.env.Environment
100102
import org.opensearch.env.NodeEnvironment
101103
import org.opensearch.index.IndexModule
104+
import org.opensearch.monitor.jvm.JvmStats
102105
import org.opensearch.painless.spi.PainlessExtension
103106
import org.opensearch.painless.spi.Whitelist
104107
import org.opensearch.painless.spi.WhitelistLoader
@@ -254,6 +257,7 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R
254257
): Collection<Any> {
255258
// Need to figure out how to use the OpenSearch DI classes rather than handwiring things here.
256259
val settings = environment.settings()
260+
val lockService = LockService(client, clusterService)
257261
alertIndices = AlertIndices(settings, client, threadPool, clusterService)
258262
runner = MonitorRunnerService
259263
.registerClusterService(clusterService)
@@ -268,7 +272,9 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R
268272
.registerTriggerService(TriggerService(scriptService))
269273
.registerAlertService(AlertService(client, xContentRegistry, alertIndices))
270274
.registerDocLevelMonitorQueries(DocLevelMonitorQueries(client, clusterService))
275+
.registerJvmStats(JvmStats.jvmStats())
271276
.registerWorkflowService(WorkflowService(client, xContentRegistry))
277+
.registerLockService(lockService)
272278
.registerConsumers()
273279
.registerDestinationSettings()
274280
scheduledJobIndices = ScheduledJobIndices(client.admin(), clusterService)
@@ -293,9 +299,9 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R
293299
settings
294300
)
295301

296-
DeleteMonitorService.initialize(client)
302+
DeleteMonitorService.initialize(client, lockService)
297303

298-
return listOf(sweeper, scheduler, runner, scheduledJobIndices, docLevelMonitorQueries, destinationMigrationCoordinator)
304+
return listOf(sweeper, scheduler, runner, scheduledJobIndices, docLevelMonitorQueries, destinationMigrationCoordinator, lockService)
299305
}
300306

301307
override fun getSettings(): List<Setting<*>> {
@@ -325,6 +331,9 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R
325331
AlertingSettings.ALERT_HISTORY_MAX_DOCS,
326332
AlertingSettings.ALERT_HISTORY_RETENTION_PERIOD,
327333
AlertingSettings.ALERTING_MAX_MONITORS,
334+
AlertingSettings.PERCOLATE_QUERY_DOCS_SIZE_MEMORY_PERCENTAGE_LIMIT,
335+
DOC_LEVEL_MONITOR_SHARD_FETCH_SIZE,
336+
AlertingSettings.PERCOLATE_QUERY_MAX_NUM_DOCS_IN_MEMORY,
328337
AlertingSettings.REQUEST_TIMEOUT,
329338
AlertingSettings.MAX_ACTION_THROTTLE_VALUE,
330339
AlertingSettings.FILTER_BY_BACKEND_ROLES,
@@ -345,6 +354,7 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R
345354
LegacyOpenDistroAlertingSettings.REQUEST_TIMEOUT,
346355
LegacyOpenDistroAlertingSettings.MAX_ACTION_THROTTLE_VALUE,
347356
LegacyOpenDistroAlertingSettings.FILTER_BY_BACKEND_ROLES,
357+
AlertingSettings.DOC_LEVEL_MONITOR_FETCH_ONLY_QUERY_FIELDS_ENABLED,
348358
DestinationSettings.EMAIL_USERNAME,
349359
DestinationSettings.EMAIL_PASSWORD,
350360
DestinationSettings.ALLOW_LIST,
@@ -357,7 +367,8 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R
357367
AlertingSettings.FINDING_HISTORY_MAX_DOCS,
358368
AlertingSettings.FINDING_HISTORY_INDEX_MAX_AGE,
359369
AlertingSettings.FINDING_HISTORY_ROLLOVER_PERIOD,
360-
AlertingSettings.FINDING_HISTORY_RETENTION_PERIOD
370+
AlertingSettings.FINDING_HISTORY_RETENTION_PERIOD,
371+
AlertingSettings.FINDINGS_INDEXING_BATCH_SIZE
361372
)
362373
}
363374

0 commit comments

Comments
 (0)