Misc fixes for 1.11 release (#274)

* delete dead code, fix flaky tests, revert to runBlocking for monitor runner

Author: skkosuri-amzn

alerting/src/main/kotlin/com/amazon/opendistroforelasticsearch/alerting/MonitorRunner.kt

@@ -60,6 +60,7 @@ import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.Job
 import kotlinx.coroutines.SupervisorJob
 import kotlinx.coroutines.launch
+import kotlinx.coroutines.runBlocking
 import kotlinx.coroutines.withContext
 import org.apache.logging.log4j.LogManager
 import org.elasticsearch.ExceptionsHelper
@@ -223,7 +224,7 @@ class MonitorRunner(
             logger.error("Error loading alerts for monitor: $id", e)
             return monitorResult.copy(error = e)
         }
-        withContext(InjectorContextElement(monitor.id, settings, threadPool.threadContext, roles)) {
+        runBlocking(InjectorContextElement(monitor.id, settings, threadPool.threadContext, roles)) {
             monitorResult = monitorResult.copy(inputResults = collectInputResults(monitor, periodStart, periodEnd))
         }
         val updatedAlerts = mutableListOf<Alert>()

alerting/src/main/kotlin/com/amazon/opendistroforelasticsearch/alerting/transport/TransportGetAlertsAction.kt

@@ -118,42 +118,7 @@ class TransportGetAlertsAction @Inject constructor(
                 resolve(getAlertsRequest, searchSourceBuilder, actionListener)
             }
         }
-/*
-if (getDestinationsRequest.authHeader.isNullOrEmpty()) {
-            // auth header is null when: 1/ security is disabled. 2/when user is super-admin.
-            search(searchSourceBuilder, actionListener)
-        } else if (!filterByEnabled) {
-            // security is enabled and filterby is disabled.
-            search(searchSourceBuilder, actionListener)
-        } else {
-            // security is enabled and filterby is enabled.
-            val authRequest = AuthUserRequestBuilder(
-                    getDestinationsRequest.authHeader
-            ).build()
-            restClient.performRequestAsync(authRequest, object : ResponseListener {
-                override fun onSuccess(response: Response) {
-                    try {
-                        val user = User(response)
-                        addFilter(user, searchSourceBuilder)
-                        search(searchSourceBuilder, actionListener)
-                    } catch (ex: IOException) {
-                        actionListener.onFailure(AlertingException.wrap(ex))
-                    }
-                }
 
-                override fun onFailure(ex: Exception) {
-                    when (ex.message?.contains("Connection refused")) {
-                        // Connection is refused when security plugin is not present. This case can happen only with integration tests.
-                        true -> {
-                            addFilter(User(), searchSourceBuilder)
-                            search(searchSourceBuilder, actionListener)
-                        }
-                        false -> actionListener.onFailure(AlertingException.wrap(ex))
-                    }
-                }
-            })
-        }
- */
         fun resolve(
             getAlertsRequest: GetAlertsRequest,
             searchSourceBuilder: SearchSourceBuilder,

alerting/src/test/kotlin/com/amazon/opendistroforelasticsearch/alerting/AlertingRestTestCase.kt

@@ -40,6 +40,7 @@ import org.apache.http.message.BasicHeader
 import org.elasticsearch.action.search.SearchResponse
 import org.elasticsearch.client.Response
 import org.elasticsearch.client.RestClient
+import org.elasticsearch.client.WarningFailureException
 import org.elasticsearch.common.settings.Settings
 import org.elasticsearch.common.unit.TimeValue
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler
@@ -460,12 +461,16 @@ abstract class AlertingRestTestCase : ODFERestTestCase() {
         return index
     }
 
-    protected fun createTestHiddenIndex(index: String = randomAlphaOfLength(10).toLowerCase(Locale.ROOT)): String {
-        createIndex(index, Settings.builder().put("hidden", true).build(), """
+    protected fun createTestConfigIndex(index: String = "." + randomAlphaOfLength(10).toLowerCase(Locale.ROOT)): String {
+        try {
+            createIndex(index, Settings.builder().build(), """
           "properties" : {
              "test_strict_date_time" : { "type" : "date", "format" : "strict_date_time" }
           }
         """.trimIndent())
+        } catch (ex: WarningFailureException) {
+            // ignore
+        }
         return index
     }
 

alerting/src/test/kotlin/com/amazon/opendistroforelasticsearch/alerting/MonitorRunnerIT.kt

@@ -272,7 +272,17 @@ class MonitorRunnerIT : AlertingRestTestCase() {
     }
 
     fun `test execute monitor search with period`() {
-        val query = QueryBuilders.rangeQuery("monitor.last_update_time").gte("{{period_start}}").lte("{{period_end}}")
+        // We cant query .opendistro-alerting-config as its system index. Create a test index starting with "."
+        val testIndex = createTestConfigIndex()
+        val fiveDaysAgo = ZonedDateTime.now().minus(5, DAYS).truncatedTo(MILLIS)
+        val testTime = DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(fiveDaysAgo)
+        val testDoc = """{ "test_strict_date_time" : "$testTime" }"""
+        indexDoc(testIndex, "1", testDoc)
+
+        val query = QueryBuilders.rangeQuery("test_strict_date_time")
+                .gt("{{period_end}}||-10d")
+                .lte("{{period_end}}")
+                .format("epoch_millis")
         val input = SearchInput(indices = listOf(".*"), query = SearchSourceBuilder().query(query))
         val triggerScript = """
             // make sure there is at least one monitor
@@ -588,6 +598,7 @@ class MonitorRunnerIT : AlertingRestTestCase() {
         verifyAlert(activeAlert1.single(), monitor, ACTIVE)
         val actionResults1 = verifyActionExecutionResultInAlert(activeAlert1[0], mutableMapOf(Pair(actionThrottleEnabled.id, 0)))
 
+        Thread.sleep(200)
         updateMonitor(monitor.copy(triggers = listOf(trigger.copy(condition = NEVER_RUN)), id = monitor.id))
         executeMonitor(monitor.id)
         val completedAlert = searchAlerts(monitor, AlertIndices.ALL_INDEX_PATTERN).single()