Skip to content

Commit 29ee558

Browse files
riysaxen-amznriysaxen-amzn
committed
Findings API Enhancements changes and integ tests fix (opensearch-project#1464)
* solution to fix integ tests Signed-off-by: Riya Saxena <[email protected]> * fix flaky DocumentMonitor Runner tests Signed-off-by: Riya Saxena <[email protected]> * fix findings API enhancemnts Signed-off-by: Riya Saxena <[email protected]> --------- Signed-off-by: Riya Saxena <[email protected]> (cherry picked from commit ba84d04)
1 parent d4f1675 commit 29ee558

File tree

2 files changed

+352
-4
lines changed

2 files changed

+352
-4
lines changed

‎alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportGetFindingsAction.kt

+5-4
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ import org.opensearch.commons.alerting.model.FindingWithDocs
3939
import org.opensearch.commons.utils.recreateObject
4040
import org.opensearch.core.action.ActionListener
4141
import org.opensearch.core.common.Strings
42+
import org.opensearch.core.common.io.stream.NamedWriteableRegistry
4243
import org.opensearch.core.xcontent.NamedXContentRegistry
4344
import org.opensearch.core.xcontent.XContentParser
4445
import org.opensearch.core.xcontent.XContentParserUtils
@@ -61,7 +62,8 @@ class TransportGetFindingsSearchAction @Inject constructor(
6162
clusterService: ClusterService,
6263
actionFilters: ActionFilters,
6364
val settings: Settings,
64-
val xContentRegistry: NamedXContentRegistry
65+
val xContentRegistry: NamedXContentRegistry,
66+
val namedWriteableRegistry: NamedWriteableRegistry
6567
) : HandledTransportAction<ActionRequest, GetFindingsResponse> (
6668
AlertingActions.GET_FINDINGS_ACTION_NAME,
6769
transportService,
@@ -82,7 +84,7 @@ class TransportGetFindingsSearchAction @Inject constructor(
8284
actionListener: ActionListener<GetFindingsResponse>
8385
) {
8486
val getFindingsRequest = request as? GetFindingsRequest
85-
?: recreateObject(request) { GetFindingsRequest(it) }
87+
?: recreateObject(request, namedWriteableRegistry) { GetFindingsRequest(it) }
8688
val tableProp = getFindingsRequest.table
8789

8890
val sortBuilder = SortBuilders
@@ -100,12 +102,11 @@ class TransportGetFindingsSearchAction @Inject constructor(
100102
.seqNoAndPrimaryTerm(true)
101103
.version(true)
102104

103-
val queryBuilder = QueryBuilders.boolQuery()
105+
val queryBuilder = getFindingsRequest.boolQueryBuilder ?: QueryBuilders.boolQuery()
104106

105107
if (!getFindingsRequest.findingId.isNullOrBlank()) {
106108
queryBuilder.filter(QueryBuilders.termQuery("_id", getFindingsRequest.findingId))
107109
}
108-
109110
if (getFindingsRequest.monitorId != null) {
110111
queryBuilder.filter(QueryBuilders.termQuery("monitor_id", getFindingsRequest.monitorId))
111112
} else if (getFindingsRequest.monitorIds.isNullOrEmpty() == false) {

‎alerting/src/test/kotlin/org/opensearch/alerting/DocumentMonitorRunnerIT.kt

+347
Original file line numberDiff line numberDiff line change
@@ -1674,6 +1674,353 @@ class DocumentMonitorRunnerIT : AlertingRestTestCase() {
16741674
assertEquals(1, output.objectMap("trigger_results").values.size)
16751675
}
16761676

1677+
fun `test execute monitor generates alerts and findings with NOT EQUALS query and EXISTS query`() {
1678+
val testIndex = createTestIndex()
1679+
val testTime = DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(ZonedDateTime.now().truncatedTo(MILLIS))
1680+
val testDoc = """{
1681+
"message" : "This is an error from IAD region",
1682+
"test_strict_date_time" : "$testTime",
1683+
"test_field" : "us-west-2"
1684+
}"""
1685+
1686+
val query = "NOT test_field: \"us-east-1\" AND _exists_: test_field"
1687+
val docQuery = DocLevelQuery(query = query, name = "3", fields = listOf())
1688+
val docLevelInput = DocLevelMonitorInput("description", listOf(testIndex), listOf(docQuery))
1689+
1690+
val trigger = randomDocumentLevelTrigger(condition = ALWAYS_RUN)
1691+
val monitor = createMonitor(randomDocumentLevelMonitor(inputs = listOf(docLevelInput), triggers = listOf(trigger)))
1692+
assertNotNull(monitor.id)
1693+
1694+
indexDoc(testIndex, "1", testDoc)
1695+
indexDoc(testIndex, "5", testDoc)
1696+
1697+
val response = executeMonitor(monitor.id)
1698+
1699+
val output = entityAsMap(response)
1700+
1701+
assertEquals(monitor.name, output["monitor_name"])
1702+
@Suppress("UNCHECKED_CAST")
1703+
val searchResult = (output.objectMap("input_results")["results"] as List<Map<String, Any>>).first()
1704+
@Suppress("UNCHECKED_CAST")
1705+
val matchingDocsToQuery = searchResult[docQuery.id] as List<String>
1706+
assertEquals("Incorrect search result", 2, matchingDocsToQuery.size)
1707+
assertTrue("Incorrect search result", matchingDocsToQuery.containsAll(listOf("1|$testIndex", "5|$testIndex")))
1708+
1709+
val alerts = searchAlertsWithFilter(monitor)
1710+
assertEquals("Alert saved for test monitor", 2, alerts.size)
1711+
1712+
val findings = searchFindings(monitor)
1713+
assertEquals("Findings saved for test monitor", 2, findings.size)
1714+
val findings0 = findings[0].relatedDocIds.contains("1") || findings[0].relatedDocIds.contains("5")
1715+
val findings1 = findings[1].relatedDocIds.contains("5") || findings[1].relatedDocIds.contains("1")
1716+
assertTrue("Findings saved for test monitor", findings0)
1717+
assertTrue("Findings saved for test monitor", findings1)
1718+
}
1719+
1720+
fun `test document-level monitor when index alias contain docs that do match a NOT EQUALS query and EXISTS query`() {
1721+
val aliasName = "test-alias"
1722+
createIndexAlias(
1723+
aliasName,
1724+
"""
1725+
"properties" : {
1726+
"test_strict_date_time" : { "type" : "date", "format" : "strict_date_time" },
1727+
"test_field" : { "type" : "keyword" },
1728+
"number" : { "type" : "keyword" }
1729+
}
1730+
""".trimIndent()
1731+
)
1732+
1733+
val docQuery = DocLevelQuery(query = "NOT test_field:\"us-east-1\" AND _exists_: test_field", name = "3", fields = listOf())
1734+
val docLevelInput = DocLevelMonitorInput("description", listOf("$aliasName"), listOf(docQuery))
1735+
1736+
val action = randomAction(template = randomTemplateScript("Hello {{ctx.monitor.name}}"), destinationId = createDestination().id)
1737+
val monitor = createMonitor(
1738+
randomDocumentLevelMonitor(
1739+
inputs = listOf(docLevelInput),
1740+
triggers = listOf(randomDocumentLevelTrigger(condition = ALWAYS_RUN, actions = listOf(action)))
1741+
)
1742+
)
1743+
1744+
val testTime = DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(ZonedDateTime.now().truncatedTo(MILLIS))
1745+
val testDoc = """{
1746+
"@timestamp": "$testTime",
1747+
"message" : "This is an error from IAD region",
1748+
"test_strict_date_time" : "$testTime",
1749+
"test_field" : "us-west-2"
1750+
}"""
1751+
indexDoc(aliasName, "1", testDoc)
1752+
var response = executeMonitor(monitor.id)
1753+
var output = entityAsMap(response)
1754+
var searchResult = (output.objectMap("input_results")["results"] as List<Map<String, Any>>).first()
1755+
@Suppress("UNCHECKED_CAST")
1756+
var matchingDocsToQuery = searchResult[docQuery.id] as List<String>
1757+
assertEquals("Incorrect search result", 1, matchingDocsToQuery.size)
1758+
1759+
rolloverDatastream(aliasName)
1760+
indexDoc(aliasName, "2", testDoc)
1761+
response = executeMonitor(monitor.id)
1762+
output = entityAsMap(response)
1763+
searchResult = (output.objectMap("input_results")["results"] as List<Map<String, Any>>).first()
1764+
@Suppress("UNCHECKED_CAST")
1765+
matchingDocsToQuery = searchResult[docQuery.id] as List<String>
1766+
assertEquals("Incorrect search result", 1, matchingDocsToQuery.size)
1767+
1768+
deleteIndexAlias(aliasName)
1769+
}
1770+
1771+
fun `test execute monitor with wildcard index that generates alerts and findings for NOT EQUALS and EXISTS query operator`() {
1772+
val testIndexPrefix = "test-index-${randomAlphaOfLength(10).lowercase(Locale.ROOT)}"
1773+
val testQueryName = "wildcard-test-query"
1774+
val testIndex = createTestIndex("${testIndexPrefix}1")
1775+
val testIndex2 = createTestIndex("${testIndexPrefix}2")
1776+
1777+
val testTime = DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(ZonedDateTime.now().truncatedTo(MILLIS))
1778+
val testDoc = """{
1779+
"message" : "This is an error from IAD region",
1780+
"test_strict_date_time" : "$testTime",
1781+
"test_field" : "us-west-2"
1782+
}"""
1783+
1784+
val query = "NOT test_field:\"us-west-1\" AND _exists_: test_field"
1785+
val docQuery = DocLevelQuery(query = query, name = testQueryName, fields = listOf())
1786+
val docLevelInput = DocLevelMonitorInput("description", listOf("$testIndexPrefix*"), listOf(docQuery))
1787+
1788+
val trigger = randomDocumentLevelTrigger(condition = Script("query[name=$testQueryName]"))
1789+
val monitor = createMonitor(randomDocumentLevelMonitor(inputs = listOf(docLevelInput), triggers = listOf(trigger)))
1790+
assertNotNull(monitor.id)
1791+
1792+
indexDoc(testIndex, "1", testDoc)
1793+
indexDoc(testIndex2, "5", testDoc)
1794+
1795+
val response = executeMonitor(monitor.id)
1796+
1797+
val output = entityAsMap(response)
1798+
1799+
assertEquals(monitor.name, output["monitor_name"])
1800+
@Suppress("UNCHECKED_CAST")
1801+
val searchResult = (output.objectMap("input_results")["results"] as List<Map<String, Any>>).first()
1802+
@Suppress("UNCHECKED_CAST")
1803+
val matchingDocsToQuery = searchResult[docQuery.id] as List<String>
1804+
assertEquals("Incorrect search result", 2, matchingDocsToQuery.size)
1805+
assertTrue("Incorrect search result", matchingDocsToQuery.containsAll(listOf("1|$testIndex", "5|$testIndex2")))
1806+
1807+
val alerts = searchAlertsWithFilter(monitor)
1808+
assertEquals("Alert saved for test monitor", 2, alerts.size)
1809+
1810+
val findings = searchFindings(monitor)
1811+
assertEquals("Findings saved for test monitor", 2, findings.size)
1812+
val foundFindings = findings.filter { it.relatedDocIds.contains("1") || it.relatedDocIds.contains("5") }
1813+
assertEquals("Didn't find findings for docs 1 and 5", 2, foundFindings.size)
1814+
}
1815+
1816+
fun `test execute monitor with indices having fields with same name different field mappings in multiple indices with NOT EQUALS`() {
1817+
val testIndex = createTestIndex(
1818+
"test1",
1819+
""""properties": {
1820+
"source": {
1821+
"properties": {
1822+
"device": {
1823+
"properties": {
1824+
"hwd": {
1825+
"properties": {
1826+
"id": {
1827+
"type":"text",
1828+
"analyzer":"whitespace"
1829+
}
1830+
}
1831+
}
1832+
}
1833+
}
1834+
}
1835+
},
1836+
"test_field" : {
1837+
"type":"text"
1838+
}
1839+
}
1840+
""".trimIndent()
1841+
)
1842+
1843+
val testIndex2 = createTestIndex(
1844+
"test2",
1845+
""""properties": {
1846+
"test_field" : {
1847+
"type":"keyword"
1848+
}
1849+
}
1850+
""".trimIndent()
1851+
)
1852+
1853+
val testIndex4 = createTestIndex(
1854+
"test4",
1855+
""""properties": {
1856+
"source": {
1857+
"properties": {
1858+
"device": {
1859+
"properties": {
1860+
"hwd": {
1861+
"properties": {
1862+
"id": {
1863+
"type":"text"
1864+
}
1865+
}
1866+
}
1867+
}
1868+
}
1869+
}
1870+
},
1871+
"test_field" : {
1872+
"type":"text"
1873+
}
1874+
}
1875+
""".trimIndent()
1876+
)
1877+
1878+
val testDoc1 = """{
1879+
"source" : {"device" : {"hwd" : {"id" : "123456"}} },
1880+
"nested_field": { "test1": "some text" }
1881+
}"""
1882+
val testDoc2 = """{
1883+
"nested_field": { "test1": "some text" },
1884+
"test_field": "123456"
1885+
}"""
1886+
1887+
val docQuery1 = DocLevelQuery(
1888+
query = "NOT test_field:\"12345\" AND _exists_: test_field",
1889+
name = "4",
1890+
fields = listOf()
1891+
)
1892+
val docQuery2 = DocLevelQuery(
1893+
query = "NOT source.device.hwd.id:\"12345\" AND _exists_: source.device.hwd.id",
1894+
name = "5",
1895+
fields = listOf()
1896+
)
1897+
1898+
val docLevelInput = DocLevelMonitorInput("description", listOf("test*"), listOf(docQuery1, docQuery2))
1899+
1900+
val trigger = randomDocumentLevelTrigger(condition = ALWAYS_RUN)
1901+
val monitor = createMonitor(randomDocumentLevelMonitor(inputs = listOf(docLevelInput), triggers = listOf(trigger)))
1902+
assertNotNull(monitor.id)
1903+
1904+
indexDoc(testIndex4, "1", testDoc1)
1905+
indexDoc(testIndex2, "1", testDoc2)
1906+
indexDoc(testIndex, "1", testDoc1)
1907+
indexDoc(testIndex, "2", testDoc2)
1908+
1909+
executeMonitor(monitor.id)
1910+
1911+
val alerts = searchAlertsWithFilter(monitor)
1912+
assertEquals("Alert saved for test monitor", 4, alerts.size)
1913+
1914+
val findings = searchFindings(monitor)
1915+
assertEquals("Findings saved for test monitor", 4, findings.size)
1916+
1917+
val request = """{
1918+
"size": 0,
1919+
"query": {
1920+
"match_all": {}
1921+
}
1922+
}"""
1923+
val httpResponse = adminClient().makeRequest(
1924+
"GET", "/${monitor.dataSources.queryIndex}/_search",
1925+
StringEntity(request, ContentType.APPLICATION_JSON)
1926+
)
1927+
assertEquals("Search failed", RestStatus.OK, httpResponse.restStatus())
1928+
1929+
val searchResponse = SearchResponse.fromXContent(createParser(JsonXContent.jsonXContent, httpResponse.entity.content))
1930+
searchResponse.hits.totalHits?.let { assertEquals(5L, it.value) }
1931+
}
1932+
1933+
fun `test execute monitor with indices having fields with same name but different field mappings with NOT EQUALS`() {
1934+
val testIndex = createTestIndex(
1935+
"test1",
1936+
""""properties": {
1937+
"source": {
1938+
"properties": {
1939+
"id": {
1940+
"type":"text",
1941+
"analyzer":"whitespace"
1942+
}
1943+
}
1944+
},
1945+
"test_field" : {
1946+
"type":"text",
1947+
"analyzer":"whitespace"
1948+
}
1949+
}
1950+
""".trimIndent()
1951+
)
1952+
1953+
val testIndex2 = createTestIndex(
1954+
"test2",
1955+
""""properties": {
1956+
"source": {
1957+
"properties": {
1958+
"id": {
1959+
"type":"text"
1960+
}
1961+
}
1962+
},
1963+
"test_field" : {
1964+
"type":"text"
1965+
}
1966+
}
1967+
""".trimIndent()
1968+
)
1969+
val testDoc = """{
1970+
"source" : {"id" : "12345" },
1971+
"nested_field": { "test1": "some text" },
1972+
"test_field": "12345"
1973+
}"""
1974+
1975+
val docQuery = DocLevelQuery(
1976+
query = "(NOT test_field:\"123456\" AND _exists_:test_field) AND source.id:\"12345\"",
1977+
name = "5",
1978+
fields = listOf()
1979+
)
1980+
val docLevelInput = DocLevelMonitorInput("description", listOf("test*"), listOf(docQuery))
1981+
1982+
val trigger = randomDocumentLevelTrigger(condition = ALWAYS_RUN)
1983+
val monitor = createMonitor(randomDocumentLevelMonitor(inputs = listOf(docLevelInput), triggers = listOf(trigger)))
1984+
assertNotNull(monitor.id)
1985+
1986+
indexDoc(testIndex, "1", testDoc)
1987+
indexDoc(testIndex2, "1", testDoc)
1988+
1989+
executeMonitor(monitor.id)
1990+
1991+
val alerts = searchAlertsWithFilter(monitor)
1992+
assertEquals("Alert saved for test monitor", 2, alerts.size)
1993+
1994+
val findings = searchFindings(monitor)
1995+
assertEquals("Findings saved for test monitor", 2, findings.size)
1996+
1997+
// as mappings of source.id & test_field are different so, both of them expands
1998+
val expectedQueries = listOf(
1999+
"(NOT test_field_test2_${monitor.id}:\"123456\" AND _exists_:test_field_test2_${monitor.id}) " +
2000+
"AND source.id_test2_${monitor.id}:\"12345\"",
2001+
"(NOT test_field_test1_${monitor.id}:\"123456\" AND _exists_:test_field_test1_${monitor.id}) " +
2002+
"AND source.id_test1_${monitor.id}:\"12345\""
2003+
)
2004+
2005+
val request = """{
2006+
"size": 10,
2007+
"query": {
2008+
"match_all": {}
2009+
}
2010+
}"""
2011+
var httpResponse = adminClient().makeRequest(
2012+
"GET", "/${monitor.dataSources.queryIndex}/_search",
2013+
StringEntity(request, ContentType.APPLICATION_JSON)
2014+
)
2015+
assertEquals("Search failed", RestStatus.OK, httpResponse.restStatus())
2016+
var searchResponse = SearchResponse.fromXContent(createParser(JsonXContent.jsonXContent, httpResponse.entity.content))
2017+
searchResponse.hits.forEach { hit ->
2018+
val query = ((hit.sourceAsMap["query"] as Map<String, Any>)["query_string"] as Map<String, Any>)["query"]
2019+
assertTrue(expectedQueries.contains(query))
2020+
}
2021+
}
2022+
2023+
>>>>>>> ba84d04d ( Findings API Enhancements changes and integ tests fix (#1464))
16772024
@Suppress("UNCHECKED_CAST")
16782025
/** helper that returns a field in a json map whose values are all json objects */
16792026
private fun Map<String, Any>.objectMap(key: String): Map<String, Map<String, Any>> {

0 commit comments

Comments
 (0)