add logging for remote monitor execution flows (#1663)

Signed-off-by: Surya Sashank Nistala <[email protected]>

Author: eirsep

alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunnerService.kt

@@ -480,6 +480,7 @@ object MonitorRunnerService : JobRunner, CoroutineScope, AbstractLifecycleCompon
         } else {
             if (monitorCtx.remoteMonitors.containsKey(monitor.monitorType)) {
                 if (monitor.monitorType.endsWith(Monitor.MonitorType.DOC_LEVEL_MONITOR.value)) {
+                    logger.info("Executing remote document monitor of type ${monitor.monitorType} id ${monitor.id}")
                     return RemoteDocumentLevelMonitorRunner().runMonitor(
                         monitor,
                         monitorCtx,
@@ -490,6 +491,7 @@ object MonitorRunnerService : JobRunner, CoroutineScope, AbstractLifecycleCompon
                         transportService = transportService
                     )
                 } else {
+                    logger.info("Executing remote monitor of type ${monitor.monitorType} id ${monitor.id}")
                     return monitorCtx.remoteMonitors[monitor.monitorType]!!.monitorRunner.runMonitor(
                         monitor,
                         periodStart,

alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportDocLevelMonitorFanOutAction.kt

@@ -340,7 +340,11 @@ class TransportDocLevelMonitorFanOutAction
                 )
             )
         } catch (e: Exception) {
-            log.error("${request.monitor.id} Failed to run fan_out on node ${clusterService.localNode().id} due to error $e")
+            log.error(
+                "${request.monitor.id} Failed to run fan_out on node ${clusterService.localNode().id}." +
+                    " for Monitor Type ${request.monitor.monitorType} ExecutionId ${request.executionId}",
+                e
+            )
             listener.onFailure(AlertingException.wrap(e))
         }
     }

spi/src/main/kotlin/org/opensearch/alerting/spi/RemoteMonitorRunner.kt

@@ -49,7 +49,7 @@ open class RemoteMonitorRunner {
     }
 
     open fun getFanOutAction(): String {
-        return DocLevelMonitorFanOutAction.NAME
+        throw UnsupportedOperationException("Fan out action needs to be implemented by remote monitor.");
     }
 
     open suspend fun doFanOut(
@@ -185,4 +185,4 @@ open class RemoteMonitorRunner {
             }
         }
     }
-}
+}