chore: move audit log processing to separate timer function

Author: JohnDuprey

CIPPTimers.json

@@ -35,12 +35,22 @@
     "PreferredProcessor": "auditlog",
     "IsSystem": true
   },
+  {
+    "Id": "01cd512a-15c4-44a9-b8cb-1e5d879cfd2d",
+    "Command": "Start-AuditLogProcessingOrchestrator",
+    "Description": "Orchestrator to process audit logs",
+    "Cron": "0 */15 * * * *",
+    "Priority": 3,
+    "RunOnProcessor": true,
+    "PreferredProcessor": "auditlog",
+    "IsSystem": true
+  },
   {
     "Id": "03475c86-4314-4d7b-90f2-5a0639e3899b",
     "Command": "Start-AuditLogSearchCreation",
     "Description": "Timer to create audit log searches",
     "Cron": "0 */30 * * * *",
-    "Priority": 3,
+    "Priority": 4,
     "RunOnProcessor": true,
     "PreferredProcessor": "auditlog",
     "IsSystem": true

Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1

@@ -29,40 +29,6 @@ function Start-AuditLogOrchestrator {
                     SkipLog          = $true
                 }
                 Start-NewOrchestration -FunctionName 'CIPPOrchestrator' -InputObject ($InputObject | ConvertTo-Json -Depth 5 -Compress)
-                Write-Information 'Starting audit log processing in batches of 1000, per tenant'
-                $WebhookCacheTable = Get-CippTable -TableName 'CacheWebhooks'
-                $WebhookCache = Get-CIPPAzDataTableEntity @WebhookCacheTable
-                $TenantGroups = $WebhookCache | Group-Object -Property PartitionKey
-
-                if ($TenantGroups) {
-                    Write-Information "Processing webhook cache for $($TenantGroups.Count) tenants"
-                    #Write-Warning "AuditLogJobs are: $($TenantGroups.Count) tenants. Tenants: $($TenantGroups.name | ConvertTo-Json -Compress) "
-                    #Write-Warning "Here are the groups: $($TenantGroups | ConvertTo-Json -Compress)"
-                    $ProcessQueue = New-CippQueueEntry -Name 'Audit Logs Process' -Reference 'AuditLogsProcess' -TotalTasks ($TenantGroups | Measure-Object -Property Count -Sum).Sum
-                    $ProcessBatch = foreach ($TenantGroup in $TenantGroups) {
-                        $TenantFilter = $TenantGroup.Name
-                        $RowIds = @($TenantGroup.Group.RowKey)
-                        for ($i = 0; $i -lt $RowIds.Count; $i += 1000) {
-                            Write-Host "Processing $TenantFilter with $($RowIds.Count) row IDs. We're processing id $($RowIds[$i]) to $($RowIds[[Math]::Min($i + 999, $RowIds.Count - 1)])"
-                            $BatchRowIds = $RowIds[$i..([Math]::Min($i + 999, $RowIds.Count - 1))]
-                            [PSCustomObject]@{
-                                TenantFilter = $TenantFilter
-                                RowIds       = $BatchRowIds
-                                QueueId      = $ProcessQueue.RowKey
-                                FunctionName = 'AuditLogTenantProcess'
-                            }
-                        }
-                    }
-                    if ($ProcessBatch) {
-                        $ProcessInputObject = [PSCustomObject]@{
-                            OrchestratorName = 'AuditLogTenantProcess'
-                            Batch            = @($ProcessBatch)
-                            SkipLog          = $true
-                        }
-                        Start-NewOrchestration -FunctionName 'CIPPOrchestrator' -InputObject ($ProcessInputObject | ConvertTo-Json -Depth 5 -Compress)
-                        Write-Information "Started audit log processing orchestration with $($ProcessBatch.Count) batches"
-                    }
-                }
             }
         }
     } catch {

Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogProcessingOrchestrator.ps1

@@ -0,0 +1,42 @@
+function Start-AuditLogProcessingOrchestrator {
+    <#
+    .SYNOPSIS
+    Start the Audit Log Processing Orchestrator
+    #>
+    [CmdletBinding(SupportsShouldProcess = $true)]
+    param()
+    Write-Information 'Starting audit log processing in batches of 1000, per tenant'
+    $WebhookCacheTable = Get-CippTable -TableName 'CacheWebhooks'
+    $WebhookCache = Get-CIPPAzDataTableEntity @WebhookCacheTable
+    $TenantGroups = $WebhookCache | Group-Object -Property PartitionKey
+
+    if ($TenantGroups) {
+        Write-Information "Processing webhook cache for $($TenantGroups.Count) tenants"
+        #Write-Warning "AuditLogJobs are: $($TenantGroups.Count) tenants. Tenants: $($TenantGroups.name | ConvertTo-Json -Compress) "
+        #Write-Warning "Here are the groups: $($TenantGroups | ConvertTo-Json -Compress)"
+        $ProcessQueue = New-CippQueueEntry -Name 'Audit Logs Process' -Reference 'AuditLogsProcess' -TotalTasks ($TenantGroups | Measure-Object -Property Count -Sum).Sum
+        $ProcessBatch = foreach ($TenantGroup in $TenantGroups) {
+            $TenantFilter = $TenantGroup.Name
+            $RowIds = @($TenantGroup.Group.RowKey)
+            for ($i = 0; $i -lt $RowIds.Count; $i += 1000) {
+                Write-Host "Processing $TenantFilter with $($RowIds.Count) row IDs. We're processing id $($RowIds[$i]) to $($RowIds[[Math]::Min($i + 999, $RowIds.Count - 1)])"
+                $BatchRowIds = $RowIds[$i..([Math]::Min($i + 999, $RowIds.Count - 1))]
+                [PSCustomObject]@{
+                    TenantFilter = $TenantFilter
+                    RowIds       = $BatchRowIds
+                    QueueId      = $ProcessQueue.RowKey
+                    FunctionName = 'AuditLogTenantProcess'
+                }
+            }
+        }
+        if ($ProcessBatch) {
+            $ProcessInputObject = [PSCustomObject]@{
+                OrchestratorName = 'AuditLogTenantProcess'
+                Batch            = @($ProcessBatch)
+                SkipLog          = $true
+            }
+            Start-NewOrchestration -FunctionName 'CIPPOrchestrator' -InputObject ($ProcessInputObject | ConvertTo-Json -Depth 5 -Compress)
+            Write-Information "Started audit log processing orchestration with $($ProcessBatch.Count) batches"
+        }
+    }
+}