From 91eaf5c1e28e430ac43545e3154f834ecf17d6ce Mon Sep 17 00:00:00 2001 From: Yusuke Kuoka Date: Mon, 3 May 2021 14:13:36 +0000 Subject: [PATCH] Experimental support for ScheduledOverrides This adds the initial version of ScheduledOverrides to HorizontalRunnerAutoscaler. `MinReplicas` overriding should just work. When there are two or more ScheduledOverrides, the earliest one that matched is activated. Each ScheduledOverride can be recurring or one-time. If you have two or more ScheduledOverrides, only one of them should be one-time. And the one-time override should be the earliest item in the list to make sense. Tests will be added in another commit. Logging improvements and additional observability in HRA.Status will also be added in yet another commits. Ref #484 --- .../horizontalrunnerautoscaler_types.go | 12 +-- api/v1alpha1/zz_generated.deepcopy.go | 14 +-- ...rwind.dev_horizontalrunnerautoscalers.yaml | 101 +++++++++--------- ...rwind.dev_horizontalrunnerautoscalers.yaml | 101 +++++++++--------- .../horizontalrunnerautoscaler_controller.go | 40 +++++++ ...ler_scheduled_overrides.go => schedule.go} | 0 ...led_overrides_test.go => schedule_test.go} | 0 7 files changed, 153 insertions(+), 115 deletions(-) rename controllers/{horizontalrunnerautoscaler_scheduled_overrides.go => schedule.go} (100%) rename controllers/{horizontalrunnerautoscaler_scheduled_overrides_test.go => schedule_test.go} (100%) diff --git a/api/v1alpha1/horizontalrunnerautoscaler_types.go b/api/v1alpha1/horizontalrunnerautoscaler_types.go index 701ded0fbd..7736c7f908 100644 --- a/api/v1alpha1/horizontalrunnerautoscaler_types.go +++ b/api/v1alpha1/horizontalrunnerautoscaler_types.go @@ -54,6 +54,12 @@ type HorizontalRunnerAutoscalerSpec struct { ScaleUpTriggers []ScaleUpTrigger `json:"scaleUpTriggers,omitempty"` CapacityReservations []CapacityReservation `json:"capacityReservations,omitempty" patchStrategy:"merge" patchMergeKey:"name"` + + // ScheduledOverrides is the list of ScheduledOverride. + // It can be used to override a few fields of HorizontalRunnerAutoscalerSpec on schedule. + // The earlier a scheduled override is, the higher it is prioritized. + // +optional + ScheduledOverrides []ScheduledOverride `json:"scheduledOverrides,omitempty"` } type ScaleUpTrigger struct { @@ -142,12 +148,6 @@ type MetricSpec struct { // You can only specify either ScaleDownFactor or ScaleDownAdjustment. // +optional ScaleDownAdjustment int `json:"scaleDownAdjustment,omitempty"` - - // ScheduledOverrides is the list of ScheduledOverride. - // It can be used to override a few fields of HorizontalRunnerAutoscalerSpec on schedule. - // The earlier a scheduled override is, the higher it is prioritized. - // +optional - ScheduledOverrides []ScheduledOverride `json:"scheduledOverrides,omitempty"` } // ScheduledOverride can be used to override a few fields of HorizontalRunnerAutoscalerSpec on schedule. diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index e47cc1c4bf..336ce61ffc 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -212,6 +212,13 @@ func (in *HorizontalRunnerAutoscalerSpec) DeepCopyInto(out *HorizontalRunnerAuto (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.ScheduledOverrides != nil { + in, out := &in.ScheduledOverrides, &out.ScheduledOverrides + *out = make([]ScheduledOverride, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HorizontalRunnerAutoscalerSpec. @@ -263,13 +270,6 @@ func (in *MetricSpec) DeepCopyInto(out *MetricSpec) { *out = make([]string, len(*in)) copy(*out, *in) } - if in.ScheduledOverrides != nil { - in, out := &in.ScheduledOverrides, &out.ScheduledOverrides - *out = make([]ScheduledOverride, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MetricSpec. diff --git a/charts/actions-runner-controller/crds/actions.summerwind.dev_horizontalrunnerautoscalers.yaml b/charts/actions-runner-controller/crds/actions.summerwind.dev_horizontalrunnerautoscalers.yaml index fe42777dc0..3056a83675 100644 --- a/charts/actions-runner-controller/crds/actions.summerwind.dev_horizontalrunnerautoscalers.yaml +++ b/charts/actions-runner-controller/crds/actions.summerwind.dev_horizontalrunnerautoscalers.yaml @@ -105,57 +105,6 @@ spec: description: ScaleUpThreshold is the percentage of busy runners greater than which will trigger the hpa to scale runners up. type: string - scheduledOverrides: - description: ScheduledOverrides is the list of ScheduledOverride. - It can be used to override a few fields of HorizontalRunnerAutoscalerSpec - on schedule. The earlier a scheduled override is, the higher - it is prioritized. - items: - description: ScheduledOverride can be used to override a few - fields of HorizontalRunnerAutoscalerSpec on schedule. A schedule - can optionally be recurring, so that the correspoding override - happens every day, week, month, or year. - properties: - endTime: - description: EndTime is the time at which the first override - ends. - format: date-time - type: string - minReplicas: - description: MinReplicas is the number of runners while - overriding. If omitted, it doesn't override minReplicas. - minimum: 0 - nullable: true - type: integer - recurrenceRule: - properties: - frequency: - description: Frequency is the name of a predefined interval - of each recurrence. The valid values are "Daily", - "Weekly", "Monthly", and "Yearly". If empty, the corresponding - override happens only once. - enum: - - Daily - - Weekly - - Monthly - - Yearly - type: string - untilTime: - description: UntilTime is the time of the final recurrence. - If empty, the schedule recurs forever. - format: date-time - type: string - type: object - startTime: - description: StartTime is the time at which the first override - starts. - format: date-time - type: string - required: - - endTime - - startTime - type: object - type: array type: description: Type is the type of metric to be used for autoscaling. The only supported Type is TotalNumberOfQueuedAndInProgressWorkflowRuns @@ -236,6 +185,56 @@ spec: type: object type: object type: array + scheduledOverrides: + description: ScheduledOverrides is the list of ScheduledOverride. It + can be used to override a few fields of HorizontalRunnerAutoscalerSpec + on schedule. The earlier a scheduled override is, the higher it is + prioritized. + items: + description: ScheduledOverride can be used to override a few fields + of HorizontalRunnerAutoscalerSpec on schedule. A schedule can optionally + be recurring, so that the correspoding override happens every day, + week, month, or year. + properties: + endTime: + description: EndTime is the time at which the first override ends. + format: date-time + type: string + minReplicas: + description: MinReplicas is the number of runners while overriding. + If omitted, it doesn't override minReplicas. + minimum: 0 + nullable: true + type: integer + recurrenceRule: + properties: + frequency: + description: Frequency is the name of a predefined interval + of each recurrence. The valid values are "Daily", "Weekly", + "Monthly", and "Yearly". If empty, the corresponding override + happens only once. + enum: + - Daily + - Weekly + - Monthly + - Yearly + type: string + untilTime: + description: UntilTime is the time of the final recurrence. + If empty, the schedule recurs forever. + format: date-time + type: string + type: object + startTime: + description: StartTime is the time at which the first override + starts. + format: date-time + type: string + required: + - endTime + - startTime + type: object + type: array type: object status: properties: diff --git a/config/crd/bases/actions.summerwind.dev_horizontalrunnerautoscalers.yaml b/config/crd/bases/actions.summerwind.dev_horizontalrunnerautoscalers.yaml index fe42777dc0..3056a83675 100644 --- a/config/crd/bases/actions.summerwind.dev_horizontalrunnerautoscalers.yaml +++ b/config/crd/bases/actions.summerwind.dev_horizontalrunnerautoscalers.yaml @@ -105,57 +105,6 @@ spec: description: ScaleUpThreshold is the percentage of busy runners greater than which will trigger the hpa to scale runners up. type: string - scheduledOverrides: - description: ScheduledOverrides is the list of ScheduledOverride. - It can be used to override a few fields of HorizontalRunnerAutoscalerSpec - on schedule. The earlier a scheduled override is, the higher - it is prioritized. - items: - description: ScheduledOverride can be used to override a few - fields of HorizontalRunnerAutoscalerSpec on schedule. A schedule - can optionally be recurring, so that the correspoding override - happens every day, week, month, or year. - properties: - endTime: - description: EndTime is the time at which the first override - ends. - format: date-time - type: string - minReplicas: - description: MinReplicas is the number of runners while - overriding. If omitted, it doesn't override minReplicas. - minimum: 0 - nullable: true - type: integer - recurrenceRule: - properties: - frequency: - description: Frequency is the name of a predefined interval - of each recurrence. The valid values are "Daily", - "Weekly", "Monthly", and "Yearly". If empty, the corresponding - override happens only once. - enum: - - Daily - - Weekly - - Monthly - - Yearly - type: string - untilTime: - description: UntilTime is the time of the final recurrence. - If empty, the schedule recurs forever. - format: date-time - type: string - type: object - startTime: - description: StartTime is the time at which the first override - starts. - format: date-time - type: string - required: - - endTime - - startTime - type: object - type: array type: description: Type is the type of metric to be used for autoscaling. The only supported Type is TotalNumberOfQueuedAndInProgressWorkflowRuns @@ -236,6 +185,56 @@ spec: type: object type: object type: array + scheduledOverrides: + description: ScheduledOverrides is the list of ScheduledOverride. It + can be used to override a few fields of HorizontalRunnerAutoscalerSpec + on schedule. The earlier a scheduled override is, the higher it is + prioritized. + items: + description: ScheduledOverride can be used to override a few fields + of HorizontalRunnerAutoscalerSpec on schedule. A schedule can optionally + be recurring, so that the correspoding override happens every day, + week, month, or year. + properties: + endTime: + description: EndTime is the time at which the first override ends. + format: date-time + type: string + minReplicas: + description: MinReplicas is the number of runners while overriding. + If omitted, it doesn't override minReplicas. + minimum: 0 + nullable: true + type: integer + recurrenceRule: + properties: + frequency: + description: Frequency is the name of a predefined interval + of each recurrence. The valid values are "Daily", "Weekly", + "Monthly", and "Yearly". If empty, the corresponding override + happens only once. + enum: + - Daily + - Weekly + - Monthly + - Yearly + type: string + untilTime: + description: UntilTime is the time of the final recurrence. + If empty, the schedule recurs forever. + format: date-time + type: string + type: object + startTime: + description: StartTime is the time at which the first override + starts. + format: date-time + type: string + required: + - endTime + - startTime + type: object + type: array type: object status: properties: diff --git a/controllers/horizontalrunnerautoscaler_controller.go b/controllers/horizontalrunnerautoscaler_controller.go index de6392b950..00dc248307 100644 --- a/controllers/horizontalrunnerautoscaler_controller.go +++ b/controllers/horizontalrunnerautoscaler_controller.go @@ -185,12 +185,52 @@ func (r *HorizontalRunnerAutoscalerReconciler) SetupWithManager(mgr ctrl.Manager Complete(r) } +func (r *HorizontalRunnerAutoscalerReconciler) matchScheduledOverrides(log logr.Logger, now time.Time, hra v1alpha1.HorizontalRunnerAutoscaler) (*int, *Period, *Period, error) { + var minReplicas *int + var active, upcoming *Period + + for _, o := range hra.Spec.ScheduledOverrides { + a, u, err := MatchSchedule( + now, o.StartTime.Time, o.EndTime.Time, + RecurrenceRule{ + Frequency: o.RecurrenceRule.Frequency, + UntilTime: o.RecurrenceRule.UntilTime.Time, + }, + ) + if err != nil { + return minReplicas, nil, nil, err + } + + // Use the first when there are two or more active scheduled overrides, + // as the spec defines that the earlier scheduled override is prioritized higher than later ones. + if active == nil { + active = a + + if o.MinReplicas != nil { + minReplicas = o.MinReplicas + } + } + + if upcoming == nil || (u != nil && u.StartTime.Before(upcoming.StartTime)) { + upcoming = u + } + } + + return minReplicas, active, upcoming, nil +} + func (r *HorizontalRunnerAutoscalerReconciler) computeReplicasWithCache(log logr.Logger, now time.Time, rd v1alpha1.RunnerDeployment, hra v1alpha1.HorizontalRunnerAutoscaler) (int, int, *int, error) { minReplicas := defaultReplicas if hra.Spec.MinReplicas != nil && *hra.Spec.MinReplicas >= 0 { minReplicas = *hra.Spec.MinReplicas } + if m, _, _, err := r.matchScheduledOverrides(log, now, hra); err != nil { + return 0, 0, nil, err + } else if m != nil { + minReplicas = *m + } + var suggestedReplicas int suggestedReplicasFromCache := r.fetchSuggestedReplicasFromCache(hra) diff --git a/controllers/horizontalrunnerautoscaler_scheduled_overrides.go b/controllers/schedule.go similarity index 100% rename from controllers/horizontalrunnerautoscaler_scheduled_overrides.go rename to controllers/schedule.go diff --git a/controllers/horizontalrunnerautoscaler_scheduled_overrides_test.go b/controllers/schedule_test.go similarity index 100% rename from controllers/horizontalrunnerautoscaler_scheduled_overrides_test.go rename to controllers/schedule_test.go