Skip to content

Commit 97c710b

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #2291 from tetianakravchenko/add_metric_kube_pod_container_status_last_terminated_timestamp
feat: add kube_pod_container_status_last_terminated_timestamp metric
2 parents 9ba1c37 + 393430e commit 97c710b

File tree

4 files changed

+58
-2
lines changed

4 files changed

+58
-2
lines changed

‎docs/pod-metrics.md

+1
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
| kube_pod_container_status_terminated_reason | Gauge | Describes the reason the container is currently in terminated state | | `container`=&lt;container-name&gt; <br> `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `reason`=&lt;container-terminated-reason&gt; <br> `uid`=&lt;pod-uid&gt; | EXPERIMENTAL | - |
2424
| kube_pod_container_status_last_terminated_reason | Gauge | Describes the last reason the container was in terminated state | | `container`=&lt;container-name&gt; <br> `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `reason`=&lt;last-terminated-reason&gt; <br> `uid`=&lt;pod-uid&gt; | EXPERIMENTAL | - |
2525
| kube_pod_container_status_last_terminated_exitcode | Gauge | Describes the exit code for the last container in terminated state. | | `container`=&lt;container-name&gt; <br> `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `uid`=&lt;pod-uid&gt; | EXPERIMENTAL | - |
26+
| kube_pod_container_status_last_terminated_timestamp | Gauge | Last terminated time for a pod container in unix timestamp. | | `container`=&lt;container-name&gt; <br> `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `uid`=&lt;pod-uid&gt; | EXPERIMENTAL | - |
2627
| kube_pod_container_status_ready | Gauge | Describes whether the containers readiness check succeeded | | `container`=&lt;container-name&gt; <br> `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `uid`=&lt;pod-uid&gt; | STABLE | - |
2728
| kube_pod_status_initialized_time | Gauge | Time when the pod is initialized. | seconds | `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `uid`=&lt;pod-uid&gt; | EXPERIMENTAL |
2829
| kube_pod_status_ready_time | Gauge | Time when pod passed readiness probes. | seconds | `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `uid`=&lt;pod-uid&gt; | EXPERIMENTAL |

‎internal/store/pod.go

+27
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ func podMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generat
4949
createPodContainerStateStartedFamilyGenerator(),
5050
createPodContainerStatusLastTerminatedReasonFamilyGenerator(),
5151
createPodContainerStatusLastTerminatedExitCodeFamilyGenerator(),
52+
createPodContainerStatusLastTerminatedTimestampFamilyGenerator(),
5253
createPodContainerStatusReadyFamilyGenerator(),
5354
createPodContainerStatusRestartsTotalFamilyGenerator(),
5455
createPodContainerStatusRunningFamilyGenerator(),
@@ -375,6 +376,32 @@ func createPodContainerStatusLastTerminatedExitCodeFamilyGenerator() generator.F
375376
)
376377
}
377378

379+
func createPodContainerStatusLastTerminatedTimestampFamilyGenerator() generator.FamilyGenerator {
380+
return *generator.NewFamilyGeneratorWithStability(
381+
"kube_pod_container_status_last_terminated_timestamp",
382+
"Last terminated time for a pod container in unix timestamp.",
383+
metric.Gauge,
384+
basemetrics.ALPHA,
385+
"",
386+
wrapPodFunc(func(p *v1.Pod) *metric.Family {
387+
ms := make([]*metric.Metric, 0, len(p.Status.ContainerStatuses))
388+
for _, cs := range p.Status.ContainerStatuses {
389+
if cs.LastTerminationState.Terminated != nil {
390+
ms = append(ms, &metric.Metric{
391+
LabelKeys: []string{"container"},
392+
LabelValues: []string{cs.Name},
393+
Value: float64(cs.LastTerminationState.Terminated.FinishedAt.Unix()),
394+
})
395+
}
396+
}
397+
398+
return &metric.Family{
399+
Metrics: ms,
400+
}
401+
}),
402+
)
403+
}
404+
378405
func createPodContainerStatusReadyFamilyGenerator() generator.FamilyGenerator {
379406
return *generator.NewFamilyGeneratorWithStability(
380407
"kube_pod_container_status_ready",

‎internal/store/pod_test.go

+24-2
Original file line numberDiff line numberDiff line change
@@ -690,7 +690,6 @@ func TestPodStore(t *testing.T) {
690690
},
691691
},
692692
{
693-
694693
Obj: &v1.Pod{
695694
ObjectMeta: metav1.ObjectMeta{
696695
Name: "pod6",
@@ -720,6 +719,9 @@ func TestPodStore(t *testing.T) {
720719
Terminated: &v1.ContainerStateTerminated{
721720
Reason: "OOMKilled",
722721
ExitCode: 137,
722+
FinishedAt: metav1.Time{
723+
Time: time.Unix(1501779547, 0),
724+
},
723725
},
724726
},
725727
},
@@ -729,6 +731,7 @@ func TestPodStore(t *testing.T) {
729731
Want: `
730732
# HELP kube_pod_container_status_last_terminated_reason Describes the last reason the container was in terminated state.
731733
# HELP kube_pod_container_status_last_terminated_exitcode Describes the exit code for the last container in terminated state.
734+
# HELP kube_pod_container_status_last_terminated_timestamp Last terminated time for a pod container in unix timestamp.
732735
# HELP kube_pod_container_status_running [STABLE] Describes whether the container is currently in running state.
733736
# HELP kube_pod_container_status_terminated [STABLE] Describes whether the container is currently in terminated state.
734737
# HELP kube_pod_container_status_terminated_reason Describes the reason the container is currently in terminated state.
@@ -737,6 +740,7 @@ func TestPodStore(t *testing.T) {
737740
# HELP kube_pod_container_state_started [STABLE] Start time in unix timestamp for a pod container.
738741
# TYPE kube_pod_container_status_last_terminated_reason gauge
739742
# TYPE kube_pod_container_status_last_terminated_exitcode gauge
743+
# TYPE kube_pod_container_status_last_terminated_timestamp gauge
740744
# TYPE kube_pod_container_status_running gauge
741745
# TYPE kube_pod_container_status_terminated gauge
742746
# TYPE kube_pod_container_status_terminated_reason gauge
@@ -749,10 +753,12 @@ func TestPodStore(t *testing.T) {
749753
kube_pod_container_status_waiting{container="container7",namespace="ns6",pod="pod6",uid="uid6"} 0
750754
kube_pod_container_status_last_terminated_reason{container="container7",namespace="ns6",pod="pod6",reason="OOMKilled",uid="uid6"} 1
751755
kube_pod_container_status_last_terminated_exitcode{container="container7",namespace="ns6",pod="pod6",uid="uid6"} 137
756+
kube_pod_container_status_last_terminated_timestamp{container="container7",namespace="ns6",pod="pod6",uid="uid6"} 1.501779547e+09
752757
`,
753758
MetricNames: []string{
754759
"kube_pod_container_status_last_terminated_reason",
755760
"kube_pod_container_status_last_terminated_exitcode",
761+
"kube_pod_container_status_last_terminated_timestamp",
756762
"kube_pod_container_status_running",
757763
"kube_pod_container_state_started",
758764
"kube_pod_container_status_terminated",
@@ -790,6 +796,9 @@ func TestPodStore(t *testing.T) {
790796
Terminated: &v1.ContainerStateTerminated{
791797
Reason: "DeadlineExceeded",
792798
ExitCode: 143,
799+
FinishedAt: metav1.Time{
800+
Time: time.Unix(1501779547, 0),
801+
},
793802
},
794803
},
795804
},
@@ -799,6 +808,7 @@ func TestPodStore(t *testing.T) {
799808
Want: `
800809
# HELP kube_pod_container_status_last_terminated_exitcode Describes the exit code for the last container in terminated state.
801810
# HELP kube_pod_container_status_last_terminated_reason Describes the last reason the container was in terminated state.
811+
# HELP kube_pod_container_status_last_terminated_timestamp Last terminated time for a pod container in unix timestamp.
802812
# HELP kube_pod_container_status_running [STABLE] Describes whether the container is currently in running state.
803813
# HELP kube_pod_container_state_started [STABLE] Start time in unix timestamp for a pod container.
804814
# HELP kube_pod_container_status_terminated [STABLE] Describes whether the container is currently in terminated state.
@@ -807,6 +817,7 @@ func TestPodStore(t *testing.T) {
807817
# HELP kube_pod_container_status_waiting_reason [STABLE] Describes the reason the container is currently in waiting state.
808818
# TYPE kube_pod_container_status_last_terminated_exitcode gauge
809819
# TYPE kube_pod_container_status_last_terminated_reason gauge
820+
# TYPE kube_pod_container_status_last_terminated_timestamp gauge
810821
# TYPE kube_pod_container_status_running gauge
811822
# TYPE kube_pod_container_state_started gauge
812823
# TYPE kube_pod_container_status_terminated gauge
@@ -816,6 +827,7 @@ func TestPodStore(t *testing.T) {
816827
kube_pod_container_state_started{container="container7",namespace="ns7",pod="pod7",uid="uid7"} 1.501777018e+09
817828
kube_pod_container_status_last_terminated_exitcode{container="container7",namespace="ns7",pod="pod7",uid="uid7"} 143
818829
kube_pod_container_status_last_terminated_reason{container="container7",namespace="ns7",pod="pod7",reason="DeadlineExceeded",uid="uid7"} 1
830+
kube_pod_container_status_last_terminated_timestamp{container="container7",namespace="ns7",pod="pod7",uid="uid7"} 1.501779547e+09
819831
kube_pod_container_status_running{container="container7",namespace="ns7",pod="pod7",uid="uid7"} 1
820832
kube_pod_container_status_terminated{container="container7",namespace="ns7",pod="pod7",uid="uid7"} 0
821833
kube_pod_container_status_waiting{container="container7",namespace="ns7",pod="pod7",uid="uid7"} 0
@@ -825,6 +837,7 @@ func TestPodStore(t *testing.T) {
825837
"kube_pod_container_state_started",
826838
"kube_pod_container_status_terminated",
827839
"kube_pod_container_status_terminated_reason",
840+
"kube_pod_container_status_last_terminated_timestamp",
828841
"kube_pod_container_status_waiting",
829842
"kube_pod_container_status_last_terminated_reason",
830843
"kube_pod_container_status_last_terminated_exitcode",
@@ -2210,6 +2223,9 @@ func BenchmarkPodStore(b *testing.B) {
22102223
},
22112224
LastTerminationState: v1.ContainerState{
22122225
Terminated: &v1.ContainerStateTerminated{
2226+
FinishedAt: metav1.Time{
2227+
Time: time.Unix(1501779547, 0),
2228+
},
22132229
Reason: "OOMKilled",
22142230
ExitCode: 137,
22152231
},
@@ -2227,6 +2243,9 @@ func BenchmarkPodStore(b *testing.B) {
22272243
},
22282244
LastTerminationState: v1.ContainerState{
22292245
Terminated: &v1.ContainerStateTerminated{
2246+
FinishedAt: metav1.Time{
2247+
Time: time.Unix(1501779547, 0),
2248+
},
22302249
Reason: "OOMKilled",
22312250
ExitCode: 137,
22322251
},
@@ -2244,6 +2263,9 @@ func BenchmarkPodStore(b *testing.B) {
22442263
},
22452264
LastTerminationState: v1.ContainerState{
22462265
Terminated: &v1.ContainerStateTerminated{
2266+
FinishedAt: metav1.Time{
2267+
Time: time.Unix(1501779547, 0),
2268+
},
22472269
Reason: "OOMKilled",
22482270
ExitCode: 137,
22492271
},
@@ -2253,7 +2275,7 @@ func BenchmarkPodStore(b *testing.B) {
22532275
},
22542276
}
22552277

2256-
expectedFamilies := 53
2278+
expectedFamilies := 54
22572279
for n := 0; n < b.N; n++ {
22582280
families := f(pod)
22592281
if len(families) != expectedFamilies {

‎pkg/app/server_test.go

+6
Original file line numberDiff line numberDiff line change
@@ -209,6 +209,7 @@ func TestFullScrapeCycle(t *testing.T) {
209209
# HELP kube_pod_container_state_started [STABLE] Start time in unix timestamp for a pod container.
210210
# HELP kube_pod_container_status_last_terminated_exitcode Describes the exit code for the last container in terminated state.
211211
# HELP kube_pod_container_status_last_terminated_reason Describes the last reason the container was in terminated state.
212+
# HELP kube_pod_container_status_last_terminated_timestamp Last terminated time for a pod container in unix timestamp.
212213
# HELP kube_pod_container_status_ready [STABLE] Describes whether the containers readiness check succeeded.
213214
# HELP kube_pod_container_status_restarts_total [STABLE] The number of container restarts per container.
214215
# HELP kube_pod_container_status_running [STABLE] Describes whether the container is currently in running state.
@@ -261,6 +262,7 @@ func TestFullScrapeCycle(t *testing.T) {
261262
# TYPE kube_pod_container_state_started gauge
262263
# TYPE kube_pod_container_status_last_terminated_exitcode gauge
263264
# TYPE kube_pod_container_status_last_terminated_reason gauge
265+
# TYPE kube_pod_container_status_last_terminated_timestamp gauge
264266
# TYPE kube_pod_container_status_ready gauge
265267
# TYPE kube_pod_container_status_restarts_total counter
266268
# TYPE kube_pod_container_status_running gauge
@@ -323,6 +325,7 @@ kube_pod_container_resource_requests{namespace="default",pod="pod0",uid="abc-0",
323325
kube_pod_container_resource_requests{namespace="default",pod="pod0",uid="abc-0",container="pod1_con2",node="node1",resource="memory",unit="byte"} 2e+08
324326
kube_pod_container_status_last_terminated_exitcode{namespace="default",pod="pod0",uid="abc-0",container="pod1_con1"} 137
325327
kube_pod_container_status_last_terminated_reason{namespace="default",pod="pod0",uid="abc-0",container="pod1_con1",reason="OOMKilled"} 1
328+
kube_pod_container_status_last_terminated_timestamp{namespace="default",pod="pod0",uid="abc-0",container="pod1_con1"} 1.501779547e+09
326329
kube_pod_container_status_ready{namespace="default",pod="pod0",uid="abc-0",container="pod1_con1"} 0
327330
kube_pod_container_status_ready{namespace="default",pod="pod0",uid="abc-0",container="pod1_con2"} 0
328331
kube_pod_container_status_restarts_total{namespace="default",pod="pod0",uid="abc-0",container="pod1_con1"} 0
@@ -834,6 +837,9 @@ func pod(client *fake.Clientset, index int) error {
834837
},
835838
LastTerminationState: v1.ContainerState{
836839
Terminated: &v1.ContainerStateTerminated{
840+
FinishedAt: metav1.Time{
841+
Time: time.Unix(1501779547, 0),
842+
},
837843
Reason: "OOMKilled",
838844
ExitCode: 137,
839845
},

0 commit comments

Comments
 (0)