fix jobs dashboard missing gpu health in table (#449) (#450) (#451)

(cherry picked from commit 8f67ad10b875f2d2b73880b77c0642383398f8aa)


(cherry picked from commit a959f23a3cfa5319f07b34bb61e19ca934f9935f)

Co-authored-by: Titus Ou <[email protected]>

Author: 2 people

grafana/dashboard_job.json

@@ -2419,7 +2419,7 @@
         "type": "prometheus",
         "uid": "${DS_PROMETHEUS}"
       },
-      "description": "List of all GPUs used by this job.",
+      "description": "List of all GPUs used by this job. Health is the last known status of the GPU during a running job in the selected time range.",
       "fieldConfig": {
         "defaults": {
           "color": {
@@ -2569,7 +2569,7 @@
         "showHeader": true,
         "sortBy": [
           {
-            "desc": true,
+            "desc": false,
             "displayName": "HEALTH"
           },
           {
@@ -2589,8 +2589,8 @@
             "type": "prometheus",
             "uid": "${DS_PROMETHEUS}"
           },
-          "editorMode": "builder",
-          "expr": "${g_metrics_prefix}gpu_health{job_id!=\"\", job_id=\"$g_job_id\"}",
+          "editorMode": "code",
+          "expr": "${g_metrics_prefix}gpu_health{job_id!=\"\", job_id=\"$g_job_id\"} or vector(0)",
           "instant": false,
           "legendFormat": "__auto",
           "range": true,
@@ -2601,8 +2601,8 @@
             "type": "prometheus",
             "uid": "${DS_PROMETHEUS}"
           },
-          "editorMode": "builder",
-          "expr": "${g_metrics_prefix}gpu_health{pod!=\"\", pod=\"$g_pod\"}",
+          "editorMode": "code",
+          "expr": "${g_metrics_prefix}gpu_health{pod!=\"\", pod=\"$g_pod\"} or vector(0)",
           "hide": false,
           "instant": false,
           "legendFormat": "__auto",
@@ -2672,6 +2672,22 @@
               "hostname (last)": "HOSTNAME"
             }
           }
+        },
+        {
+          "id": "filterByValue",
+          "options": {
+            "filters": [
+              {
+                "config": {
+                  "id": "isNull",
+                  "options": {}
+                },
+                "fieldName": "GPU UUID"
+              }
+            ],
+            "match": "any",
+            "type": "exclude"
+          }
         }
       ],
       "type": "table"

grafana/dashboard_node.json

@@ -2232,7 +2232,7 @@
         "showHeader": true,
         "sortBy": [
           {
-            "desc": true,
+            "desc": false,
             "displayName": "HEALTH"
           },
           {

grafana/dashboard_overview.json

@@ -2120,7 +2120,7 @@
         "showHeader": true,
         "sortBy": [
           {
-            "desc": true,
+            "desc": false,
             "displayName": "HEALTH"
           },
           {