Skip to content

feat(metrics): add volume filesystem read only state metrics #3082

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Sep 18, 2024
Merged

feat(metrics): add volume filesystem read only state metrics #3082

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 4 additions & 19 deletions controller/volume_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1491,25 +1491,10 @@ func (c *VolumeController) requestRemountIfFileSystemReadOnly(v *longhorn.Volume
if v.Status.State == longhorn.VolumeStateAttached && e.Status.CurrentState == longhorn.InstanceStateRunning {
fileSystemReadOnlyCondition := types.GetCondition(e.Status.Conditions, imtypes.EngineConditionFilesystemReadOnly)

isPVMountOptionReadOnly := false
kubeStatus := v.Status.KubernetesStatus
if kubeStatus.PVName != "" {
pv, err := c.ds.GetPersistentVolumeRO(kubeStatus.PVName)
if err != nil {
if apierrors.IsNotFound(err) {
return
}
log.WithError(err).Warnf("Failed to get PV when checking the mount option of the volume")
return
}
if pv != nil {
for _, opt := range pv.Spec.MountOptions {
if opt == "ro" {
isPVMountOptionReadOnly = true
break
}
}
}
isPVMountOptionReadOnly, err := c.ds.IsPVMountOptionReadOnly(v)
if err != nil {
log.WithError(err).Warn("Failed to check if volume's PV mount option is read only")
return
}

if fileSystemReadOnlyCondition.Status == longhorn.ConditionStatusTrue && !isPVMountOptionReadOnly {
Expand Down
20 changes: 20 additions & 0 deletions datastore/longhorn.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5556,3 +5556,23 @@ func (s *DataStore) GetOneBackingImageReadyNodeDisk(backingImage *longhorn.Backi

return nil, "", fmt.Errorf("failed to find one ready backing image %v", backingImage.Name)
}

func (s *DataStore) IsPVMountOptionReadOnly(volume *longhorn.Volume) (bool, error) {
kubeStatus := volume.Status.KubernetesStatus
if kubeStatus.PVName == "" {
return false, nil
}

pv, err := s.GetPersistentVolumeRO(kubeStatus.PVName)
if err != nil {
return false, err
}
if pv != nil {
for _, opt := range pv.Spec.MountOptions {
if opt == "ro" {
return true, nil
}
}
}
return false, nil
}
36 changes: 31 additions & 5 deletions metrics_collector/volume_collector.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,15 @@ package metricscollector

import (
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
"github.com/sirupsen/logrus"

"github.com/prometheus/client_golang/prometheus"
imtypes "github.com/longhorn/longhorn-instance-manager/pkg/types"

"github.com/longhorn/longhorn-manager/controller"
"github.com/longhorn/longhorn-manager/datastore"
"github.com/longhorn/longhorn-manager/engineapi"
"github.com/longhorn/longhorn-manager/types"
"github.com/longhorn/longhorn-manager/util"

longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2"
Expand All @@ -19,10 +21,11 @@ type VolumeCollector struct {

proxyConnCounter util.Counter

capacityMetric metricInfo
sizeMetric metricInfo
stateMetric metricInfo
robustnessMetric metricInfo
capacityMetric metricInfo
sizeMetric metricInfo
stateMetric metricInfo
robustnessMetric metricInfo
fileSystemReadOnlyMetric metricInfo

volumePerfMetrics
}
Expand All @@ -48,6 +51,16 @@ func NewVolumeCollector(
proxyConnCounter: util.NewAtomicCounter(),
}

vc.fileSystemReadOnlyMetric = metricInfo{
Desc: prometheus.NewDesc(
prometheus.BuildFQName(longhornName, subsystemVolume, "file_system_read_only"),
"Volume whose mount point is in read-only mode",
[]string{nodeLabel, volumeLabel, pvcLabel, pvcNamespaceLabel},
nil,
),
Type: prometheus.GaugeValue,
}

vc.capacityMetric = metricInfo{
Desc: prometheus.NewDesc(
prometheus.BuildFQName(longhornName, subsystemVolume, "capacity_bytes"),
Expand Down Expand Up @@ -156,6 +169,7 @@ func (vc *VolumeCollector) Describe(ch chan<- *prometheus.Desc) {
ch <- vc.sizeMetric.Desc
ch <- vc.stateMetric.Desc
ch <- vc.robustnessMetric.Desc
ch <- vc.fileSystemReadOnlyMetric.Desc
}

func (vc *VolumeCollector) Collect(ch chan<- prometheus.Metric) {
Expand Down Expand Up @@ -214,6 +228,18 @@ func (vc *VolumeCollector) collectMetrics(ch chan<- prometheus.Metric, v *longho
ch <- prometheus.MustNewConstMetric(vc.volumePerfMetrics.iopsMetrics.write.Desc, vc.volumePerfMetrics.iopsMetrics.write.Type, float64(vc.getVolumeWriteIOPS(metrics)), vc.currentNodeID, v.Name, v.Status.KubernetesStatus.PVCName, v.Status.KubernetesStatus.Namespace)
ch <- prometheus.MustNewConstMetric(vc.volumePerfMetrics.latencyMetrics.read.Desc, vc.volumePerfMetrics.latencyMetrics.read.Type, float64(vc.getVolumeReadLatency(metrics)), vc.currentNodeID, v.Name, v.Status.KubernetesStatus.PVCName, v.Status.KubernetesStatus.Namespace)
ch <- prometheus.MustNewConstMetric(vc.volumePerfMetrics.latencyMetrics.write.Desc, vc.volumePerfMetrics.latencyMetrics.write.Type, float64(vc.getVolumeWriteLatency(metrics)), vc.currentNodeID, v.Name, v.Status.KubernetesStatus.PVCName, v.Status.KubernetesStatus.Namespace)

fileSystemReadOnlyCondition := types.GetCondition(e.Status.Conditions, imtypes.EngineConditionFilesystemReadOnly)
isPVMountOptionReadOnly, err := vc.ds.IsPVMountOptionReadOnly(v)
if err != nil {
vc.logger.WithError(err).Warn("Failed to check if volume's PV mount option is read only during metric collection")
return
}

if fileSystemReadOnlyCondition.Status == longhorn.ConditionStatusTrue && !isPVMountOptionReadOnly {
ch <- prometheus.MustNewConstMetric(vc.fileSystemReadOnlyMetric.Desc, vc.fileSystemReadOnlyMetric.Type, float64(1), vc.currentNodeID, v.Name, v.Status.KubernetesStatus.PVCName, v.Status.KubernetesStatus.Namespace)
}

}

func (vc *VolumeCollector) getEngineClientProxy(engine *longhorn.Engine) (c engineapi.EngineClientProxy, err error) {
Expand Down