Do not update JobSets that are not suspended

Signed-off-by: Antonin Stefanutti <[email protected]>

Author: astefanutti

pkg/runtime/framework/plugins/jobset/jobset.go

@@ -23,12 +23,14 @@ import (
 	"maps"
 
 	"github.com/go-logr/logr"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	apiruntime "k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	metav1ac "k8s.io/client-go/applyconfigurations/meta/v1"
+	"k8s.io/utils/ptr"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/cache"
@@ -90,6 +92,20 @@ func (j *JobSet) Build(ctx context.Context, info *runtime.Info, trainJob *traine
 		return nil, fmt.Errorf("runtime info or object is missing")
 	}
 
+	// Do not update the JobSet if it already exists and is not suspended
+	oldJobSet := &jobsetv1alpha2.JobSet{}
+	if err := j.client.Get(ctx, client.ObjectKeyFromObject(trainJob), oldJobSet); err != nil {
+		if !apierrors.IsNotFound(err) {
+			return nil, err
+		}
+		oldJobSet = nil
+	}
+	if oldJobSet != nil &&
+		!ptr.Deref(trainJob.Spec.Suspend, false) &&
+		!ptr.Deref(oldJobSet.Spec.Suspend, false) {
+		return nil, nil
+	}
+
 	// Get the runtime as unstructured from the TrainJob ref
 	runtimeJobTemplate := &unstructured.Unstructured{}
 	runtimeJobTemplate.SetAPIVersion(trainer.GroupVersion.String())