Add more telemetry to the role editor (#54486)

Whenever the user clicks the save button, report whether the standard
and YAML modes were used and whether there were any YAML fields that
prevented the user from accessing the standard editor.

Author: bl-nero

api/gen/proto/go/usageevents/v1/usageevents.pb.go

@@ -409,7 +409,12 @@ message UIDiscoverCompletedEvent {
 message UICreateNewRoleClickEvent {}
 
 // UICreateNewRoleSaveClickEvent is an event that can be triggered during custom role creation
-message UICreateNewRoleSaveClickEvent {}
+message UICreateNewRoleSaveClickEvent {
+  bool standard_used = 1;
+  bool yaml_used = 2;
+  string mode_when_saved = 3;
+  repeated string fields_with_conversion_errors = 4;
+}
 
 // UICreateNewRoleCancelClickEvent is an event that can be triggered during custom role creation
 message UICreateNewRoleCancelClickEvent {}

api/proto/teleport/usageevents/v1/usageevents.proto

@@ -504,7 +504,11 @@ func (u *UICreateNewRoleSaveClickEvent) Anonymize(a utils.Anonymizer) prehogv1a.
 	return prehogv1a.SubmitEventRequest{
 		Event: &prehogv1a.SubmitEventRequest_UiCreateNewRoleSaveClick{
 			UiCreateNewRoleSaveClick: &prehogv1a.UICreateNewRoleSaveClickEvent{
-				UserName: a.AnonymizeString(u.UserName),
+				UserName:                   a.AnonymizeString(u.UserName),
+				StandardUsed:               u.StandardUsed,
+				YamlUsed:                   u.YamlUsed,
+				ModeWhenSaved:              u.ModeWhenSaved,
+				FieldsWithConversionErrors: u.FieldsWithConversionErrors,
 			},
 		},
 	}
@@ -1426,7 +1430,11 @@ func ConvertUsageEvent(event *usageeventsv1.UsageEventOneOf, userMD UserMetadata
 		}, nil
 	case *usageeventsv1.UsageEventOneOf_UiCreateNewRoleSaveClick:
 		return &UICreateNewRoleSaveClickEvent{
-			UserName: userMD.Username,
+			UserName:                   userMD.Username,
+			StandardUsed:               e.UiCreateNewRoleSaveClick.StandardUsed,
+			YamlUsed:                   e.UiCreateNewRoleSaveClick.YamlUsed,
+			ModeWhenSaved:              e.UiCreateNewRoleSaveClick.ModeWhenSaved,
+			FieldsWithConversionErrors: e.UiCreateNewRoleSaveClick.FieldsWithConversionErrors,
 		}, nil
 	case *usageeventsv1.UsageEventOneOf_UiCreateNewRoleCancelClick:
 		return &UICreateNewRoleCancelClickEvent{

gen/proto/go/prehog/v1alpha/teleport.pb.go

@@ -354,8 +354,22 @@ func ConvertUserEventRequestToUsageEvent(req CreateUserEventRequest) (*usageeven
 			nil
 
 	case createNewRoleSaveClickEvent:
+		roleEvent := struct {
+			StandardUsed               bool     `json:"standardUsed"`
+			YAMLUsed                   bool     `json:"yamlUsed"`
+			ModeWhenSaved              string   `json:"modeWhenSaved"`
+			FieldsWithConversionErrors []string `json:"fieldsWithConversionErrors"`
+		}{}
+		if err := json.Unmarshal([]byte(*req.EventData), &roleEvent); err != nil {
+			return nil, trace.BadParameter("eventData is invalid: %v", err)
+		}
 		return &usageeventsv1.UsageEventOneOf{Event: &usageeventsv1.UsageEventOneOf_UiCreateNewRoleSaveClick{
-				UiCreateNewRoleSaveClick: &usageeventsv1.UICreateNewRoleSaveClickEvent{},
+				UiCreateNewRoleSaveClick: &usageeventsv1.UICreateNewRoleSaveClickEvent{
+					StandardUsed:               roleEvent.StandardUsed,
+					YamlUsed:                   roleEvent.YAMLUsed,
+					ModeWhenSaved:              roleEvent.ModeWhenSaved,
+					FieldsWithConversionErrors: roleEvent.FieldsWithConversionErrors,
+				},
 			}},
 			nil
 

gen/proto/ts/prehog/v1alpha/teleport_pb.ts

@@ -801,6 +801,21 @@ message UICreateNewRoleClickEvent {
 message UICreateNewRoleSaveClickEvent {
   //anonymized
   string user_name = 1;
+  // StandardUsed indicates if the user interacted with the standard editor
+  // tab.
+  bool standard_used = 2;
+  // YamlUsed indicates if the user interacted with the YAML editor tab.
+  bool yaml_used = 3;
+  // ModeWhenSaved indicates which editor tab was active when the Save button
+  // was clicked ("standard" or "yaml"). Note that it's inherently different
+  // from what `StandardUsed` or `YamlUsed` describe; the user can interact
+  // with one kind of editor, then verify the results in another, and
+  // ultimately click Save on any of them.
+  string mode_when_saved = 4;
+  // A list of field paths that prevented the standard editor from being
+  // operational, if any. Looking at these may provide insight into which
+  // missing features should be implemented first.
+  repeated string fields_with_conversion_errors = 5;
 }
 
 // UICreateNewRoleCancelClickEvent is an event that can be triggered during custom role creation

lib/usagereporter/teleport/types.go

@@ -29,7 +29,12 @@ import ResourceService, {
   RoleWithYaml,
 } from 'teleport/services/resources';
 import { storageService } from 'teleport/services/storageService';
-import { CaptureEvent, userEventService } from 'teleport/services/userEvent';
+import {
+  CaptureEvent,
+  CreateNewRoleSaveClickEventData,
+  RoleEditorMode,
+  userEventService,
+} from 'teleport/services/userEvent';
 import { yamlService } from 'teleport/services/yaml';
 import {
   YamlStringifyRequest,
@@ -78,6 +83,9 @@ beforeEach(() => {
       return toFauxYaml(withDefaults(req.resource));
     });
   jest.spyOn(userEventService, 'captureUserEvent').mockImplementation(() => {});
+  jest
+    .spyOn(userEventService, 'captureCreateNewRoleSaveClickEvent')
+    .mockImplementation(() => {});
   jest
     .spyOn(ResourceService.prototype, 'fetchRole')
     .mockImplementation(async name => {
@@ -133,6 +141,7 @@ test('rendering and switching tabs for new role', async () => {
 }, 10000);
 
 test('rendering and switching tabs for a non-standard role', async () => {
+  const onSave = jest.fn();
   const originalRole = withDefaults({
     metadata: {
       name: 'some-role',
@@ -142,7 +151,12 @@ test('rendering and switching tabs for a non-standard role', async () => {
       deny: { node_labels: { foo: ['bar'] } },
     },
   });
-  render(<TestRoleEditor originalRole={newRoleWithYaml(originalRole)} />);
+  render(
+    <TestRoleEditor
+      originalRole={newRoleWithYaml(originalRole)}
+      onSave={onSave}
+    />
+  );
   expect(getYamlEditorTab()).toHaveAttribute('aria-selected', 'true');
   expect(fromFauxYaml(await getTextEditorContents())).toEqual(originalRole);
   expect(screen.getByRole('button', { name: 'Save Changes' })).toBeDisabled();
@@ -156,6 +170,22 @@ test('rendering and switching tabs for a non-standard role', async () => {
   await user.click(getYamlEditorTab());
   expect(fromFauxYaml(await getTextEditorContents())).toEqual(originalRole);
   expect(screen.getByRole('button', { name: 'Save Changes' })).toBeDisabled();
+
+  await user.clear(await findTextEditor());
+  await user.type(await findTextEditor(), '{{"asdf":"qwer"}');
+  await user.click(screen.getByRole('button', { name: 'Save Changes' }));
+
+  expect(onSave).toHaveBeenCalledWith({
+    yaml: '{"asdf":"qwer"}',
+  });
+  expect(
+    userEventService.captureCreateNewRoleSaveClickEvent
+  ).toHaveBeenCalledWith({
+    standardUsed: false,
+    yamlUsed: true,
+    modeWhenSaved: RoleEditorMode.Yaml,
+    fieldsWithConversionErrors: ['spec.deny.node_labels'],
+  } as CreateNewRoleSaveClickEventData);
 });
 
 it('calls onRoleUpdate on each modification in the standard editor', async () => {
@@ -355,9 +385,14 @@ test('saving a new role', async () => {
       version: 'v8',
     },
   });
-  expect(userEventService.captureUserEvent).toHaveBeenCalledWith({
-    event: CaptureEvent.CreateNewRoleSaveClickEvent,
-  });
+  expect(
+    userEventService.captureCreateNewRoleSaveClickEvent
+  ).toHaveBeenCalledWith({
+    standardUsed: true,
+    yamlUsed: false,
+    modeWhenSaved: RoleEditorMode.Standard,
+    fieldsWithConversionErrors: [],
+  } as CreateNewRoleSaveClickEventData);
 });
 
 describe('saving a new role after editing as YAML', () => {
@@ -373,9 +408,14 @@ describe('saving a new role after editing as YAML', () => {
     expect(onSave).toHaveBeenCalledWith({
       yaml: '{"foo":"bar"}',
     });
-    expect(userEventService.captureUserEvent).toHaveBeenCalledWith({
-      event: CaptureEvent.CreateNewRoleSaveClickEvent,
-    });
+    expect(
+      userEventService.captureCreateNewRoleSaveClickEvent
+    ).toHaveBeenCalledWith({
+      standardUsed: false,
+      yamlUsed: true,
+      modeWhenSaved: RoleEditorMode.Yaml,
+      fieldsWithConversionErrors: [],
+    } as CreateNewRoleSaveClickEventData);
   });
 
   test('with Policy enabled', async () => {
@@ -407,9 +447,14 @@ describe('saving a new role after editing as YAML', () => {
     expect(onSave).toHaveBeenCalledWith({
       yaml: '{"metadata":{"description":"foo"}}',
     });
-    expect(userEventService.captureUserEvent).toHaveBeenCalledWith({
-      event: CaptureEvent.CreateNewRoleSaveClickEvent,
-    });
+    expect(
+      userEventService.captureCreateNewRoleSaveClickEvent
+    ).toHaveBeenCalledWith({
+      standardUsed: false,
+      yamlUsed: true,
+      modeWhenSaved: RoleEditorMode.Yaml,
+      fieldsWithConversionErrors: [],
+    } as CreateNewRoleSaveClickEventData);
   });
 });