Add failure reasons to tool call analytics events

pkgs/dart_mcp_server/CHANGELOG.md

@@ -2,6 +2,8 @@
 
 * Change tools that accept multiple roots to not return immediately on the first
   failure.
+* Add failure reason field to analytics events so we can know why tool calls are
+  failing.
 
 # 0.1.0 (Dart SDK 3.9.0)
 

pkgs/dart_mcp_server/lib/src/mixins/analyzer.dart

@@ -13,6 +13,7 @@ import 'package:language_server_protocol/protocol_generated.dart' as lsp;
 import 'package:meta/meta.dart';
 
 import '../lsp/wire_format.dart';
+import '../utils/analytics.dart';
 import '../utils/constants.dart';
 import '../utils/sdk.dart';
 
@@ -474,7 +475,7 @@ base mixin DartAnalyzerSupport
             'tool.',
       ),
     ],
-  );
+  )..failureReason = CallToolFailureReason.noRootsSet;
 }
 
 /// Common schema for tools that require a file URI, line, and column.

pkgs/dart_mcp_server/lib/src/mixins/dash_cli.dart

@@ -8,6 +8,7 @@ import 'dart:convert';
 import 'package:dart_mcp/server.dart';
 import 'package:path/path.dart' as p;
 
+import '../utils/analytics.dart';
 import '../utils/cli_utils.dart';
 import '../utils/constants.dart';
 import '../utils/file_system.dart';
@@ -141,7 +142,7 @@ base mixin DashCliSupport on ToolsSupport, LoggingSupport, RootsTrackingSupport
           for (final error in errors) Content.text(text: error.toErrorString()),
         ],
         isError: true,
-      );
+      )..failureReason = CallToolFailureReason.argumentError;
     }
 
     final template = args[ParameterNames.template] as String?;

pkgs/dart_mcp_server/lib/src/mixins/dtd.dart

@@ -199,7 +199,7 @@ base mixin DartToolingDaemonSupport
             text: 'Connection failed, make sure your DTD Uri is up to date.',
           ),
         ],
-      );
+      )..failureReason = CallToolFailureReason.webSocketException;
     } catch (e) {
       return CallToolResult(
         isError: true,
@@ -532,7 +532,7 @@ base mixin DartToolingDaemonSupport
                 'boolean.',
           ),
         ],
-      );
+      )..failureReason = CallToolFailureReason.argumentError;
     }
 
     return _callOnVmService(
@@ -750,7 +750,7 @@ base mixin DartToolingDaemonSupport
             'connect to Dart and Flutter applications.',
       ),
     ],
-  );
+  )..failureReason = CallToolFailureReason.connectedAppServiceNotSupported;
 
   static final _dtdNotConnected = CallToolResult(
     isError: true,
@@ -761,7 +761,7 @@ base mixin DartToolingDaemonSupport
             '"${connectTool.name}" first.',
       ),
     ],
-  );
+  )..failureReason = CallToolFailureReason.dtdNotConnected;
 
   static final _dtdAlreadyConnected = CallToolResult(
     isError: true,
@@ -772,14 +772,14 @@ base mixin DartToolingDaemonSupport
             '"${connectTool.name}" again.',
       ),
     ],
-  );
+  )..failureReason = CallToolFailureReason.dtdAlreadyConnected;
 
   static final _noActiveDebugSession = CallToolResult(
     content: [
       TextContent(text: 'No active debug session to take a screenshot'),
     ],
     isError: true,
-  );
+  )..failureReason = CallToolFailureReason.noActiveDebugSession;
 
   static final runtimeErrorsScheme = 'runtime-errors';
 }

pkgs/dart_mcp_server/lib/src/mixins/pub.dart

@@ -6,6 +6,7 @@ import 'dart:async';
 
 import 'package:dart_mcp/server.dart';
 
+import '../utils/analytics.dart';
 import '../utils/cli_utils.dart';
 import '../utils/constants.dart';
 import '../utils/file_system.dart';
@@ -47,7 +48,7 @@ base mixin PubSupport on ToolsSupport, LoggingSupport, RootsTrackingSupport
           ),
         ],
         isError: true,
-      );
+      )..failureReason ??= CallToolFailureReason.noSuchCommand;
     }
 
     final packageName =
@@ -62,7 +63,7 @@ base mixin PubSupport on ToolsSupport, LoggingSupport, RootsTrackingSupport
           ),
         ],
         isError: true,
-      );
+      )..failureReason ??= CallToolFailureReason.argumentError;
     }
 
     return runCommandInRoots(

pkgs/dart_mcp_server/lib/src/mixins/pub_dev_search.dart

@@ -9,6 +9,7 @@ import 'package:dart_mcp/server.dart';
 import 'package:http/http.dart';
 import 'package:pool/pool.dart';
 
+import '../utils/analytics.dart';
 import '../utils/json.dart';
 
 /// Limit the number of concurrent requests.
@@ -36,7 +37,7 @@ base mixin PubDevSupport on ToolsSupport {
       return CallToolResult(
         content: [TextContent(text: 'Missing required argument `query`.')],
         isError: true,
-      );
+      )..failureReason = CallToolFailureReason.argumentError;
     }
     final searchUrl = Uri.https('pub.dev', 'api/search', {'q': query});
     final Object? result;
@@ -54,7 +55,6 @@ base mixin PubDevSupport on ToolsSupport {
               text: 'No packages matched the query, consider simplifying it.',
             ),
           ],
-          isError: true,
         );
       }
 

pkgs/dart_mcp_server/lib/src/server.dart

@@ -198,6 +198,7 @@ final class DartMCPServer extends MCPServer
                         tool: request.name,
                         success: result != null && result.isError != true,
                         elapsedMilliseconds: watch.elapsedMilliseconds,
+                        failureReason: result?.failureReason,
                       ),
                     ),
                   );

pkgs/dart_mcp_server/lib/src/utils/analytics.dart

@@ -2,6 +2,7 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
+import 'package:dart_mcp/server.dart';
 import 'package:unified_analytics/unified_analytics.dart';
 
 /// An interface class that provides a access to an [Analytics] instance, if
@@ -53,23 +54,57 @@ final class CallToolMetrics extends CustomMetrics {
   /// The time it took to invoke the tool.
   final int elapsedMilliseconds;
 
+  /// The reason for the failure, if [success] is `false`.
+  final CallToolFailureReason? failureReason;
+
   CallToolMetrics({
     required this.tool,
     required this.success,
     required this.elapsedMilliseconds,
+    required this.failureReason,
   });
 
   @override
   Map<String, Object> toMap() => {
     _tool: tool,
     _success: success,
     _elapsedMilliseconds: elapsedMilliseconds,
+    _failureReason: ?failureReason?.name,
   };
 }
 
 enum ResourceKind { runtimeErrors }
 
+/// Extension which tracks failure reasons for [CallToolResult] objects in an
+/// [Expando].
+extension WithFailureReason on CallToolResult {
+  static final _expando = Expando<CallToolFailureReason>();
+
+  CallToolFailureReason? get failureReason => _expando[this as Object];
+
+  set failureReason(CallToolFailureReason? value) =>
+      _expando[this as Object] = value;
+}
+
+/// Known reasons for failed tool calls.
+enum CallToolFailureReason {
+  argumentError,
+  connectedAppServiceNotSupported,
+  dtdAlreadyConnected,
+  dtdNotConnected,
+  invalidPath,
+  invalidRootPath,
+  invalidRootScheme,
+  noActiveDebugSession,
+  noRootGiven,
+  noRootsSet,
+  noSuchCommand,
+  nonZeroExitCode,
+  webSocketException,
+}
+
 const _elapsedMilliseconds = 'elapsedMilliseconds';
+const _failureReason = 'failureReason';
 const _kind = 'kind';
 const _length = 'length';
 const _success = 'success';

pkgs/dart_mcp_server/lib/src/utils/cli_utils.dart

@@ -10,6 +10,7 @@ import 'package:file/file.dart';
 import 'package:process/process.dart';
 import 'package:yaml/yaml.dart';
 
+import 'analytics.dart';
 import 'constants.dart';
 import 'sdk.dart';
 
@@ -159,7 +160,7 @@ Future<CallToolResult> runCommandInRoot(
         TextContent(text: 'Invalid root configuration: missing `root` key.'),
       ],
       isError: true,
-    );
+    )..failureReason ??= CallToolFailureReason.noRootGiven;
   }
 
   final root = knownRoots.firstWhereOrNull(
@@ -175,7 +176,7 @@ Future<CallToolResult> runCommandInRoot(
         ),
       ],
       isError: true,
-    );
+    )..failureReason ??= CallToolFailureReason.invalidRootPath;
   }
 
   final rootUri = Uri.parse(rootUriString);
@@ -189,7 +190,7 @@ Future<CallToolResult> runCommandInRoot(
         ),
       ],
       isError: true,
-    );
+    )..failureReason ??= CallToolFailureReason.invalidRootScheme;
   }
   final projectRoot = fileSystem.directory(rootUri);
 
@@ -213,7 +214,7 @@ Future<CallToolResult> runCommandInRoot(
         ),
       ],
       isError: true,
-    );
+    )..failureReason ??= CallToolFailureReason.invalidPath;
   }
   commandWithPaths.addAll(paths);
 
@@ -238,7 +239,7 @@ Future<CallToolResult> runCommandInRoot(
         ),
       ],
       isError: true,
-    );
+    )..failureReason ??= CallToolFailureReason.nonZeroExitCode;
   }
   return CallToolResult(
     content: [

pkgs/dart_mcp_server/test/dart_tooling_mcp_server_test.dart

@@ -7,6 +7,7 @@ import 'dart:io';
 
 import 'package:dart_mcp/server.dart';
 import 'package:dart_mcp_server/src/server.dart';
+import 'package:dart_mcp_server/src/utils/analytics.dart';
 import 'package:test/test.dart';
 import 'package:test_descriptor/test_descriptor.dart' as d;
 import 'package:unified_analytics/testing.dart';
@@ -56,32 +57,43 @@ void main() {
     });
 
     test('sends analytics for failed tool calls', () async {
-      server.registerTool(
-        Tool(name: 'hello', inputSchema: Schema.object()),
-        (_) => CallToolResult(isError: true, content: []),
-      );
-      final result = await testHarness.mcpServerConnection.callTool(
-        CallToolRequest(name: 'hello'),
-      );
-      expect(result.isError, true);
-      expect(
-        analytics.sentEvents.single,
-        isA<Event>()
-            .having((e) => e.eventName, 'eventName', DashEvent.dartMCPEvent)
-            .having(
-              (e) => e.eventData,
-              'eventData',
-              equals({
-                'client': server.clientInfo.name,
-                'clientVersion': server.clientInfo.version,
-                'serverVersion': server.implementation.version,
-                'type': 'callTool',
-                'tool': 'hello',
-                'success': false,
-                'elapsedMilliseconds': isA<int>(),
-              }),
-            ),
-      );
+      for (var reason in [null, CallToolFailureReason.nonZeroExitCode]) {
+        analytics.sentEvents.clear();
+
+        final tool = Tool(
+          name: 'hello${reason?.name ?? ''}',
+          inputSchema: Schema.object(),
+        );
+        server.registerTool(
+          tool,
+          (_) =>
+              CallToolResult(isError: true, content: [])
+                ..failureReason = reason,
+        );
+        final result = await testHarness.mcpServerConnection.callTool(
+          CallToolRequest(name: tool.name),
+        );
+        expect(result.isError, true);
+        expect(
+          analytics.sentEvents.single,
+          isA<Event>()
+              .having((e) => e.eventName, 'eventName', DashEvent.dartMCPEvent)
+              .having(
+                (e) => e.eventData,
+                'eventData',
+                equals({
+                  'client': server.clientInfo.name,
+                  'clientVersion': server.clientInfo.version,
+                  'serverVersion': server.implementation.version,
+                  'type': 'callTool',
+                  'tool': tool.name,
+                  'success': false,
+                  'elapsedMilliseconds': isA<int>(),
+                  'failureReason': ?reason?.name,
+                }),
+              ),
+        );
+      }
     });
 
     test('Changelog version matches dart server version', () {

pkgs/dart_mcp_server/test/tools/pub_dev_search_test.dart

@@ -175,7 +175,7 @@ void main() {
     );
   });
 
-  test('No matching packages gets reported as an error', () async {
+  test('No matching packages gets special handling', () async {
     await runWithClient(
       () async {
         await runWithHarness((testHarness, pubDevSearchTool) async {
@@ -187,9 +187,8 @@ void main() {
           final result = await testHarness.callToolWithRetry(
             request,
             maxTries: 1,
-            expectError: true,
           );
-          expect(result.isError, isTrue);
+          expect(result.isError, isNot(true));
           expect(
             (result.content[0] as TextContent).text,
             contains('No packages matched the query, consider simplifying it'),