feat: add comprehensive testing guidelines and implement tests for LLM constants and providers

Author: mnismt

.sourcegraph/tests.rule.md

@@ -0,0 +1,77 @@
+# Testing Guidelines for Cody++
+
+## File Patterns
+
+- `src/**/__tests__/**/*.test.ts` - All test files
+
+## Testing Framework
+
+Cody++ uses:
+
+- Mocha testing framework with `suite` and `test` functions
+- Sinon for stubbing and mocking dependencies
+- Assert from Node.js for assertions
+- VS Code test API for extension testing
+
+## Test Structure Guidelines
+
+1. Location and Naming
+
+- Place test files in a **tests** directory adjacent to the file being tested
+- Name test files with the pattern [filename].test.ts
+- Use descriptive test names following the pattern: should <expected behavior> when <condition>
+
+2. Tests should follow the pattern:
+
+   - `setup()` - Set up test dependencies and stubs
+   - Test cases with descriptive names
+   - `teardown()` - Clean up stubs and resources
+
+3. Use Sinon sandbox pattern:
+
+```typescript
+let sandbox: sinon.SinonSandbox
+
+setup(() => {
+  sandbox = sinon.createSandbox()
+  // Setup stubs
+})
+
+teardown(() => {
+  sandbox.restore()
+})
+```
+
+4. For VS Code API testing:
+
+   - Use `vscode.workspace` stubs for configuration
+   - Mock filesystem operations
+   - Use proper event emitters for testing events
+
+5. For service testing (like telemetry):
+   - Reset singletons between tests
+   - Mock external dependencies
+   - Verify correct calls are made
+
+## Extension Testing
+
+For extension-level tests:
+
+- Verify command registration
+- Test activation events
+- Use timeouts for async operations
+
+## Running Tests
+
+After writing or modifying tests, always run the full test suite to ensure all tests pass:
+
+```
+pnpm test
+```
+
+This command will:
+
+- Compile the TypeScript files
+- Start the VS Code extension host
+- Execute all test suites
+- Report test results and any failures

src/core/llm/__tests__/constants.test.ts

@@ -0,0 +1,68 @@
+import * as assert from 'assert'
+import {
+  API_ENDPOINTS,
+  CONFIG_KEYS,
+  CONTENT_TYPES,
+  DEFAULT_MODELS,
+  ERROR_MESSAGES,
+  HEADERS,
+  SOURCEGRAPH_SUPPORTED_LLM_PROVIDERS
+} from '../constants'
+
+suite('LLM Constants', () => {
+  test('should export CONFIG_KEYS with correct values', () => {
+    assert.deepStrictEqual(CONFIG_KEYS, {
+      API_KEY: 'llmApiKey',
+      MODEL: 'llmModel',
+      OPENAI_BASE_URL: 'openaiBaseUrl'
+    })
+  })
+
+  test('should export API_ENDPOINTS with correct values', () => {
+    assert.ok(API_ENDPOINTS.SOURCEGRAPH)
+    assert.strictEqual(API_ENDPOINTS.SOURCEGRAPH.BASE_URL, 'https://sourcegraph.com')
+    assert.strictEqual(API_ENDPOINTS.SOURCEGRAPH.MODELS, '/.api/modelconfig/supported-models.json')
+    assert.strictEqual(API_ENDPOINTS.SOURCEGRAPH.COMPLETIONS, '/.api/completions/stream')
+    assert.strictEqual(API_ENDPOINTS.SOURCEGRAPH.GRAPHQL, '/.api/graphql')
+
+    assert.ok(API_ENDPOINTS.OPENAI)
+    assert.strictEqual(API_ENDPOINTS.OPENAI.DEFAULT_BASE_URL, 'https://api.openai.com/v1')
+    assert.strictEqual(API_ENDPOINTS.OPENAI.MODELS, '/models')
+    assert.strictEqual(API_ENDPOINTS.OPENAI.CHAT_COMPLETIONS, '/chat/completions')
+  })
+
+  test('should export DEFAULT_MODELS with correct values', () => {
+    assert.deepStrictEqual(DEFAULT_MODELS, {
+      SOURCEGRAPH: 'claude-3.5-sonnet',
+      OPENAI: 'gpt-4o-mini'
+    })
+  })
+
+  test('should export SOURCEGRAPH_SUPPORTED_LLM_PROVIDERS with correct values', () => {
+    assert.deepStrictEqual(SOURCEGRAPH_SUPPORTED_LLM_PROVIDERS, ['anthropic', 'google', 'openai'])
+  })
+
+  test('should export ERROR_MESSAGES with correct values', () => {
+    assert.deepStrictEqual(ERROR_MESSAGES, {
+      NOT_AUTHENTICATED: 'Authentication required. Please sign in.',
+      INVALID_TOKEN: 'Invalid authentication token.',
+      NO_TOKEN: 'No token provided.',
+      NETWORK_ERROR: 'Network request failed.',
+      UNKNOWN_ERROR: 'An unknown error occurred.',
+      INVALID_RESPONSE: 'Invalid response format from API.'
+    })
+  })
+
+  test('should export HEADERS with correct values', () => {
+    assert.deepStrictEqual(HEADERS, {
+      CONTENT_TYPE: 'Content-Type',
+      AUTHORIZATION: 'Authorization'
+    })
+  })
+
+  test('should export CONTENT_TYPES with correct values', () => {
+    assert.deepStrictEqual(CONTENT_TYPES, {
+      JSON: 'application/json'
+    })
+  })
+})

src/core/llm/__tests__/index.test.ts

@@ -0,0 +1,70 @@
+import * as assert from 'assert'
+import * as sinon from 'sinon'
+import * as vscode from 'vscode'
+import { LLMProvider } from '../../../constants/llm'
+import { createProvider } from '../index'
+import { OpenAIProvider } from '../providers/openai'
+import { SourcegraphProvider } from '../providers/sourcegraph'
+
+suite('LLM Provider Factory', () => {
+  let sandbox: sinon.SinonSandbox
+  let getConfigurationStub: sinon.SinonStub
+  let configGetStub: sinon.SinonStub
+
+  setup(() => {
+    sandbox = sinon.createSandbox()
+
+    // Mock VS Code workspace configuration
+    configGetStub = sandbox.stub()
+
+    getConfigurationStub = sandbox.stub(vscode.workspace, 'getConfiguration')
+    getConfigurationStub.returns({
+      get: configGetStub
+    })
+  })
+
+  teardown(() => {
+    sandbox.restore()
+  })
+
+  test('should create OpenAI provider when configured', () => {
+    configGetStub.withArgs('llmProvider').returns(LLMProvider.OpenAI)
+
+    const provider = createProvider()
+
+    // Verify getConfiguration was called at least once with 'codyPlusPlus'
+    sinon.assert.calledWith(getConfigurationStub, 'codyPlusPlus')
+    // Verify the provider type get was called
+    sinon.assert.calledWith(configGetStub, 'llmProvider')
+
+    assert.ok(provider instanceof OpenAIProvider)
+    assert.strictEqual(provider.providerIdentifier, LLMProvider.OpenAI)
+  })
+
+  test('should create Sourcegraph provider when configured', () => {
+    configGetStub.withArgs('llmProvider').returns(LLMProvider.Sourcegraph)
+
+    const provider = createProvider()
+
+    // Verify getConfiguration was called at least once with 'codyPlusPlus'
+    sinon.assert.calledWith(getConfigurationStub, 'codyPlusPlus')
+    // Verify the provider type get was called
+    sinon.assert.calledWith(configGetStub, 'llmProvider')
+
+    assert.ok(provider instanceof SourcegraphProvider)
+    assert.strictEqual(provider.providerIdentifier, LLMProvider.Sourcegraph)
+  })
+
+  test('should throw error when invalid provider configured', () => {
+    configGetStub.withArgs('llmProvider').returns('invalid-provider')
+
+    assert.throws(() => {
+      createProvider()
+    }, /Unsupported LLM provider: invalid-provider/)
+
+    // Verify getConfiguration was called with 'codyPlusPlus'
+    sinon.assert.calledWith(getConfigurationStub, 'codyPlusPlus')
+    // Verify the provider type get was called
+    sinon.assert.calledWith(configGetStub, 'llmProvider')
+  })
+})

src/core/llm/__tests__/types.test.ts

@@ -0,0 +1,68 @@
+import * as assert from 'assert'
+import { LLMProvider } from '../../../constants/llm'
+import { BaseLLMProvider, CompletionConfig, CompletionRequest, CompletionResponse } from '../types'
+
+suite('LLM Types', () => {
+  test('should correctly type CompletionConfig', () => {
+    const config: CompletionConfig = {
+      model: 'test-model',
+      maxTokens: 500,
+      temperature: 0.7,
+      responseFormat: {
+        type: 'json',
+        schema: { type: 'object' }
+      }
+    }
+
+    assert.strictEqual(config.model, 'test-model')
+    assert.strictEqual(config.maxTokens, 500)
+    assert.strictEqual(config.temperature, 0.7)
+    assert.strictEqual(config.responseFormat?.type, 'json')
+  })
+
+  test('should correctly type CompletionRequest', () => {
+    const request: CompletionRequest = {
+      messages: [
+        { role: 'system', content: 'You are a helpful assistant' },
+        { role: 'user', content: 'Hello' }
+      ],
+      config: {
+        model: 'test-model',
+        maxTokens: 100
+      }
+    }
+
+    assert.strictEqual(request.messages.length, 2)
+    assert.strictEqual(request.messages[0].role, 'system')
+    assert.strictEqual(request.messages[0].content, 'You are a helpful assistant')
+    assert.strictEqual(request.messages[1].role, 'user')
+    assert.strictEqual(request.messages[1].content, 'Hello')
+    assert.strictEqual(request.config?.model, 'test-model')
+  })
+
+  test('should correctly type CompletionResponse', () => {
+    const response: CompletionResponse = {
+      text: 'This is a response'
+    }
+
+    assert.strictEqual(response.text, 'This is a response')
+  })
+
+  test('should correctly implement BaseLLMProvider interface', async () => {
+    // Create a mock implementation of BaseLLMProvider
+    const mockProvider: BaseLLMProvider = {
+      providerIdentifier: LLMProvider.OpenAI,
+      complete: async (request: CompletionRequest): Promise<CompletionResponse> => {
+        return { text: 'Mock response' }
+      }
+    }
+
+    assert.strictEqual(mockProvider.providerIdentifier, LLMProvider.OpenAI)
+
+    const response = await mockProvider.complete({
+      messages: [{ role: 'user', content: 'Test' }]
+    })
+
+    assert.strictEqual(response.text, 'Mock response')
+  })
+})

src/core/llm/__tests__/utils.test.ts

@@ -0,0 +1,108 @@
+import * as assert from 'assert'
+import * as sinon from 'sinon'
+import * as vscode from 'vscode'
+import { FEW_SHOT_EXAMPLES, SYSTEM_PROMPT } from '../../../constants/llm'
+import * as fileOperations from '../../filesystem/operations'
+import { createCompletionRequestMessages, parseLLMResponse } from '../utils'
+
+// Define FileMetadata interface to match what's expected by filesystem operations
+interface FileMetadata {
+  name: string
+  path: string
+  isDirectory: boolean
+  type: string
+}
+
+suite('LLM Utils', () => {
+  let sandbox: sinon.SinonSandbox
+
+  setup(() => {
+    sandbox = sinon.createSandbox()
+  })
+
+  teardown(() => {
+    sandbox.restore()
+  })
+
+  suite('createCompletionRequestMessages', () => {
+    test('should create messages with correct format and content', async () => {
+      const mockRootUri = { fsPath: '/test/workspace' } as vscode.Uri
+
+      // Mock the filesystem operations without specific type dependencies
+      const mockFileTree = [
+        {
+          name: 'file1.ts',
+          path: '/test/workspace/file1.ts',
+          isDirectory: false,
+          type: 'file'
+        },
+        {
+          name: 'folder1',
+          path: '/test/workspace/folder1',
+          isDirectory: true,
+          type: 'directory'
+        }
+      ]
+
+      const mockFormattedTree = '├── file1.ts\n└── folder1/'
+
+      // Stub the filesystem operations with any type to avoid type conflicts
+      const getWorkspaceFileTreeStub = sandbox
+        .stub(fileOperations, 'getWorkspaceFileTree')
+        .resolves(mockFileTree as any)
+      const formatFileTreeStub = sandbox
+        .stub(fileOperations, 'formatFileTree')
+        .returns(mockFormattedTree)
+
+      const userPrompt = 'Find all typescript files'
+      const result = await createCompletionRequestMessages(userPrompt, mockRootUri)
+
+      // Verify the filesystem operations were called correctly
+      sinon.assert.calledOnce(getWorkspaceFileTreeStub)
+      sinon.assert.calledWith(getWorkspaceFileTreeStub, mockRootUri)
+      sinon.assert.calledOnce(formatFileTreeStub)
+      sinon.assert.calledWith(formatFileTreeStub, mockRootUri.fsPath, mockFileTree as any)
+
+      // Check the result messages
+      assert.strictEqual(result.length, FEW_SHOT_EXAMPLES.length + 2) // system + examples + user
+      assert.strictEqual(result[0].role, 'system')
+      assert.strictEqual(result[0].content, SYSTEM_PROMPT)
+
+      // The last message should be the user message
+      const userMessage = result[result.length - 1]
+      assert.strictEqual(userMessage.role, 'user')
+      assert.ok(userMessage.content.includes('<file-tree>'))
+      assert.ok(userMessage.content.includes(mockRootUri.fsPath))
+      assert.ok(userMessage.content.includes(mockFormattedTree))
+      assert.ok(userMessage.content.includes(`User request: ${userPrompt}`))
+    })
+  })
+
+  suite('parseLLMResponse', () => {
+    test('should correctly parse valid JSON array response', () => {
+      const response = '["file1.ts", "src/file2.ts"]'
+      const result = parseLLMResponse(response)
+
+      assert.deepStrictEqual(result, ['file1.ts', 'src/file2.ts'])
+    })
+
+    test('should correctly parse response with files property', () => {
+      const response = '{"files": ["file1.ts", "src/file2.ts"]}'
+      const result = parseLLMResponse(response)
+
+      assert.deepStrictEqual(result, ['file1.ts', 'src/file2.ts'])
+    })
+
+    test('should return empty array for empty response', () => {
+      const result = parseLLMResponse('')
+      assert.deepStrictEqual(result, [])
+    })
+
+    test('should return empty array for response not matching expected format', () => {
+      const response = '{"notFiles": ["file1.ts"]}'
+      const result = parseLLMResponse(response)
+
+      assert.deepStrictEqual(result, [])
+    })
+  })
+})