feat(puterai): add mistral

Author: KernelDeimos

package-lock.json

@@ -13,6 +13,7 @@
     "@heyputer/kv.js": "^0.1.3",
     "@heyputer/multest": "^0.0.2",
     "@heyputer/puter-js-common": "^1.0.0",
+    "@mistralai/mistralai": "^1.0.3",
     "@opentelemetry/api": "^1.4.1",
     "@opentelemetry/auto-instrumentations-node": "^0.43.0",
     "@opentelemetry/exporter-trace-otlp-grpc": "^0.40.0",

src/backend/package.json

@@ -0,0 +1,83 @@
+const { PassThrough } = require("stream");
+const BaseService = require("../../services/BaseService");
+const { TypedValue } = require("../../services/drivers/meta/Runtime");
+const { nou } = require("../../util/langutil");
+
+class MistralAIService extends BaseService {
+    static MODULES = {
+        '@mistralai/mistralai': require('@mistralai/mistralai'),
+    }
+    async _init () {
+        const require = this.require;
+        const { Mistral } = require('@mistralai/mistralai');
+        this.client = new Mistral({
+            apiKey: this.config.apiKey,
+        });
+    }
+    static IMPLEMENTS = {
+        'puter-chat-completion': {
+            async list () {
+                // They send: { "object": "list", data }
+                const funny_wrapper = await this.client.models.list();
+                return funny_wrapper.data;
+            },
+            async complete ({ messages, stream, model }) {
+
+                for ( let i = 0; i < messages.length; i++ ) {
+                    const message = messages[i];
+                    if ( ! message.role ) message.role = 'user';
+                }
+
+                if ( stream ) {
+                    const stream = new PassThrough();
+                    const retval = new TypedValue({
+                        $: 'stream',
+                        content_type: 'application/x-ndjson',
+                        chunked: true,
+                    }, stream);
+                    const completion = await this.client.chat.stream({
+                        model: model ?? 'mistral-large-latest',
+                        messages,
+                    });
+                    (async () => {
+                        for await ( let chunk of completion ) {
+                            // just because Mistral wants to be different
+                            chunk = chunk.data;
+
+                            if ( chunk.choices.length < 1 ) continue;
+                            if ( chunk.choices[0].finish_reason ) {
+                                stream.end();
+                                break;
+                            }
+                            if ( nou(chunk.choices[0].delta.content) ) continue;
+                            const str = JSON.stringify({
+                                text: chunk.choices[0].delta.content
+                            });
+                            stream.write(str + '\n');
+                        }
+                    })();
+                    return retval;
+                }
+
+                try {
+                    const completion = await this.client.chat.complete({
+                        model: model ?? 'mistral-large-latest',
+                        messages,
+                    });
+                    // Expected case when mistralai/client-ts#23 is fixed
+                    return completion.choices[0];
+                } catch (e) {
+                    if ( ! e?.rawValue?.choices[0] ) {
+                        throw e;
+                    }
+                    // The SDK attempts to validate APIs response and throws
+                    // an exception, even if the response was successful
+                    // https://github.com/mistralai/client-ts/issues/23
+                    return e.rawValue.choices[0];
+                }
+            }
+        }
+    }
+}
+
+module.exports = { MistralAIService };

src/backend/src/modules/puterai/MistralAIService.js

@@ -38,6 +38,11 @@ class PuterAIModule extends AdvancedBase {
             const { TogetherAIService } = require('./TogetherAIService');
             services.registerService('together-ai', TogetherAIService);
         }
+        
+        if ( !! config?.services?.['mistral'] ) {
+            const { MistralAIService } = require('./MistralAIService');
+            services.registerService('mistral', MistralAIService);
+        }
     }
 }
 

src/backend/src/modules/puterai/PuterAIModule.js

@@ -48,7 +48,6 @@ class TogetherAIService extends BaseService {
                     }, stream);
                     (async () => {
                         for await ( const chunk of completion ) {
-                            console.log('IT IS THIS STRING', chunk);
                             if ( chunk.choices.length < 1 ) continue;
                             if ( chunk.choices[0].finish_reason ) {
                                 stream.end();

src/backend/src/modules/puterai/TogetherAIService.js

@@ -226,6 +226,9 @@ class AI{
         if( options.model === 'claude-3-5-sonnet' || options.model === 'claude'){
             options.model = 'claude-3-5-sonnet-20240620';
         }
+        if ( options.model === 'mistral' ) {
+            options.model = 'mistral-large-latest';
+        }
 
         // map model to the appropriate driver
         if (!options.model || options.model === 'gpt-4o' || options.model === 'gpt-4o-mini') {
@@ -234,6 +237,8 @@ class AI{
             driver = 'claude';
         }else if(options.model === 'meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo' || options.model === 'meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo' || options.model === 'meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo' || options.model === `google/gemma-2-27b-it`){
             driver = 'together-ai';
+        }else if(options.model === 'mistral-large-latest' || options.model === 'codestral-latest'){
+            driver = 'mistral';
         }
 
         // stream flag from settings