Add model category <> messages check (#1335)

* Verify model category matches the messages

* Add vision chat

* Fixes

Author: EricLBuehler

.cargo/config.toml

@@ -4,7 +4,7 @@ rustflags = ["-C", "target-cpu=native"]
 [target.aarch64-apple-darwin]
 rustflags = [
   "-C", "target-cpu=native",
-  "-C", "target-feature=+aes,+sha2,+fp16,+fp16fml",
+  "-C", "target-feature=+aes,+sha2,+fp16",
 ]
 
 [target.x86_64-apple-darwin]

mistralrs-core/src/engine/add_request.rs

@@ -3,7 +3,7 @@ use crate::{
     request::{DetokenizationRequest, NormalRequest, TokenizationRequest},
     sequence::SeqStepType,
     tools::{ToolCallingMatcher, ToolChoice},
-    RequestMessage, Response,
+    ModelCategory, RequestMessage, Response,
 };
 use candle_core::Tensor;
 use either::Either;
@@ -88,6 +88,32 @@ impl Engine {
             return;
         }
 
+        // Verify the model's category matches the messages received.
+        match (
+            get_mut_arcmutex!(self.pipeline).category(),
+            &request.messages,
+        ) {
+            (
+                ModelCategory::Text | ModelCategory::Vision { .. },
+                RequestMessage::Chat { .. }
+                | RequestMessage::VisionChat { .. }
+                | RequestMessage::Completion { .. }
+                | RequestMessage::CompletionTokens(_),
+            ) => (),
+            (ModelCategory::Diffusion, RequestMessage::ImageGeneration { .. }) => (),
+            (ModelCategory::Speech, RequestMessage::SpeechGeneration { .. }) => (),
+            _ => {
+                request
+                    .response
+                    .send(Response::ValidationError(
+                        "Received a request incompatible for this model's category.".into(),
+                    ))
+                    .await
+                    .expect("Expected receiver.");
+                return;
+            }
+        }
+
         let images = match request.messages {
             RequestMessage::VisionChat {
                 ref images,