fix grounding dataset concat

docs/source/Instruction/支持的模型和数据集.md

@@ -633,7 +633,7 @@
 |[LLM-Research/Llama-4-Scout-17B-16E-Instruct](https://modelscope.cn/models/LLM-Research/Llama-4-Scout-17B-16E-Instruct)|llama4|llama4|transformers>=4.51|✘|vision|[meta-llama/Llama-4-Scout-17B-16E-Instruct](https://huggingface.co/meta-llama/Llama-4-Scout-17B-16E-Instruct)|
 |[LLM-Research/Llama-4-Maverick-17B-128E-Instruct-FP8](https://modelscope.cn/models/LLM-Research/Llama-4-Maverick-17B-128E-Instruct-FP8)|llama4|llama4|transformers>=4.51|✘|vision|[meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8](https://huggingface.co/meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8)|
 |[LLM-Research/Llama-4-Maverick-17B-128E-Instruct](https://modelscope.cn/models/LLM-Research/Llama-4-Maverick-17B-128E-Instruct)|llama4|llama4|transformers>=4.51|✘|vision|[meta-llama/Llama-4-Maverick-17B-128E-Instruct](https://huggingface.co/meta-llama/Llama-4-Maverick-17B-128E-Instruct)|
-|[ICTNLP/Llama-3.1-8B-Omni](https://modelscope.cn/models/ICTNLP/Llama-3.1-8B-Omni)|llama3_1_omni|llama3_1_omni|whisper, openai-whisper|✘|audio|[ICTNLP/Llama-3.1-8B-Omni](https://huggingface.co/ICTNLP/Llama-3.1-8B-Omni)|
+|[ICTNLP/Llama-3.1-8B-Omni](https://modelscope.cn/models/ICTNLP/Llama-3.1-8B-Omni)|llama3_1_omni|llama3_1_omni|openai-whisper|✘|audio|[ICTNLP/Llama-3.1-8B-Omni](https://huggingface.co/ICTNLP/Llama-3.1-8B-Omni)|
 |[llava-hf/llava-1.5-7b-hf](https://modelscope.cn/models/llava-hf/llava-1.5-7b-hf)|llava1_5_hf|llava1_5_hf|transformers>=4.36|✘|vision|[llava-hf/llava-1.5-7b-hf](https://huggingface.co/llava-hf/llava-1.5-7b-hf)|
 |[llava-hf/llava-1.5-13b-hf](https://modelscope.cn/models/llava-hf/llava-1.5-13b-hf)|llava1_5_hf|llava1_5_hf|transformers>=4.36|✘|vision|[llava-hf/llava-1.5-13b-hf](https://huggingface.co/llava-hf/llava-1.5-13b-hf)|
 |[llava-hf/llava-v1.6-mistral-7b-hf](https://modelscope.cn/models/llava-hf/llava-v1.6-mistral-7b-hf)|llava1_6_mistral_hf|llava1_6_mistral_hf|transformers>=4.39|✘|vision|[llava-hf/llava-v1.6-mistral-7b-hf](https://huggingface.co/llava-hf/llava-v1.6-mistral-7b-hf)|

docs/source_en/Instruction/Supported-models-and-datasets.md

@@ -633,7 +633,7 @@ The table below introduces the models integrated with ms-swift:
 |[LLM-Research/Llama-4-Scout-17B-16E-Instruct](https://modelscope.cn/models/LLM-Research/Llama-4-Scout-17B-16E-Instruct)|llama4|llama4|transformers>=4.51|✘|vision|[meta-llama/Llama-4-Scout-17B-16E-Instruct](https://huggingface.co/meta-llama/Llama-4-Scout-17B-16E-Instruct)|
 |[LLM-Research/Llama-4-Maverick-17B-128E-Instruct-FP8](https://modelscope.cn/models/LLM-Research/Llama-4-Maverick-17B-128E-Instruct-FP8)|llama4|llama4|transformers>=4.51|✘|vision|[meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8](https://huggingface.co/meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8)|
 |[LLM-Research/Llama-4-Maverick-17B-128E-Instruct](https://modelscope.cn/models/LLM-Research/Llama-4-Maverick-17B-128E-Instruct)|llama4|llama4|transformers>=4.51|✘|vision|[meta-llama/Llama-4-Maverick-17B-128E-Instruct](https://huggingface.co/meta-llama/Llama-4-Maverick-17B-128E-Instruct)|
-|[ICTNLP/Llama-3.1-8B-Omni](https://modelscope.cn/models/ICTNLP/Llama-3.1-8B-Omni)|llama3_1_omni|llama3_1_omni|whisper, openai-whisper|✘|audio|[ICTNLP/Llama-3.1-8B-Omni](https://huggingface.co/ICTNLP/Llama-3.1-8B-Omni)|
+|[ICTNLP/Llama-3.1-8B-Omni](https://modelscope.cn/models/ICTNLP/Llama-3.1-8B-Omni)|llama3_1_omni|llama3_1_omni|openai-whisper|✘|audio|[ICTNLP/Llama-3.1-8B-Omni](https://huggingface.co/ICTNLP/Llama-3.1-8B-Omni)|
 |[llava-hf/llava-1.5-7b-hf](https://modelscope.cn/models/llava-hf/llava-1.5-7b-hf)|llava1_5_hf|llava1_5_hf|transformers>=4.36|✘|vision|[llava-hf/llava-1.5-7b-hf](https://huggingface.co/llava-hf/llava-1.5-7b-hf)|
 |[llava-hf/llava-1.5-13b-hf](https://modelscope.cn/models/llava-hf/llava-1.5-13b-hf)|llava1_5_hf|llava1_5_hf|transformers>=4.36|✘|vision|[llava-hf/llava-1.5-13b-hf](https://huggingface.co/llava-hf/llava-1.5-13b-hf)|
 |[llava-hf/llava-v1.6-mistral-7b-hf](https://modelscope.cn/models/llava-hf/llava-v1.6-mistral-7b-hf)|llava1_6_mistral_hf|llava1_6_mistral_hf|transformers>=4.39|✘|vision|[llava-hf/llava-v1.6-mistral-7b-hf](https://huggingface.co/llava-hf/llava-v1.6-mistral-7b-hf)|

swift/llm/dataset/preprocessor/core.py

@@ -8,7 +8,7 @@
 from datasets import Dataset as HfDataset
 from datasets import Image
 from datasets import IterableDataset as HfIterableDataset
-from datasets import Value
+from datasets import Sequence, Value
 
 from swift.llm import history_to_messages
 from swift.utils import get_logger
@@ -142,10 +142,13 @@ def _check_objects(row):
         objects = row.get('objects')
         if objects is None:
             return
-        for k in list(objects.keys()):
-            if k not in {'bbox', 'ref', 'bbox_type', 'image_id'}:
-                objects.pop(k)
-        bbox = objects['bbox']
+        new_objects = {}
+        # Ensure the order
+        for k in ['ref', 'bbox', 'bbox_type', 'image_id']:
+            if k in objects.keys():
+                new_objects[k] = objects[k]
+        row['objects'] = new_objects
+        bbox = new_objects['bbox']
 
         # check bbox
         for box in bbox:
@@ -254,11 +257,12 @@ def _patch_arrow_writer():
         def _new_init(self, schema=None, features=None, *args, **kwargs):
 
             if features is not None:
-                features['messages'] = [{
-                    'role': Value(dtype='string', id=None),
-                    'content': Value(dtype='string', id=None)
-                }]
-                features['images'] = [{'bytes': Value(dtype='binary', id=None), 'path': Value(dtype='string', id=None)}]
+                features['messages'] = [{'role': Value(dtype='string'), 'content': Value(dtype='string')}]
+                features['images'] = [{'bytes': Value(dtype='binary'), 'path': Value(dtype='string')}]
+                features['objects'] = {
+                    'ref': Sequence(feature=Value(dtype='string'), length=-1),
+                    'bbox': Sequence(feature=Sequence(feature=Value(dtype='float64'), length=-1), length=-1)
+                }
             ArrowWriter.__origin_init__(self, schema, features, *args, **kwargs)
 
         ArrowWriter.__origin_init__ = ArrowWriter.__init__
@@ -287,7 +291,7 @@ def __call__(
         if batch_size is None:
             batch_size = 1000 if isinstance(dataset, HfDataset) else 16
         if self.dataset_sample is not None:
-            dataset = sample_dataset(dataset, self.dataset_sample, self.random_state)
+            dataset = sample_dataset(dataset, self.dataset_sample, True, self.random_state)
 
         map_kwargs = {'batched': True, 'batch_size': batch_size}
         if isinstance(dataset, HfDataset):

swift/llm/infer/infer.py

@@ -158,7 +158,7 @@ def _prepare_val_dataset(self) -> HfDataset:
         else:
             _, val_dataset = load_dataset(args.dataset, split_dataset_ratio=args.split_dataset_ratio, **dataset_kwargs)
         assert val_dataset is not None
-        val_dataset = sample_dataset(val_dataset, args.val_dataset_sample, self.random_state)
+        val_dataset = sample_dataset(val_dataset, args.val_dataset_sample, args.dataset_shuffle, self.random_state)
         return val_dataset
 
     def _calc_metric(self):

swift/llm/model/model/llama.py

@@ -279,7 +279,7 @@ def get_model_tokenizer_omnli(model_dir: str,
         get_model_tokenizer_omnli,
         architectures=['OmniSpeech2SLlamaForCausalLM'],
         model_arch=ModelArch.llama3_1_omni,
-        requires=['whisper', 'openai-whisper'],
+        requires=['openai-whisper'],
         tags=['audio'],
     ))