Skip to content

Add support for OpenVINO text2text generation models #3101

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 2 commits into
base: main
Choose a base branch
from
Draft

Add support for OpenVINO text2text generation models #3101

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
027a6da
Add support for OVModelForSeq2SeqLM
nikita-savelyevv Jul 1, 2025
b6d6a98
Add test
nikita-savelyevv Jul 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 5 additions & 11 deletions lm_eval/models/optimum_lm.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,8 @@ def __init__(
**kwargs,
) -> None:
if "backend" in kwargs:
# optimum currently only supports causal models
assert kwargs["backend"] == "causal", (
"Currently, only OVModelForCausalLM is supported."
assert kwargs["backend"] in ["causal", "seq2seq"], (
"Currently, only OVModelForCausalLM or OVModelForSeq2SeqLM are supported."
)

self.openvino_device = device
Expand All @@ -54,7 +53,7 @@ def _create_model(
"package `optimum` is not installed. Please install it via `pip install optimum[openvino]`"
)
else:
from optimum.intel.openvino import OVModelForCausalLM
from optimum.intel.openvino import OVModelForCausalLM, OVModelForSeq2SeqLM

model_kwargs = kwargs if kwargs else {}
if "ov_config" in model_kwargs:
Expand All @@ -76,17 +75,12 @@ def _create_model(
model_kwargs["ov_config"]["MODEL_DISTRIBUTION_POLICY"] = (
"PIPELINE_PARALLEL"
)
model_file = Path(pretrained) / "openvino_model.xml"
if model_file.exists():
export = False
else:
export = True

self._model = OVModelForCausalLM.from_pretrained(
model_cls = OVModelForCausalLM if self.backend == "causal" else OVModelForSeq2SeqLM
self._model = model_cls.from_pretrained(
pretrained,
revision=revision,
trust_remote_code=trust_remote_code,
export=export,
device=self.openvino_device.upper(),
**model_kwargs,
)
20 changes: 11 additions & 9 deletions tests/models/test_openvino.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,31 +3,33 @@
from pathlib import Path

import pytest
from optimum.intel import OVModelForCausalLM
from optimum.intel import OVModelForCausalLM, OVModelForSeq2SeqLM
from transformers import AutoTokenizer

from lm_eval import evaluator
from lm_eval.api.registry import get_model


SUPPORTED_ARCHITECTURES_TASKS = {
"facebook/opt-125m": "lambada_openai",
"hf-internal-testing/tiny-random-gpt2": "wikitext",
}
SUPPORTED_ARCHITECTURES_TASKS = [
("causal", "facebook/opt-125m", "lambada_openai",),
("causal", "hf-internal-testing/tiny-random-gpt2", "wikitext",),
("seq2seq", "hf-internal-testing/tiny-random-t5", "sst2",),
]


@pytest.mark.parametrize("model_id,task", SUPPORTED_ARCHITECTURES_TASKS.items())
def test_evaluator(model_id, task):
@pytest.mark.parametrize("backend,model_id,task", SUPPORTED_ARCHITECTURES_TASKS)
def test_evaluator(backend, model_id, task):
with tempfile.TemporaryDirectory() as tmpdirname:
model = OVModelForCausalLM.from_pretrained(
model_cls = OVModelForCausalLM if backend == "causal" else OVModelForSeq2SeqLM
model = model_cls.from_pretrained(
model_id, export=True, use_cache=True
)
model.save_pretrained(tmpdirname)
tokenizer = AutoTokenizer.from_pretrained(model_id)
tokenizer.save_pretrained(tmpdirname)

lm = get_model("openvino").create_from_arg_string(
f"pretrained={tmpdirname}",
f"pretrained={tmpdirname},backend={backend}",
{
"batch_size": 1,
"device": "cpu",
Expand Down