support selection of possible languages for lingua

Author: svirpioj

docs/CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- support for selection of possible languages for `lingua` language detection
+
 ## [3.2.0] - 2024-08-14
 
 ### Changed

docs/filters/script_and_language_identification_filters.md

@@ -38,7 +38,7 @@ Parameters:
 * `id_method`: language indentification method (`langid`, `lingua`, `cld2`, `fasttext`; default `langid`)
 * `thresholds`: minimum identification confidence score for the segments (a single float or a list of floats per language)
 * `fasttext_model_path`: path for a `fasttext` model (required only for the `fasttext` method; default `null`)
-* `langid_languages`: limit detection to a list of possible languages (valid only for the `langid` method; default `null`)
+* `langid_languages`: limit detection to a list of ISO 639-1 codes for possible languages (valid only for the `langid` and `lingua` methods; default `null`)
 * `cld2_options`: a dictionary of options for the `cld2` method (valid only for the `cld2` method; default `null`)
 * `lingua_mode`: a string specifying whether to use lingua's `high` or `low` accuracy mode
 

opusfilter/filters.py

@@ -305,6 +305,9 @@ def __init__(self, languages=None, id_method='langid', thresholds=None,
         self.cld2_options = None
         self.fasttext_model = None
         self.lingua_detector = None
+        if langid_languages and id_method not in {'langid', 'lingua'}:
+            raise ConfigurationError(
+                "langid_languages option is supported only by the langid and lingua methods")
         if id_method == 'fasttext':
             self.init_fastttext(fasttext_model_path)
         else:
@@ -313,17 +316,13 @@ def __init__(self, languages=None, id_method='langid', thresholds=None,
                                          "path to model was set")
         if id_method == 'langid':
             self.init_langid(langid_languages)
-        else:
-            if langid_languages:
-                raise ConfigurationError(
-                    "langid_languages option is supported only by the method langid")
         if id_method == 'cld2':
             self.cld2_options = cld2_options if cld2_options else {}
         else:
             if cld2_options:
                 raise ConfigurationError("cld2_options is supported only by the method cld2")
         if id_method == "lingua":
-            self.init_lingua(lingua_mode if lingua_mode else 'low')
+            self.init_lingua(lingua_mode if lingua_mode else 'low', langid_languages)
         else:
             if lingua_mode:
                 raise ConfigurationError("lingua_mode is supported only by the method lingua")
@@ -351,11 +350,17 @@ def init_fastttext(self, fasttext_model_path):
             raise
         self.fasttext_model = fasttext.load_model(os.path.join(self.workdir, fasttext_model_path))
 
-    def init_lingua(self, lingua_mode):
+    def init_lingua(self, lingua_mode, languages):
         """Initialize lingua identifier"""
-        from lingua import LanguageDetectorBuilder
-        # TODO: support lingua_languages just like langid_languages
-        from_languages = LanguageDetectorBuilder.from_all_languages()
+        from lingua import LanguageDetectorBuilder, IsoCode639_1
+        if languages:
+            for code in languages:
+                if not hasattr(IsoCode639_1, code.upper()):
+                    raise ConfigurationError(f"Language {code} not supported by lingua")
+            from_languages = LanguageDetectorBuilder.from_iso_codes_639_1(
+                *[getattr(IsoCode639_1, code.upper()) for code in languages])
+        else:
+            from_languages = LanguageDetectorBuilder.from_all_languages()
         if lingua_mode == "high":
             self.lingua_detector = from_languages.with_preloaded_language_models().build()
         elif lingua_mode == "low":

tests/test_lid.py

@@ -149,3 +149,21 @@ def test_accept_high(self):
         pair_expecteds = [True, False]
         for pair_score, pair_expected in zip(pair_scores, pair_expecteds):
             self.assertEqual(model.accept(pair_score), pair_expected)
+
+    def test_limited_languages(self):
+        model = LanguageIDFilter(
+            languages=['en', 'fr'], id_method='lingua', thresholds=[0.4, 0.99],
+            langid_languages=['en', 'fr', 'de', 'fi'])
+        pair_scores = model.score(self.pairs_inputs)
+        pair_expecteds = [True, False]
+        for pair_score, pair_expected in zip(pair_scores, pair_expecteds):
+            self.assertEqual(model.accept(pair_score), pair_expected)
+
+    def test_badly_limited_languages(self):
+        model = LanguageIDFilter(
+            languages=['en', 'fr'], id_method='lingua', thresholds=[0.4, 0.99],
+            langid_languages=['en', 'de', 'fi'])
+        pair_scores = model.score(self.pairs_inputs)
+        pair_expecteds = [False, False]
+        for pair_score, pair_expected in zip(pair_scores, pair_expecteds):
+            self.assertEqual(model.accept(pair_score), pair_expected)