Replace pybtex with bibtexparser (#4361)

* partial migrate

* change dep

* recover TODO

* recover some comments

* remove pybtex check

* avoid import pybtext for now

* (need double check) migrate is_valid_bibtex

* lower version of bibtexparser seem to throw error for test_provenance

* uv pip compile pyproject.toml -o requirements.txt

* mark implicit text mode as error

* migrate cif bibtex str getter

---------

Co-authored-by: Shyue Ping Ong <[email protected]>

Author: DanielYang59

pyproject.toml

@@ -61,7 +61,7 @@ dependencies = [
     "palettable>=3.3.3",
     "pandas>=2",
     "plotly>=5.0.0",
-    "pybtex>=0.24.0",
+    "bibtexparser>=1.4.0",
     "requests>=2.32",
     "ruamel.yaml>=0.17.0",
     "scipy>=1.13.0",
@@ -260,6 +260,7 @@ filterwarnings = [
     # NOTE: the LAST matching option would be used
     "ignore::UserWarning", # Ignore UserWarning
     "error:We strongly encourage explicit `encoding`:EncodingWarning", # Mark `zopen` EncodingWarning as error
+    "error:We strongly discourage using implicit binary/text:FutureWarning", # Mark `zopen` FutureWarning as error
     # TODO: remove the following filter once `monty.io` dropped custom EncodingWarning
     "error:We strongly encourage explicit `encoding`:monty.io.EncodingWarning",
     # TODO: pybtex (perhaps some others) emits the following warnings

requirements.txt

@@ -1,88 +1,87 @@
 # This file was autogenerated by uv via the following command:
 #    uv pip compile pyproject.toml -o requirements.txt
+bibtexparser==1.4.3
+    # via pymatgen (pyproject.toml)
 certifi==2024.8.30
-# via requests
+    # via requests
 charset-normalizer==3.4.0
-# via requests
+    # via requests
 contourpy==1.3.1
-# via matplotlib
+    # via matplotlib
 cycler==0.12.1
-# via matplotlib
+    # via matplotlib
 fonttools==4.55.0
-# via matplotlib
+    # via matplotlib
 idna==3.10
-# via requests
+    # via requests
 joblib==1.4.2
-# via pymatgen (pyproject.toml)
+    # via pymatgen (pyproject.toml)
 kiwisolver==1.4.7
-# via matplotlib
-latexcodec==3.0.0
-# via pybtex
+    # via matplotlib
 matplotlib==3.9.2
-# via pymatgen (pyproject.toml)
+    # via pymatgen (pyproject.toml)
 monty==2025.1.9
-# via pymatgen (pyproject.toml)
+    # via pymatgen (pyproject.toml)
 mpmath==1.3.0
-# via sympy
+    # via sympy
 networkx==3.4.2
-# via pymatgen (pyproject.toml)
+    # via pymatgen (pyproject.toml)
 numpy==2.0.0
-# via
-#   pymatgen (pyproject.toml)
-#   contourpy
-#   matplotlib
-#   pandas
-#   scipy
-#   spglib
+    # via
+    #   pymatgen (pyproject.toml)
+    #   contourpy
+    #   matplotlib
+    #   monty
+    #   pandas
+    #   scipy
+    #   spglib
 packaging==24.2
-# via
-#   matplotlib
-#   plotly
+    # via
+    #   matplotlib
+    #   plotly
 palettable==3.3.3
-# via pymatgen (pyproject.toml)
+    # via pymatgen (pyproject.toml)
 pandas==2.2.3
-# via pymatgen (pyproject.toml)
+    # via pymatgen (pyproject.toml)
 pillow==11.0.0
-# via matplotlib
+    # via matplotlib
 plotly==5.24.1
-# via pymatgen (pyproject.toml)
-pybtex==0.24.0
-# via pymatgen (pyproject.toml)
+    # via pymatgen (pyproject.toml)
 pyparsing==3.2.0
-# via matplotlib
+    # via
+    #   bibtexparser
+    #   matplotlib
 python-dateutil==2.9.0.post0
-# via
-#   matplotlib
-#   pandas
+    # via
+    #   matplotlib
+    #   pandas
 pytz==2024.2
-# via pandas
-pyyaml==6.0.2
-# via pybtex
+    # via pandas
 requests==2.32.3
-# via pymatgen (pyproject.toml)
+    # via pymatgen (pyproject.toml)
 ruamel-yaml==0.18.10
-# via pymatgen (pyproject.toml)
+    # via
+    #   pymatgen (pyproject.toml)
+    #   monty
 ruamel-yaml-clib==0.2.12
-# via ruamel-yaml
+    # via ruamel-yaml
 scipy==1.14.1
-# via pymatgen (pyproject.toml)
+    # via pymatgen (pyproject.toml)
 six==1.16.0
-# via
-#   pybtex
-#   python-dateutil
+    # via python-dateutil
 spglib==2.5.0
-# via pymatgen (pyproject.toml)
+    # via pymatgen (pyproject.toml)
 sympy==1.13.1
-# via pymatgen (pyproject.toml)
+    # via pymatgen (pyproject.toml)
 tabulate==0.9.0
-# via pymatgen (pyproject.toml)
+    # via pymatgen (pyproject.toml)
 tenacity==9.0.0
-# via plotly
+    # via plotly
 tqdm==4.67.1
-# via pymatgen (pyproject.toml)
+    # via pymatgen (pyproject.toml)
 tzdata==2024.2
-# via pandas
+    # via pandas
 uncertainties==3.2.2
-# via pymatgen (pyproject.toml)
+    # via pymatgen (pyproject.toml)
 urllib3==2.2.3
-# via requests
+    # via requests

src/pymatgen/io/cif.py

@@ -1349,16 +1349,15 @@ def get_structures(self, *args, **kwargs) -> list[Structure]:
     def get_bibtex_string(self) -> str:
         """Get BibTeX reference from CIF file.
 
-        Args:
-            data:
+        TODO:
+            - parse '_publ_section_references' when it exists?
+            - CIF specification supports multiple citations.
 
         Returns:
             BibTeX string.
         """
-        try:
-            from pybtex.database import BibliographyData, Entry
-        except ImportError:
-            raise RuntimeError("Bibliographic data extraction requires pybtex.")
+        from bibtexparser.bibdatabase import BibDatabase
+        from bibtexparser.bwriter import BibTexWriter
 
         bibtex_keys: dict[str, tuple[str, ...]] = {
             "author": ("_publ_author_name", "_citation_author_name"),
@@ -1377,44 +1376,45 @@ def get_bibtex_string(self) -> str:
             "doi": ("_journal_DOI", "_citation_DOI"),
         }
 
-        entries: dict[str, Entry] = {}
-
-        # TODO: parse '_publ_section_references' when it exists?
-        # TODO: CIF specification supports multiple citations.
+        db = BibDatabase()
+        db.entries = []
 
         for idx, data in enumerate(self._cif.data.values()):
             # Convert to lower-case keys, some CIF files inconsistent
             _data = {k.lower(): v for k, v in data.data.items()}
-
-            bibtex_entry = {}
+            entry = {"ENTRYTYPE": "article", "ID": f"cifref{idx}"}
 
             for field, tags in bibtex_keys.items():
                 for tag in tags:
                     if tag in _data:
-                        if isinstance(_data[tag], list):
-                            bibtex_entry[field] = _data[tag][0]
-                        else:
-                            bibtex_entry[field] = _data[tag]
+                        value = _data[tag]
+                        entry[field] = value[0] if isinstance(value, list) else value
+                        break
 
             # Convert to bibtex author format ("and" delimited)
-            if "author" in bibtex_entry:
+            if "author" in entry:
                 # Separate out semicolon authors
-                if isinstance(bibtex_entry["author"], str) and ";" in bibtex_entry["author"]:
-                    bibtex_entry["author"] = bibtex_entry["author"].split(";")
-
-                if isinstance(bibtex_entry["author"], list):
-                    bibtex_entry["author"] = " and ".join(bibtex_entry["author"])
+                if isinstance(entry["author"], str) and ";" in entry["author"]:
+                    entry["author"] = entry["author"].split(";")
+                if isinstance(entry["author"], list):
+                    entry["author"] = " and ".join(entry["author"])
 
             # Convert to bibtex page range format, use empty string if not specified
-            if ("page_first" in bibtex_entry) or ("page_last" in bibtex_entry):
-                bibtex_entry["pages"] = bibtex_entry.get("page_first", "") + "--" + bibtex_entry.get("page_last", "")
-                bibtex_entry.pop("page_first", None)  # and remove page_first, page_list if present
-                bibtex_entry.pop("page_last", None)
+            if "page_first" in entry or "page_last" in entry:
+                entry["pages"] = f"{entry.get('page_first', '')}--{entry.get('page_last', '')}"
+                entry.pop("page_first", None)  # and remove page_first, page_list if present
+                entry.pop("page_last", None)
+
+            db.entries.append(entry)
 
-            # Cite keys are given as cif-reference-idx in order they are found
-            entries[f"cifref{idx}"] = Entry("article", list(bibtex_entry.items()))
+        # NOTE: the following is added to make output consistent with
+        # previous pybtex implementation
+        writer = BibTexWriter()
+        writer.indent = "    "
+        writer.display_order = ("author", "title", "journal", "volume", "year", "pages")
 
-        return BibliographyData(entries).to_string(bib_format="bibtex")
+        # Replace curly brackets with double quotes (skip the first and last one)
+        return re.sub(r"(^\s*\w+\s*=\s*)\{([^{}]*)\}", r'\1"\2"', writer.write(db), flags=re.MULTILINE)
 
     def as_dict(self) -> dict:
         """MSONable dict."""

src/pymatgen/util/provenance.py

@@ -6,19 +6,12 @@
 import re
 import sys
 from datetime import datetime, timezone
-from io import StringIO
 from typing import TYPE_CHECKING, NamedTuple
 
 from monty.json import MontyDecoder, MontyEncoder
 
 from pymatgen.core.structure import Molecule, Structure
 
-try:
-    from pybtex import errors
-    from pybtex.database.input import bibtex
-except ImportError:
-    pybtex = bibtex = errors = None
-
 if TYPE_CHECKING:
     from collections.abc import Sequence
     from typing import Any
@@ -37,21 +30,22 @@
 
 
 def is_valid_bibtex(reference: str) -> bool:
-    """Use pybtex to validate that a reference is in proper BibTeX format.
+    """Validate that a reference is in proper BibTeX format.
 
     Args:
         reference (str): Reference in BibTeX format.
 
     Returns:
         bool: True if reference is valid BibTeX.
     """
-    # str is necessary since pybtex seems to have an issue with unicode.
-    # The filter expression removes all non-ASCII characters.
-    str_io = StringIO(reference.encode("ascii", "ignore").decode("ascii"))
-    parser = bibtex.Parser()
-    errors.set_strict_mode(enable=False)
-    bib_data = parser.parse_stream(str_io)
-    return len(bib_data.entries) > 0
+    from bibtexparser.bparser import BibTexParser
+
+    parser = BibTexParser()
+    try:
+        bib_database = parser.parse(reference)
+        return bool(bib_database.entries)
+    except Exception:
+        return False
 
 
 class HistoryNode(NamedTuple):
@@ -213,14 +207,14 @@ def __init__(
         self.structure = struct_or_mol
 
         # Turn `authors` into list of `Author` objects
-        authors = authors.split(",") if isinstance(authors, str) else authors
-        self.authors = [Author.parse_author(a) for a in authors]
+        _authors: list = authors.split(",") if isinstance(authors, str) else authors
+        self.authors: list[Author] = [Author.parse_author(a) for a in _authors]
 
         # Turn `projects` into list of strings
         projects = projects or []
         self.projects: list[str] = [projects] if isinstance(projects, str) else projects
 
-        # Check that references are valid BibTeX
+        # Check that references are valid BibTeX string
         if not isinstance(references, str):
             raise TypeError("Invalid format for SNL reference! Should be empty string or BibTeX string.")
 

tests/io/test_cif.py

@@ -12,12 +12,6 @@
 from pymatgen.symmetry.structure import SymmetrizedStructure
 from pymatgen.util.testing import TEST_FILES_DIR, VASP_IN_DIR, MatSciTest
 
-try:
-    import pybtex
-except ImportError:
-    pybtex = None
-
-
 MCIF_TEST_DIR = f"{TEST_FILES_DIR}/io/cif/mcif"
 
 
@@ -926,7 +920,7 @@ def test_cif_writer_write_file(self):
 
         # test write_file append mode='a'
         struct2 = Structure.from_file(f"{TEST_FILES_DIR}/cif/Graphite.cif")
-        CifWriter(struct2).write_file(out_path, mode="a")
+        CifWriter(struct2).write_file(out_path, mode="at")
 
         read_structs = CifParser(out_path).parse_structures()
         assert len(read_structs) == 2
@@ -1139,7 +1133,6 @@ def test_write(self):
         cw = CifWriter(s_manual, write_magmoms=True)
         assert str(cw) == cw_manual_oxi_string
 
-    @pytest.mark.skipif(pybtex is None, reason="pybtex not present")
     def test_bibtex(self):
         ref_bibtex_string = """@article{cifref0,
     author = "Blanco, J.A.",

tests/util/test_provenance.py

@@ -45,8 +45,7 @@ def setup_method(self):
             ".com/retrieve/pii/S0927025612006295},\n volume = {68},"
             "\n year = {2013}\n}"
         )
-        repeat = "REPEAT" * 10000
-        self.superlong = f"@misc{{SuperLong,\ntitle = {{{repeat}}}}}"
+        self.superlong = f"@misc{{SuperLong,\ntitle = {{{'REPEAT' * 10000}}}}}"
         self.unicode_title = "@misc{Unicode_Title,\ntitle = {{A \u73ab is a rose}}}"
         self.junk = "This is junk text, not a BibTeX reference"