Merged in shibu/sphinx/add_stemmer (pull request #214)

Author: birkenfeld

CHANGES

@@ -18,6 +18,8 @@ New features
 
 * Added ``sphinx.ext.napoleon`` extension for NumPy and Google style docstring
   support.
+* PR#214: Added stemming support for 14 languages, so that the built-in document
+  search can now handle these.  Thanks to Shibukawa Yoshiki.
 * PR#202: Allow "." and "~" prefixed references in ``:param:`` doc fields
   for Python.
 * PR#184: Add :confval:`autodoc_mock_imports`, allowing to mock imports of

doc/config.rst

@@ -748,10 +748,37 @@ that use Sphinx' HTMLWriter class.
 
    Support is present for these languages:
 
+   * ``da`` -- Danish
+   * ``nl`` -- Dutch
    * ``en`` -- English
+   * ``fi`` -- Finnish
+   * ``fr`` -- French
+   * ``de`` -- German
+   * ``hu`` -- Hungarian
+   * ``it`` -- Italian
    * ``ja`` -- Japanese
+   * ``no`` -- Norwegian
+   * ``pr`` -- Portuguese
+   * ``ro`` -- Romanian
+   * ``ru`` -- Russian
+   * ``es`` -- Spanish
+   * ``sv`` -- Swedish
+   * ``tr`` -- Turkish
+
+   .. admonition:: Accelerate build speed
+
+      Each language (except Japanese) provides its own stemming algorithm.
+      Sphinx uses a Python implementation by default.  You can use a C
+      implementation to accelerate building the index file.
+
+      * `PorterStemmer <https://pypi.python.org/pypi/PorterStemmer>`_ (``en``)
+      * `PyStemmer <https://pypi.python.org/pypi/PyStemmer>`_ (all languages)
 
    .. versionadded:: 1.1
+      With support for ``en`` and ``ja``.
+
+   .. versionchanged:: 1.3
+      Added additional languages.
 
 .. confval:: html_search_options
 

doc/devguide.rst

@@ -243,3 +243,15 @@ Debugging Tips
 
 * Set the debugging options in the `Docutils configuration file
   <http://docutils.sourceforge.net/docs/user/config.html>`_.
+
+* JavaScript stemming algorithms in `sphinx/search/*.py` (except `en.py`) are
+  genereted by
+  `modified snowballcode generator <https://github.com/shibukawa/snowball>`_.
+  Generated `JSX <http://jsx.github.io/>`_ files are
+  in `this repository <https://github.com/shibukawa/snowball-stemmer.jsx>`_.
+  You can get resulting JavaScript files by the following command:
+
+  .. code-block:: bash
+
+     $ npm install
+     $ node_modules/.bin/grunt build # -> dest/*.global.js

setup.py

@@ -45,7 +45,7 @@
     print('ERROR: Sphinx requires at least Python 2.6 or 3.2 to run.')
     sys.exit(1)
 
-requires = ['Pygments>=1.2', 'docutils>=0.10']
+requires = ['Pygments>=1.2', 'docutils>=0.10', 'snowballstemmer>=1.1']
 
 if (3, 0) <= sys.version_info < (3, 3):
     requires.append('Jinja2>=2.3,<2.7')

sphinx/builders/html.py

@@ -32,7 +32,6 @@
 from sphinx.util.nodes import inline_all_toctrees
 from sphinx.util.matching import patmatch, compile_matchers
 from sphinx.util.pycompat import b
-from sphinx.errors import SphinxError
 from sphinx.locale import _
 from sphinx.search import js_index
 from sphinx.theming import Theme
@@ -818,7 +817,7 @@ def dump_inventory(self):
         self.info('done')
 
     def dump_search_index(self):
-        self.info(bold('dumping search index... '), nonl=True)
+        self.info(bold('dumping search index in %s ... ' % self.indexer.label()), nonl=True)
         self.indexer.prune(self.env.all_docs)
         searchindexfn = path.join(self.outdir, self.searchindex_filename)
         # first write to a temporary file, so that if dumping fails,

sphinx/quickstart.py

@@ -231,10 +231,23 @@
 # This is the file name suffix for HTML files (e.g. ".xhtml").
 #html_file_suffix = None
 
+# Language to be used for generating the HTML full-text search index.
+# Sphinx supports the following languages:
+#   'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja'
+#   'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr'
+#html_search_language = 'en'
+
+# A dictionary with options for the search language support, empty by default.
+# Now only 'ja' uses this config value
+#html_search_options = {'type': 'default'}
+
+# The name of a javascript file (relative to the configuration directory) that
+# implements a search results scorer. If empty, the default will be used.
+#html_search_scorer = 'scorer.js'
+
 # Output file base name for HTML help builder.
 htmlhelp_basename = '%(project_fn)sdoc'
 
-
 # -- Options for LaTeX output ---------------------------------------------
 
 latex_elements = {

sphinx/search/__init__.py

@@ -9,12 +9,14 @@
     :license: BSD, see LICENSE for details.
 """
 from __future__ import with_statement
+
 import re
 import cPickle as pickle
 
 from docutils.nodes import raw, comment, title, Text, NodeVisitor, SkipNode
 
 from sphinx.util import jsdump, rpartition
+from sphinx.search.en import SearchEnglish
 
 
 class SearchLanguage(object):
@@ -40,6 +42,7 @@ class SearchLanguage(object):
        type, before searching index. Default implementation does nothing.
     """
     lang = None
+    language_name = None
     stopwords = set()
     js_stemmer_code = """
 /**
@@ -89,16 +92,42 @@ def word_filter(self, word):
         Return true if the target word should be registered in the search index.
         This method is called after stemming.
         """
-        return not (((len(word) < 3) and (12353 < ord(word[0]) < 12436)) or
+        return len(word) == 0 or not (((len(word) < 3) and (12353 < ord(word[0]) < 12436)) or
             (ord(word[0]) < 256 and (len(word) < 3 or word in self.stopwords or
                                      word.isdigit())))
 
 
-from sphinx.search import en, ja
+def parse_stop_word(source):
+    """
+    parse snowball style word list like this:
+
+    * http://snowball.tartarus.org/algorithms/finnish/stop.txt
+    """
+    result = set()
+    for line in source.splitlines():
+        line = line.split('|')[0] # remove comment
+        result.update(line.split())
+    return result
 
+
+# maps language name to module.class or directly a class
 languages = {
-    'en': en.SearchEnglish,
-    'ja': ja.SearchJapanese,
+    'da': 'sphinx.search.da.SearchDanish',
+    'de': 'sphinx.search.de.SearchGerman',
+    'en': SearchEnglish,
+    'es': 'sphinx.search.es.SearchSpanish',
+    'fi': 'sphinx.search.fi.SearchFinnish',
+    'fr': 'sphinx.search.fr.SearchFrench',
+    'hu': 'sphinx.search.hu.SearchHungarian',
+    'it': 'sphinx.search.it.SearchItalian',
+    'ja': 'sphinx.search.ja.SearchJapanese',
+    'nl': 'sphinx.search.nl.SearchDutch',
+    'no': 'sphinx.search.no.SearchNorwegian',
+    'pt': 'sphinx.search.pt.SearchPortuguese',
+    'ro': 'sphinx.search.ro.SearchRomanian',
+    'ru': 'sphinx.search.ru.SearchRussian',
+    'sv': 'sphinx.search.sv.SearchSwedish',
+    'tr': 'sphinx.search.tr.SearchTurkish',
 }
 
 
@@ -185,7 +214,17 @@ def __init__(self, env, lang, options, scoring):
         # objtype index -> (domain, type, objname (localized))
         self._objnames = {}
         # add language-specific SearchLanguage instance
-        self.lang = languages[lang](options)
+        lang_class = languages.get(lang)
+        if lang_class is None:
+            self.lang = SearchEnglish(options)
+        elif isinstance(lang_class, str):
+            module, classname = lang_class.rsplit('.', 1)
+            lang_class = getattr(__import__(module, None, None, [classname]),
+                                 classname)
+            self.lang = lang_class(options)
+        else:
+            # it's directly a class (e.g. added by app.add_search_language)
+            self.lang = lang_class(options)
 
         if scoring:
             with open(scoring, 'rb') as fp:
@@ -286,6 +325,9 @@ def freeze(self):
                     objects=objects, objtypes=objtypes, objnames=objnames,
                     titleterms=title_terms, envversion=self.env.version)
 
+    def label(self):
+        return "%s (code: %s)" % (self.lang.language_name, self.lang.lang)
+
     def prune(self, filenames):
         """Remove data for all filenames not in the list."""
         new_titles = {}