remove solr5555 from search engine code

Remove one layer of abstraction by letting
Solr9PySolrSearchEngine inherit from SearchEngineBase
directly

Author: Bomme

freesound/settings.py

@@ -609,7 +609,6 @@
 SEARCH_FORUM_SORT_DEFAULT = SEARCH_FORUM_SORT_OPTION_THREAD_DATE_FIRST
 
 SEARCH_ENGINE_BACKEND_CLASS = 'utils.search.backends.solr9pysolr.Solr9PySolrSearchEngine'
-SOLR5_BASE_URL = "http://search:8983/solr"
 SOLR9_BASE_URL = "http://search:8983/solr"
 
 SEARCH_ENGINE_SIMILARITY_ANALYZERS = {

freesound/test_settings.py

@@ -41,9 +41,6 @@
 from .logger import LOGGING
 LOGGING['handlers']['stdout']['class'] = 'logging.NullHandler'
 
-SOLR5_SOUNDS_URL = "http://fakehost:8080/fs2/"  # Avoid making accidental queries to "real" search server if running
-SOLR5_FORUM_URL = "http://fakehost:8080/forum/"  # Avoid making accidental requests to "real" search server if running
-SEARCH_ENGINE_BACKEND_CLASS = 'utils.search.backends.solr555pysolr.Solr555PySolrSearchEngine' # Test with our own custom search engine functions
 SIMILARITY_ADDRESS = 'fakehost' # Avoid making accidental requests to "real" similarity server if running
 TAGRECOMMENDATION_ADDRESS = 'fakehost'  # Avoid making accidental requests to "real" tag rec server if running
 

requirements.in

@@ -44,7 +44,7 @@ psycopg2-binary==2.9.6
 PyJWT==2.6.0
 pyparsing==2.4.7
 pysndfile==1.4.4
-pysolr==3.10.0b1
+pysolr==3.10.0
 python-louvain==0.16  # community detection in clustering
 pytest~=8.0.2
 pytest-django~=4.8.0

requirements.txt

@@ -268,7 +268,7 @@ pyproject-hooks==1.0.0
     # via build
 pysndfile==1.4.4
     # via -r requirements.in
-pysolr==3.10.0b1
+pysolr==3.10.0
     # via -r requirements.in
 pytest==8.0.2
     # via

search/management/commands/test_search_engine_backend.py

@@ -38,12 +38,8 @@
 def core_exists(solr_base_url, core_name):
     r = requests.get(f'{solr_base_url}/admin/cores?action=STATUS&core={core_name}')
     r.raise_for_status()
-    try:
-        status = r.json()
-        return status['status'][core_name] != {}
-    except ValueError:
-        # Solr 5 returns xml. "Empty list" means that the core does not exist
-        return f"""<lst name="{core_name}"/></lst>""" not in r.text
+    status = r.json()
+    return status['status'][core_name] != {}
 
 
 def create_core(solr_base_url, core_name, configSet, delete_core=False):

search/templatetags/search.py

@@ -23,8 +23,7 @@
 from django.conf import settings
 
 from sounds.models import License
-from utils.search import search_query_processor_options
-from utils.search.backends.solr555pysolr import FIELD_NAMES_MAP
+from utils.search import search_query_processor_options, FIELD_NAMES_MAP
 from utils.tags import annotate_tags
 
 register = template.Library()

utils/search/__init__.py

@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-
 #
 # Freesound is (c) MUSIC TECHNOLOGY GROUP, UNIVERSITAT POMPEU FABRA
 #
@@ -24,41 +22,46 @@
 import pysolr
 from django.conf import settings
 
-from utils.search.backends import solr555pysolr
+from utils.search.backends import SearchEngineBase
+from utils.search.backends.solr_common import FreesoundSoundJsonEncoder, SolrResponseInterpreter
 
 SOLR_FORUM_URL = f"{settings.SOLR9_BASE_URL}/forum"
 SOLR_SOUNDS_URL = f"{settings.SOLR9_BASE_URL}/freesound"
 
 
-class Solr9PySolrSearchEngine(solr555pysolr.Solr555PySolrSearchEngine):
+class Solr9PySolrSearchEngine(SearchEngineBase):
+
     def __init__(self, sounds_index_url=None, forum_index_url=None):
         if sounds_index_url is None:
             sounds_index_url = SOLR_SOUNDS_URL
         if forum_index_url is None:
             forum_index_url = SOLR_FORUM_URL
         self.sounds_index_url = sounds_index_url
         self.forum_index_url = forum_index_url
+        self._sounds_index = None
+        self._forum_index = None
 
-
-    def get_sounds_index(self):
-        if self.sounds_index is None:
-            self.sounds_index = pysolr.Solr(
+    @property
+    def sounds_index(self):
+        if self._sounds_index is None:
+            self._sounds_index = pysolr.Solr(
                 self.sounds_index_url,
-                encoder=solr555pysolr.FreesoundSoundJsonEncoder(),
-                results_cls=solr555pysolr.SolrResponseInterpreter,
+                encoder=FreesoundSoundJsonEncoder(),
+                results_cls=SolrResponseInterpreter,
                 always_commit=True
             )
-        return self.sounds_index
+        return self._sounds_index
 
-    def get_forum_index(self):
-        if self.forum_index is None:
-            self.forum_index = pysolr.Solr(
+    @property
+    def forum_index(self):
+        if self._forum_index is None:
+            self._forum_index = pysolr.Solr(
                 self.forum_index_url,
-                encoder=solr555pysolr.FreesoundSoundJsonEncoder(),
-                results_cls=solr555pysolr.SolrResponseInterpreter,
+                encoder=FreesoundSoundJsonEncoder(),
+                results_cls=SolrResponseInterpreter,
                 always_commit=True
             )
-        return self.forum_index
+        return self._forum_index
 
 
     def search_process_filter(self, query_filter, only_sounds_within_ids=False, only_sounds_with_pack=False):
@@ -92,23 +95,23 @@ def search_process_filter(self, query_filter, only_sounds_within_ids=False, only
             query_filter = query_filter.replace('created:', 'created_range:')
 
         # If we only want sounds with packs and there is no pack filter, add one
-        if only_sounds_with_pack and not 'pack:' in query_filter:
+        if only_sounds_with_pack and 'pack:' not in query_filter:
             query_filter += ' pack:*'
 
         if 'geotag:"Intersects(' in query_filter:
             # Replace geotag:"Intersects(<MINIMUM_LONGITUDE> <MINIMUM_LATITUDE> <MAXIMUM_LONGITUDE> <MAXIMUM_LATITUDE>)"
             #    with geotag:["<MINIMUM_LATITUDE>, <MINIMUM_LONGITUDE>" TO "<MAXIMUM_LONGITUDE> <MAXIMUM_LATITUDE>"]
-            query_filter = re.sub('geotag:"Intersects\((.+?) (.+?) (.+?) (.+?)\)"', r'geotag:["\2,\1" TO "\4,\3"]', query_filter)
+            query_filter = re.sub(r'geotag:"Intersects\((.+?) (.+?) (.+?) (.+?)\)"', r'geotag:["\2,\1" TO "\4,\3"]', query_filter)
 
         query_filter = self.search_filter_make_intersection(query_filter)
 
         # When calculating results form clustering, the "only_sounds_within_ids" argument is passed and we filter
         # our query to the sounds in that list of IDs.
         if only_sounds_within_ids:
-            sounds_within_ids_filter = ' OR '.join(['id:{}'.format(sound_id) for sound_id in only_sounds_within_ids])
+            sounds_within_ids_filter = ' OR '.join([f'id:{sound_id}' for sound_id in only_sounds_within_ids])
             if query_filter:
-                query_filter += ' AND ({})'.format(sounds_within_ids_filter)
+                query_filter += f' AND ({sounds_within_ids_filter})'
             else:
-                query_filter = '({})'.format(sounds_within_ids_filter)
+                query_filter = f'({sounds_within_ids_filter})'
 
-        return query_filter
+        return query_filter

utils/search/backends/__init__.py

@@ -18,7 +18,10 @@
 #     Bram de Jong
 #
 import json
-import urllib.request, urllib.parse, urllib.error
+from datetime import date, datetime
+import urllib.request
+import urllib.parse
+import urllib.error
 
 from utils.search import SearchEngineException
 
@@ -324,3 +327,13 @@ def __init__(self, response, next_page_query=None):
             self.highlighting = response["highlighting"]
         except KeyError:
             self.highlighting = {}
+
+
+class FreesoundSoundJsonEncoder(json.JSONEncoder):
+    def default(self, value):
+        if isinstance(value, datetime):
+            return value.strftime('%Y-%m-%dT%H:%M:%S.000Z')
+        elif isinstance(value, date):
+            return value.strftime('%Y-%m-%dT00:00:00.000Z')
+
+        return json.JSONEncoder.default(self, value)

utils/search/backends/solr555pysolr.py

@@ -1,14 +1,14 @@
 from django.test import TestCase
 
-from utils.search.backends import solr555pysolr
+from utils.search.backends import solr9pysolr
 
 class SolrCommonTest(TestCase):
     def test_search_filter_make_intersection(self):
 
         filter_query = "username:alastairp"
-        updated = solr555pysolr.Solr555PySolrSearchEngine().search_filter_make_intersection(filter_query)
+        updated = solr9pysolr.Solr9PySolrSearchEngine().search_filter_make_intersection(filter_query)
         self.assertEqual(updated, "+username:alastairp")
 
         filter_query = "username:alastairp license:(a OR b)"
-        updated = solr555pysolr.Solr555PySolrSearchEngine().search_filter_make_intersection(filter_query)
+        updated = solr9pysolr.Solr9PySolrSearchEngine().search_filter_make_intersection(filter_query)
         self.assertEqual(updated, "+username:alastairp +license:(a OR b)")

utils/search/backends/solr9pysolr.py

@@ -31,7 +31,6 @@
 
 from utils.clustering_utilities import get_ids_in_cluster, get_clusters_for_query
 from utils.encryption import create_hash
-from utils.search.backends.solr555pysolr import Solr555PySolrSearchEngine
 from utils.search.search_sounds import allow_beta_search_features
 from .search_query_processor_options import SearchOptionStr, SearchOptionChoice, \
     SearchOptionInt, SearchOptionBool, SearchOptionRange, SearchOptionMultipleChoice, \

utils/search/backends/solr_common.py

@@ -175,7 +175,7 @@ def get_sound_similarity_from_search_engine_query(query_params, analyzer_name=se
 
     # Update query params to get similarity vectors of the first 
     config_options = settings.SEARCH_ENGINE_SIMILARITY_ANALYZERS[analyzer_name]
-    vector_field_name = utils.search.backends.solr555pysolr.get_solr_dense_vector_search_field_name(config_options['vector_size'], config_options('l2_norm', False))
+    vector_field_name = utils.search.get_solr_dense_vector_search_field_name(config_options['vector_size'], config_options('l2_norm', False))
     query_params.update({
         'facets': None,
         'current_page': current_page if current_page is not None else query_params['current_page'],