Indexing performance improvements

Author: marcus-bcl

projects/person-search-index-from-delius/container/pipelines/contact/index/index-template-semantic.yml

@@ -1236,8 +1236,8 @@ template:
       endTime:
         type: date
         format: hour_minute_second
-      text:
-        type: text
+      textChunks:
+        type: keyword
       textEmbedding:
         type: nested
         properties:
@@ -1249,17 +1249,14 @@ template:
               name: hnsw
               engine: lucene
       notes:
-        copy_to: text
         type: text
         analyzer: probation_text_analyzer
         boost: 5
       description:
-        copy_to: text
         type: text
         analyzer: probation_text_analyzer
         boost: 4
       outcome:
-        copy_to: text
         type: text
         analyzer: probation_text_analyzer
         store: true
@@ -1268,7 +1265,6 @@ template:
           keyword:
             type: keyword
       type:
-        copy_to: text
         type: text
         analyzer: probation_text_analyzer
         store: true

projects/person-search-index-from-delius/container/pipelines/contact/index/ingest-pipeline.tpl.json

@@ -16,6 +16,7 @@
         "algorithm": {
           "fixed_token_length": {
             "token_limit": 32,
+            "max_chunk_limit": -1,
             "overlap_rate": 0.125,
             "tokenizer": "standard"
           }

projects/person-search-index-from-delius/container/pipelines/contact/logstash-incremental.conf

@@ -49,6 +49,7 @@ filter {
             jdbc_password => "${JDBC_PASSWORD}"
             jdbc_validate_connection => true
             jdbc_validation_timeout => 120
+            sequel_opts => { max_connections => 100 }
             statement => "
                 select contact_id
                 from contact join offender on offender.offender_id = contact.offender_id
@@ -89,6 +90,7 @@ filter {
         jdbc_password => "${JDBC_PASSWORD}"
         jdbc_validate_connection => true
         jdbc_validation_timeout => 120
+        sequel_opts => { max_connections => 100 }
         statement => "${INCREMENTAL_STATEMENT_SQL}"
         use_prepared_statements => true
         prepared_statement_name => "search_indexer_contact_incremental"

projects/person-search-index-from-delius/deploy/values.yaml

@@ -29,8 +29,8 @@ generic-service:
     CONTACT_INDEX_PREFIX: contact-search
     PERSON_INDEX_PREFIX: person-search
     PIPELINES_ENABLED: person-incremental,contact-incremental
-    PIPELINE_WORKERS: 10
-    PIPELINE_BATCH_SIZE: 1
+    PIPELINE_WORKERS: 50
+    PIPELINE_BATCH_SIZE: 5
     JDK_JAVA_OPTIONS: ''
     LS_JAVA_OPTS: -Xms3g -Xmx3g # = 75% of pod memory limit (see above)