[#26629] YSQL: Push down ef_search parameter to ybhnsw

Summary:
The GUC `hnsw.ef_search` in pgvector controls the expansion factor for HNSW searches, where higher values improve accuracy at the cost of increased latency. This change introduces a corresponding GUC, `ybhnsw.ef_search`, and propagates it to DocDB. Additional work is required in DocDB to utilize the propagated value.

**Upgrade/Rollback safety:**
This change adds new fields to `PgVectorReadOptionsPB` in common.proto that are unused in DocDB code as of this change. This does not suggest any upgrade/rollback issues.

Test Plan:
Jenkins

Manual testing by observing the docdb requests of ybhnsw queries to make sure the correct ef_search parameter is passed down

Reviewers: kramanathan

Reviewed By: kramanathan

Subscribers: yql

Differential Revision: https://phorge.dev.yugabyte.com/D42934

Author: Tanuj Nayak

src/postgres/third-party-extensions/pgvector/src/ybvector/ybhnsw.c

@@ -42,18 +42,25 @@
 #define YBHNSW_MIN_EF_CONSTRUCTION 50
 #define YBHNSW_MAX_EF_CONSTRUCTION 1000
 
+/* Imported from pgvector defaults as of v0.8.0. */
+#define YBHNSW_DEFAULT_EF_SEARCH 40
+#define YBHNSW_MIN_EF_SEARCH 1
+#define YBHNSW_MAX_EF_SEARCH 1000
+
 static relopt_kind ybhnsw_relopt_kind;
 
+int			ybhnsw_ef_search;
+
 /* 
  * Copied from pgvector's HnswInit (as of pgvector v0.8.0).
  */
-typedef struct YbHnswOptions
+typedef struct YbHnswCreateOptions
 {
 	int32		vl_len_;		/* varlena header (do not touch directly!) */
 	int			m;				/* number of connections per node */
 	int			m0;				/* number of connections per node in base level */
 	int			ef_construction;	/* size of dynamic candidate list */
-}			YbHnswOptions;
+}			YbHnswCreateOptions;
 
 void
 YbHnswInit(void)
@@ -71,6 +78,11 @@ YbHnswInit(void)
 					  YBHNSW_DEFAULT_EF_CONSTRUCTION,
 					  YBHNSW_MIN_EF_CONSTRUCTION, YBHNSW_MAX_EF_CONSTRUCTION,
 					  AccessExclusiveLock);
+	
+	DefineCustomIntVariable("ybhnsw.ef_search", "Sets the size of the dynamic candidate list for search",
+							"Valid range is 1..1000.", &ybhnsw_ef_search,
+							YBHNSW_DEFAULT_EF_SEARCH, YBHNSW_MIN_EF_SEARCH, YBHNSW_MAX_EF_SEARCH, PGC_USERSET, 0, NULL, NULL, NULL);
+	MarkGUCPrefixReserved("ybhnsw");
 }
 
 /*
@@ -83,22 +95,22 @@ ybhnswoptions(Datum reloptions, bool validate)
  	 * Copied from pgvector's hnswoptions (as of pgvector v0.8.0).
  	 */
 	static const relopt_parse_elt tab[] = {
-		{"m", RELOPT_TYPE_INT, offsetof(YbHnswOptions, m)},
-		{"m0", RELOPT_TYPE_INT, offsetof(YbHnswOptions, m0)},
+		{"m", RELOPT_TYPE_INT, offsetof(YbHnswCreateOptions, m)},
+		{"m0", RELOPT_TYPE_INT, offsetof(YbHnswCreateOptions, m0)},
 		{"ef_construction", RELOPT_TYPE_INT,
-		offsetof(YbHnswOptions, ef_construction)},
+		offsetof(YbHnswCreateOptions, ef_construction)},
 	};
 
 	return (bytea *) build_reloptions(reloptions, validate,
 									  ybhnsw_relopt_kind,
-									  sizeof(YbHnswOptions),
+									  sizeof(YbHnswCreateOptions),
 									  tab, lengthof(tab));
 }
 
 static void
 bindYbHnswIndexOptions(YbcPgStatement handle, Datum reloptions)
 {
-	YbHnswOptions *hnsw_options = (YbHnswOptions *) ybhnswoptions(reloptions, false);
+	YbHnswCreateOptions *hnsw_options = (YbHnswCreateOptions *) ybhnswoptions(reloptions, false);
 	int			m = YBHNSW_DEFAULT_M;
 	int         m0 = YBHNSW_DEFAULT_M;
 	int			ef_construction = YBHNSW_DEFAULT_EF_CONSTRUCTION;
@@ -149,6 +161,26 @@ ybhnswbindcolumnschema(YbcPgStatement handle,
 	YBCBindCreateIndexColumns(handle, indexInfo, indexTupleDesc, coloptions, 0);
 }
 
+static void
+ybBindHnswReadOptions(YbScanDesc yb_scan)
+{
+	YBCPgDmlHnswSetReadOptions(yb_scan->handle, ybhnsw_ef_search);
+}
+
+/*
+ * ybvectorrescan
+ *		Reset temporary structures to prepare for rescan.
+ */
+void
+ybhnswrescan(IndexScanDesc scan, ScanKey scankeys, int nscankeys,
+			 ScanKey orderbys, int norderbys)
+{
+	ybvectorrescan(scan, scankeys, nscankeys, orderbys, norderbys);
+	YbVectorScanOpaque so = (YbVectorScanOpaque) scan->opaque;
+	YbScanDesc ybscan = so->yb_scan_desc;
+	ybBindHnswReadOptions(ybscan);
+}
+
 
 /*
  * ybusearchhandler handler function: return
@@ -164,5 +196,7 @@ ybhnswhandler(PG_FUNCTION_ARGS)
 	amroutine->yb_ambindschema = ybhnswbindcolumnschema;
 	amroutine->amoptions = ybhnswoptions;
 
+	amroutine->amrescan = ybhnswrescan;
+
 	PG_RETURN_POINTER(amroutine);
 }

src/yb/common/common.proto

@@ -150,9 +150,16 @@ message PgVectorIdxOptionsPB {
   optional PgHnswIndexOptionsPB hnsw = 7;
 }
 
+message PgHnswReadOptionsPB {
+  required uint32 ef_search = 1;
+}
+
 message PgVectorReadOptionsPB {
   required QLValuePB vector = 1;
   required int32 prefetch_size = 2;
+  oneof index_params {
+    PgHnswReadOptionsPB hnsw_options = 3;
+  }
 }
 
 message TablePropertiesPB {

src/yb/yql/pggate/pg_dml.cc

@@ -251,6 +251,14 @@ Status PgDml::ANNSetPrefetchSize(int32_t prefetch_size) {
   return down_cast<PgDmlRead*>(this)->ANNSetPrefetchSize(prefetch_size);
 }
 
+Status PgDml::HnswSetReadOptions(int ef_search) {
+  if (auto* secondary_index = SecondaryIndexQuery(); secondary_index) {
+    return secondary_index->HnswSetReadOptions(ef_search);
+  }
+
+  return down_cast<PgDmlRead*>(this)->HnswSetReadOptions(ef_search);
+}
+
 Status PgDml::BindTable() {
   bind_table_ = true;
   return Status::OK();

src/yb/yql/pggate/pg_dml.h

@@ -73,6 +73,9 @@ class PgDml : public PgStatement {
   // Bind prefetch size to the current vector index search.
   Status ANNSetPrefetchSize(int32_t prefetch_size);
 
+  // Set HNSW read options.
+  Status HnswSetReadOptions(int ef_search);
+
   // Bind the whole table.
   Status BindTable();
 

src/yb/yql/pggate/pg_dml_read.cc

@@ -431,6 +431,11 @@ Status PgDmlRead::ANNSetPrefetchSize(int32_t prefetch_size) {
   return Status::OK();
 }
 
+Status PgDmlRead::HnswSetReadOptions(int ef_search) {
+  read_req_->mutable_vector_idx_options()->mutable_hnsw_options()->set_ef_search(ef_search);
+  return Status::OK();
+}
+
 Status PgDmlRead::Exec(const YbcPgExecParameters* exec_params) {
   RSTATUS_DCHECK(
       !pg_exec_params_ || pg_exec_params_ == exec_params,

src/yb/yql/pggate/pg_dml_read.h

@@ -113,6 +113,8 @@ class PgDmlRead : public PgDml {
   Status ANNBindVector(PgExpr* vector);
   Status ANNSetPrefetchSize(int32_t prefetch_size);
 
+  Status HnswSetReadOptions(int ef_search);
+
   void SetCatalogCacheVersion(std::optional<PgOid> db_oid, uint64_t version) override {
     DoSetCatalogCacheVersion(read_req_.get(), db_oid, version);
   }

src/yb/yql/pggate/pggate.cc

@@ -1565,6 +1565,10 @@ Status PgApiImpl::DmlANNSetPrefetchSize(PgStatement* handle, int prefetch_size)
   return VERIFY_RESULT_REF(GetStatementAs<PgDml>(handle)).ANNSetPrefetchSize(prefetch_size);
 }
 
+Status PgApiImpl::DmlHnswSetReadOptions(PgStatement* handle, int ef_search) {
+  return VERIFY_RESULT_REF(GetStatementAs<PgDml>(handle)).HnswSetReadOptions(ef_search);
+}
+
 Status PgApiImpl::ExecSelect(PgStatement* handle, const YbcPgExecParameters* exec_params) {
   auto& select = VERIFY_RESULT_REF(GetStatementAs<PgSelect>(handle));
   if (pg_sys_table_prefetcher_ && select.IsReadFromYsqlCatalog() && select.read_req()) {

src/yb/yql/pggate/pggate.h

@@ -607,6 +607,8 @@ class PgApiImpl {
 
   Status DmlANNSetPrefetchSize(PgStatement *handle, int prefetch_size);
 
+  Status DmlHnswSetReadOptions(PgStatement *handle, int ef_search);
+
 
   //------------------------------------------------------------------------------------------------
   // Functions.

src/yb/yql/pggate/ybc_pggate.cc

@@ -1514,6 +1514,10 @@ YbcStatus YBCPgDmlANNSetPrefetchSize(YbcPgStatement handle, int prefetch_size) {
   return ToYBCStatus(pgapi->DmlANNSetPrefetchSize(handle, prefetch_size));
 }
 
+YbcStatus YBCPgDmlHnswSetReadOptions(YbcPgStatement handle, int ef_search) {
+  return ToYBCStatus(pgapi->DmlHnswSetReadOptions(handle, ef_search));
+}
+
 YbcStatus YBCPgDmlFetch(YbcPgStatement handle, int32_t natts, uint64_t *values, bool *isnulls,
                         YbcPgSysColumns *syscols, bool *has_data) {
   return ToYBCStatus(pgapi->DmlFetch(handle, natts, values, isnulls, syscols, has_data));

src/yb/yql/pggate/ybc_pggate.h

@@ -565,6 +565,8 @@ YbcStatus YBCPgDmlANNBindVector(YbcPgStatement handle, YbcPgExpr vector);
 
 YbcStatus YBCPgDmlANNSetPrefetchSize(YbcPgStatement handle, int prefetch_size);
 
+YbcStatus YBCPgDmlHnswSetReadOptions(YbcPgStatement handle, int ef_search);
+
 // This function is to fetch the targets in YBCPgDmlAppendTarget() from the rows that were defined
 // by YBCPgDmlBindColumn().
 YbcStatus YBCPgDmlFetch(YbcPgStatement handle, int32_t natts, uint64_t *values, bool *isnulls,