vary selector type based on selectivity

faiss_vector_posting.go

@@ -426,13 +426,35 @@ func (sb *SegmentBase) InterpretVectorIndex(field string, requiresFiltering bool
 					}
 					eligibleVecIDsBitmap.AddMany(vecIDsUint32)
 
+					var selector faiss.Selector
+					var err error
+					excludeVecIDs := make([]int64, 0, len(vecDocIDMap)-
+						len(vectorIDsToInclude))
+					if float32(eligibleVecIDsBitmap.GetCardinality())/
+						float32(len(vecDocIDMap)) > 0.5 {
+						for _, eligibleDocID := range eligibleDocIDs {
+							vecIDs := docVecIDMap[uint32(eligibleDocID)]
+							for _, vecID := range vecIDs {
+								if !eligibleVecIDsBitmap.Contains(uint32(vecID)) {
+									excludeVecIDs = append(excludeVecIDs, (vecID))
+								}
+							}
+						}
+						selector, err = faiss.NewIDSelectorNot(excludeVecIDs)
+						if err != nil {
+							return nil, err
+						}
+					} else {
+						selector, err = faiss.NewIDSelectorBatch(vectorIDsToInclude)
+					}
+
 					// Determining which clusters, identified by centroid ID,
 					// have at least one eligible vector and hence, ought to be
 					// probed.
 					eligibleCentroidIDs := make([]int64, 0)
 					for centroidID, vecIDs := range centroidVecIDMap {
 						vecIDs.And(eligibleVecIDsBitmap)
-						if vecIDs.GetCardinality() > 0 {
+						if !vecIDs.IsEmpty() {
 							// The mapping is now reduced to those vectors which
 							// are also eligible docs for the filter query.
 							centroidVecIDMap[centroidID] = vecIDs
@@ -469,8 +491,8 @@ func (sb *SegmentBase) InterpretVectorIndex(field string, requiresFiltering bool
 					// Search the clusters specified by 'closestCentroidIDs' for
 					// vectors whose IDs are present in 'vectorIDsToInclude'
 					scores, ids, err := vecIndex.SearchClustersFromIVFIndex(
-						vectorIDsToInclude, closestCentroidIDs, minEligibleCentroids,
-						k, qVector, centroidDistances, params)
+						selector, len(vectorIDsToInclude), closestCentroidIDs,
+						minEligibleCentroids, k, qVector, centroidDistances, params)
 					if err != nil {
 						return nil, err
 					}