Skip to content

Commit

Permalink
PoC for Vectara integration
Browse files Browse the repository at this point in the history
  • Loading branch information
@mkr
mkr committed Sep 11, 2023
1 parent 174df58 commit 9ddc721
Show file tree
Hide file tree
Showing 9 changed files with 110 additions and 35 deletions.
3 changes: 2 additions & 1 deletion app/assets/javascripts/controllers/settings.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,8 @@ angular.module('QuepidApp')
});

function submit () {
if ( $scope.pendingSettings.searchEngine === 'es' || $scope.pendingSettings.searchEngine === 'os') {
if ( $scope.pendingSettings.searchEngine === 'es' || $scope.pendingSettings.searchEngine === 'os' ||
$scope.pendingSettings.searchEngine === 'vectara') {
// Verify that JSON is valid
try {
var jsonObject = JSON.parse($scope.pendingSettings.selectedTry.queryParams);
Expand Down
3 changes: 2 additions & 1 deletion app/assets/javascripts/filters/searchEngineName.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@ angular.module('QuepidApp')
var searchEngineName = {
solr: 'Solr',
es: 'Elasticsearch',
os: 'OpenSearch'
os: 'OpenSearch',
vectara: 'Vectara'
};

return function (input) {
Expand Down
2 changes: 1 addition & 1 deletion app/assets/javascripts/services/docCacheSvc.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ angular.module('QuepidApp')
var docIds = Object.keys(docsToFetch);
var resolver = docResolverSvc.createResolver(docIds, settings, 15);

if ( docIds.length > 0 ) {
if ( docIds.length > 0 && settings.searchEngine !== 'vectara') { // 'vectara' does not support doc lookup by ID
return resolver.fetchDocs()
.then(function () {
angular.forEach(resolver.docs, function (doc) {
Expand Down
27 changes: 21 additions & 6 deletions app/assets/javascripts/services/queriesSvc.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ angular.module('QuepidApp')
'searchErrorTranslatorSvc',
'esExplainExtractorSvc',
'solrExplainExtractorSvc',
'normalDocsSvc',
function queriesSvc(
$scope,
$http,
Expand All @@ -35,7 +36,8 @@ angular.module('QuepidApp')
diffResultsSvc,
searchErrorTranslatorSvc,
esExplainExtractorSvc,
solrExplainExtractorSvc
solrExplainExtractorSvc,
normalDocsSvc
) {

let caseNo = -1;
Expand Down Expand Up @@ -69,7 +71,7 @@ angular.module('QuepidApp')
});

function createSearcherFromSettings(passedInSettings, queryText, query) {
let args = angular.copy(passedInSettings.selectedTry.args);
let args = angular.copy(passedInSettings.selectedTry.args) || {};

if (passedInSettings && passedInSettings.selectedTry) {

Expand Down Expand Up @@ -100,13 +102,17 @@ angular.module('QuepidApp')
'filter': query.filterToRatings(passedInSettings)
}
};
} else {
} else if (passedInSettings.searchEngine === 'solr') {
if (args['fq'] === undefined) {
args['fq'] = [];
}
args['fq'].push(query.filterToRatings(passedInSettings));
} else if (passedInSettings.searchEngine === 'vectara') {
// currently doc id filtering frequently produces 0 results
// args['query'] = args['query'].map(function addFilter(query) {
// query['metadata_filter'] = query.filterToRatings(passedInSettings);
// });
}

}


Expand All @@ -128,8 +134,13 @@ angular.module('QuepidApp')

if (searcher.type === 'es' || searcher.type === 'os') {
normed = esExplainExtractorSvc.docsWithExplainOther(searcher.docs, fieldSpec);
} else {
} else if (searcher.type === 'solr') {
normed = solrExplainExtractorSvc.docsWithExplainOther(searcher.docs, fieldSpec, searcher.othersExplained);
} else {
// search engine with no explain output
normed = searcher.docs.map(function(doc) {
return normalDocsSvc.createNormalDoc(fieldSpec, doc);
})
}

let docs = [];
Expand Down Expand Up @@ -659,8 +670,12 @@ angular.module('QuepidApp')
};
esQuery['terms'][fieldSpec.id] = ratedIDs;
return esQuery;
} else {
} else if (settings.searchEngine === 'solr') {
return '{!terms f=' + fieldSpec.id + '}' + ratedIDs.join(',');
} else if (settings.searchEngine === 'vectara') {
return ratedIDs.map(function(id) {
return "doc.id = '" + id + "'";
}).join(' OR ');
}
};
};
Expand Down
50 changes: 40 additions & 10 deletions app/assets/javascripts/services/settingsSvc.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,7 @@ angular.module('QuepidApp')
// perfect world we wouldn't have this here and we would instead populate with urlFormat instead.
insecureSearchUrl:'http://quepid-solr.dev.o19s.com:8985/solr/tmdb/select',
secureSearchUrl: 'https://quepid-solr.dev.o19s.com/solr/tmdb/select',
urlFormat: 'http(s?)://yourdomain.com:8983/<index>/select',
customHeaders: '',
urlFormat: 'http(s?)://yourdomain.com:8983/<index>/select'
},
es: {
queryParams: [
Expand All @@ -65,8 +64,7 @@ angular.module('QuepidApp')
numberOfRows: 10,
searchEngine: 'es',
searchUrl: 'http://quepid-elasticsearch.dev.o19s.com:9206/tmdb/_search',
urlFormat: 'http(s?)://yourdomain.com:9200/<index>/_search',
customHeaders: '',
urlFormat: 'http(s?)://yourdomain.com:9200/<index>/_search'
},
os: {
queryParams: [
Expand All @@ -83,6 +81,7 @@ angular.module('QuepidApp')

escapeQuery: true,
apiMethod: 'POST',
customHeaders: '',
fieldSpec: 'id:_id',
idField: '_id',
titleField: '',
Expand All @@ -91,7 +90,39 @@ angular.module('QuepidApp')
searchEngine: 'os',
searchUrl: 'https://reader:[email protected]:9000/tmdb/_search',
urlFormat: 'http(s?)://yourdomain.com:9200/<index>/_search',
},
vectara: {
queryParams: [
'{',
' "query": [',
' {',
' "query": "#$query##",',
' "start": 0,',
' "numResults": 10,',
' "corpusKey": [{',
' "customerId": 123456789,',
' "corpusId": 1,',
' "lexicalInterpolationConfig": {',
' "lambda": 0.025',
' },',
' "dim": []',
' }]',
' }',
' ]',
'}'
].join('\n'),

escapeQuery: true,
apiMethod: 'POST',
customHeaders: '',
fieldSpec: 'id:id',
idField: 'id',
titleField: 'title',
additionalFields: [],
numberOfRows: 10,
searchEngine: 'vectara',
searchUrl: 'https://api.vectara.io/v1/query',
urlFormat: 'https://api.vectara.io/v1/query'
}
};
// used by the wizard for TMDB demo search engine
Expand All @@ -118,8 +149,7 @@ angular.module('QuepidApp')
searchEngine: 'solr',
insecureSearchUrl:'http://quepid-solr.dev.o19s.com:8985/solr/tmdb/select',
secureSearchUrl: 'https://quepid-solr.dev.o19s.com/solr/tmdb/select',
urlFormat: 'http(s?)://yourdomain.com:8983/<index>/select',
customHeaders: '',
urlFormat: 'http(s?)://yourdomain.com:8983/<index>/select'
},
es: {
queryParams: [
Expand Down Expand Up @@ -149,8 +179,7 @@ angular.module('QuepidApp')
numberOfRows: 10,
searchEngine: 'es',
searchUrl: 'http://quepid-elasticsearch.dev.o19s.com:9206/tmdb/_search',
urlFormat: 'http(s?)://yourdomain.com:9200/<index>/_search',
customHeaders: '',
urlFormat: 'http(s?)://yourdomain.com:9200/<index>/_search'
},
os: {
queryParams: [
Expand Down Expand Up @@ -200,8 +229,9 @@ angular.module('QuepidApp')
else {
useTMDBDemoSettings = false;
}
}
else {
} else if (searchEngine === 'vectara') {
useTMDBDemoSettings = false;
} else {
if (newUrl === this.tmdbSettings[searchEngine].searchUrl) {
useTMDBDemoSettings = true;
}
Expand Down
16 changes: 16 additions & 0 deletions app/assets/templates/views/devQueryParams.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,15 @@ <h4 class="pull-left">Query Sandbox:</h4>
theme: 'chrome'
}" ng-model="settings.selectedTry.queryParams">
</div>

<div id='vectara-query-params-editor' class='vectara-query-params' ng-show="settings.searchEngine == 'vectara'" ui-ace="{
require: ['ace/ext/language_tools'],
useWrapMode: false,
mode: 'json',
theme: 'chrome'
}" ng-model="settings.selectedTry.queryParams">
</div>

</div> <!-- end .rerunnable-settings-tab -->
</div> <!-- end .settings-tab (Query Sandbox) -->

Expand Down Expand Up @@ -105,6 +114,13 @@ <h4>Tuning Knobs</h4>
OpenSearch
</label>
</div>

<div>
<label>
<input type="radio" value="vectara" ng-model="settings.searchEngine" ng-change="changeSearchEngine()">
Vectara
</label>
</div>
</div>
</div>
<!-- end of Search Engine -->
Expand Down
35 changes: 20 additions & 15 deletions app/assets/templates/views/wizardModal.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,10 @@ <h2>Solr, Elasticsearch, or OpenSearch?</h2>
<input type="radio" value="os" ng-model="pendingWizardSettings.searchEngine" ng-change="changeSearchEngine()">
<img ng-src="images/opensearch.png" alt='OpenSearch' width="150px" />
</label>
<label>
<input type="radio" value="vectara" ng-model="pendingWizardSettings.searchEngine" ng-change="changeSearchEngine()">
<img ng-src="images/vectara.png" alt='Vectara' width="150px" />
</label>
</div>

<br>
Expand All @@ -62,7 +66,8 @@ <h2>Solr, Elasticsearch, or OpenSearch?</h2>
Sorry, we're not getting any search results from your
<span ng-show="pendingWizardSettings.searchEngine === 'es'">Elasticsearch</span>
<span ng-show="pendingWizardSettings.searchEngine === 'os'">OpenSearch</span>
<span ng-show="pendingWizardSettings.searchEngine === 'solr'">Solr</span>.
<span ng-show="pendingWizardSettings.searchEngine === 'solr'">Solr</span>
<span ng-show="pendingWizardSettings.searchEngine === 'vectara'">Vectara</span>.

<ul ng-if="pendingWizardSettings.searchEngine === 'es'">
<li>Is your Elasticsearch behind a firewall or proxy? Try acessing the <a target="_blank" href="{{pendingWizardSettings.searchUrl}}">URL directly</a> and see if you get a response</li>
Expand Down Expand Up @@ -122,23 +127,23 @@ <h2>Solr, Elasticsearch, or OpenSearch?</h2>
</div>

<span ng-if="pendingWizardSettings.searchEngine === 'solr'">
<p>
<button type="button" class="btn btn-sm btn-default" ng-click="isSolrCollapsed = !isSolrCollapsed">Solr Configuration</button>
</p>
<div uib-collapse="!isSolrCollapsed">
<div class="well well-sm">
<div class="form-group clearfix">
<label class="col-sm-3 control-label">API Method</label>
<div class="col-sm-9">
<select class="form-control" ng-model="pendingWizardSettings.apiMethod" ng-change="resetUrlValid()">
<option>JSONP</option>
<option>GET</option>
</select>
<p class="help-block">Use JSONP if you are talking directly to Solr, otherwise if you are talking to an API you can use GET.</p>
<p>
<button type="button" class="btn btn-sm btn-default" ng-click="isSolrCollapsed = !isSolrCollapsed">Solr Configuration</button>
</p>
<div uib-collapse="!isSolrCollapsed">
<div class="well well-sm">
<div class="form-group clearfix">
<label class="col-sm-3 control-label">API Method</label>
<div class="col-sm-9">
<select class="form-control" ng-model="pendingWizardSettings.apiMethod" ng-change="resetUrlValid()">
<option>JSONP</option>
<option>GET</option>
</select>
<p class="help-block">Use JSONP if you are talking directly to Solr, otherwise if you are talking to an API you can use GET.</p>
</div>
</div>
</div>
</div>
</div>
</span>


Expand Down
7 changes: 7 additions & 0 deletions app/models/try.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,8 @@ def args
es_args
when 'os'
os_args
when 'vectara'
vectara_args
end
end

Expand Down Expand Up @@ -92,6 +94,11 @@ def os_args
EsArgParser.parse(query_params, curator_vars_map)
end

def vectara_args
# Use the EsArgParser as currently queries are the same
EsArgParser.parse(query_params, curator_vars_map)
end

def id_from_field_spec
# logic is inspired by https://github.com/o19s/splainer-search/blob/main/services/fieldSpecSvc.js

Expand Down
2 changes: 1 addition & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@
"ngclipboard": "^2.0.0",
"party-js": "^2.2.0",
"popper.js": "^1.16.1",
"splainer-search": "2.24.0",
"splainer-search": "2.25.0",
"tether-shepherd": "latest",
"turbolinks": "^5.2.0",
"vega": "5.25.0"
Expand Down

0 comments on commit 9ddc721

Please sign in to comment.