Skip to content

Commit

Permalink
working on being able to export/import snapshots better. (#821)
Browse files Browse the repository at this point in the history 
* Can now export and import snapshots with document fields.
  • Loading branch information
@epugh
epugh authored Sep 6, 2023
1 parent ffde2be commit 5a4c579
Show file tree
Hide file tree
Showing 10 changed files with 147 additions and 75 deletions.
1 change: 1 addition & 0 deletions app/assets/javascripts/components/export_case/_modal.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ <h3 class="modal-title">Export Case: <span class="modal-case">{{ ctrl.theCase.ca
Snapshot's represent a point in time, and you may want to download them for further analysis, rework them, and then
import them back into Quepid using the <strong>Import Snapshots from CSV</strong> function on the Relevancy Cases listing page.
You will have a CSV file with <code>Snapshot Name,Snapshot Time,Case ID,Query Text,Doc ID,Doc Position</code> fields.
If you have recorded the document fields when you took the snapshot, then you will also have <code>Field1,...,FieldN</code> columns.
</span>
<label for="snapshot_choice">Snapshot
<select
Expand Down
1 change: 0 additions & 1 deletion app/assets/javascripts/components/export_case/export_case_controller.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,6 @@ angular.module('QuepidApp')
}
else if ( options.which === 'snapshot' ) {
$log.info('Selected "snapshot" as export option.');
$log.info('Exporting snapshot ' + options.snapshot_snapshot + '.');
snapshotId = options.snapshot_snapshot;
// Snapshot Name Snapshot Time Case ID Query Text Doc ID Doc Position

Expand Down
11 changes: 8 additions & 3 deletions app/assets/javascripts/components/import_ratings/_modal.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,10 +132,12 @@ <h2>CSV</h2>
<span class="help-block">
The CSV file should have the headers: <code>Snapshot Name,Snapshot Time,Case ID,Query Text,Doc ID,Doc Position</code> and lines similar to:
<pre>
Snapshot: testsnap1,2023-05-12T18:22:22.245Z,6,test,527641,1
Snapshot: testsnap1,2023-05-12T18:22:22.245Z,6,test,9426,2
Snapshot: testsnap1,2023-05-12T18:22:22.245Z,6,test,1921,3
Snapshot: Baseline,2023-05-12T18:22:22.245Z,6,star wars,527641,1
Snapshot: Baseline,2023-05-12T18:22:22.245Z,6,star wars,9426,2
Snapshot: Baseline,2023-05-12T18:22:22.245Z,6,star wars,1921,3
</pre>
Please note that the <code>Case ID</code> column is ignored so you can export from one case and import into another.
Any additional columns will be treated as document fields and stored in the Snapshot.
</span>
<p>Select Snapshot CSV file to import:</p>

Expand Down Expand Up @@ -173,6 +175,9 @@ <h2>CSV</h2>
<span ng-show="ctrl.informationNeedsTypePicked()">
This operation WILL override your existing information needs. Proceed with caution!
</span>
<span ng-show="ctrl.snapshotTypePicked()">
This operation WILL replace any snapshots you have created that have the same Snapshot Name in the csv.
</span>
</p>

<i class="fa fa-spinner fa-spin" aria-hidden="true" ng-show="ctrl.loading"></i>
Expand Down
12 changes: 7 additions & 5 deletions app/assets/javascripts/components/import_ratings/import_ratings_controller.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,11 +39,13 @@ angular.module('QuepidApp')
modalInstance.result.then(
function (response) {
if (!response.error) {
queriesSvc.reset();
queriesSvc.bootstrapQueries(ctrl.acase.caseNo)
.then(function () {
queriesSvc.searchAll();
});
if (response.message !== 'Snapshots imported successfully!') {
queriesSvc.reset();
queriesSvc.bootstrapQueries(ctrl.acase.caseNo)
.then(function () {
queriesSvc.searchAll();
});
}

flash.success = response.message;
} else {
Expand Down
86 changes: 46 additions & 40 deletions app/assets/javascripts/components/import_ratings/import_ratings_modal_instance_controller.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,8 @@ angular.module('QuepidApp')
'importRatingsSvc',
'theCase',
'querySnapshotSvc',
function ($scope, $uibModalInstance, importRatingsSvc, theCase, querySnapshotSvc) {
'caseCSVSvc',
function ($scope, $uibModalInstance, importRatingsSvc, theCase, querySnapshotSvc, caseCSVSvc) {
var ctrl = this;

ctrl.theCase = theCase;
Expand Down Expand Up @@ -75,7 +76,7 @@ angular.module('QuepidApp')
ctrl.options.which = 'snapshots';
ctrl.snapshots.import.alert = undefined;
ctrl.checkSnapshotHeaders();
ctrl.checkSnapshotBody();
//ctrl.checkSnapshotBody();
}
}, true);

Expand Down Expand Up @@ -127,47 +128,34 @@ angular.module('QuepidApp')
ctrl.informationNeedsTypePicked = function () {
return (ctrl.options.which === 'information_needs');
};

ctrl.snapshotTypePicked = function () {
return (ctrl.options.which === 'snapshots');
};

ctrl.ok = function () {
if (ctrl.options.which === 'snapshots') {
var headers = ctrl.snapshots.content.split('\n')[0];
headers = headers.split(ctrl.snapshots.separator);

var expectedHeaders = [
'Snapshot Name', 'Snapshot Time', 'Case ID', 'Query Text', 'Doc ID', 'Doc Position'
];

if (!angular.equals(headers, expectedHeaders)) {
var alert = 'Headers mismatch! Please make sure you have the correct headers in you file (check for correct spelling and capitalization): ';
alert += '<br /><strong>';
alert += expectedHeaders.join(',');
alert += '</strong>';

ctrl.snapshots.import.alert = {
'text': alert,
'type': 'text-danger'
};
} else {
ctrl.snapshots.import.loading = true;
querySnapshotSvc.importSnapshotsToSpecificCase(ctrl.snapshots.result, theCase.caseNo)
.then(function () {
var result = {
success: true,
message: 'Snapshots imported successfully!',
};
ctrl.snapshots.import.loading = false;
$uibModalInstance.close(result);
}, function () {
var result = {
error: true,
message: 'Could not import snapshots successfully! Please try again.',
};

ctrl.snapshots.import.loading = false;
$uibModalInstance.close(result);
});
}

//ctrl.checkCSVHeaders();

ctrl.snapshots.import.loading = true;
querySnapshotSvc.importSnapshotsToSpecificCase(ctrl.snapshots.result, theCase.caseNo)
.then(function () {
var result = {
success: true,
message: 'Snapshots imported successfully!',
};
ctrl.snapshots.import.loading = false;
$uibModalInstance.close(result);
}, function () {
var result = {
error: true,
message: 'Could not import snapshots successfully! Please try again.',
};

ctrl.snapshots.import.loading = false;
$uibModalInstance.close(result);
});

} else if (ctrl.options.which === 'csv') {
ctrl.checkCSVHeaders();
ctrl.checkCSVBody();
Expand Down Expand Up @@ -338,6 +326,24 @@ angular.module('QuepidApp')
}
};

ctrl.checkSnapshotHeaders = function () {
var headers = ctrl.snapshots.content.split('\n')[0];
headers = headers.split(ctrl.snapshots.separator);

var expectedHeaders = [
'Snapshot Name', 'Snapshot Time', 'Case ID', 'Query Text', 'Doc ID', 'Doc Position'
];

if (!caseCSVSvc.arrayContains(headers, expectedHeaders)) {
var alert = 'Required headers mismatch! Please make sure you have the correct headers in you file (check for correct spelling and capitalization): ';
alert += '<br /><strong>';
alert += expectedHeaders.join(',');
alert += '</strong>';

ctrl.snapshots.import.alert = alert;
}
};

ctrl.checkInformationNeedsHeaders = function () {
var headers = ctrl.information_needs.content.split('\n')[0];
headers = headers.split(ctrl.information_needs.separator);
Expand Down
46 changes: 43 additions & 3 deletions app/assets/javascripts/services/caseCSVSvc.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
self.stringifyQueriesDetailed = stringifyQueriesDetailed;
self.stringifySnapshot = stringifySnapshot;
self.formatDownloadFileName = formatDownloadFileName;
self.arrayContains = arrayContains;

function caseHeaderToCSV () {
var header = [
Expand Down Expand Up @@ -88,7 +89,7 @@
return '' + headerString + EOL;
}

function snapshotHeaderToCSV () {
function snapshotHeaderToCSV (fieldList) {
var header = [
'Snapshot Name',
'Snapshot Time',
Expand All @@ -97,6 +98,10 @@
'Doc ID',
'Doc Position',
];

angular.forEach(fieldList, function(fieldName) {
header.push(fieldName);
});

var headerString = header.join(',');

Expand Down Expand Up @@ -385,9 +390,16 @@
const snapshotTime = snapshot.time;
const caseNumber = aCase.caseNo;
let csvContent = '';

var fields = [];
angular.forEach(snapshot.docs, function (docs) {
angular.forEach(docs, function (doc) {
fields = mergeArrays(fields, Object.keys(doc.fields));
});
});

if (withHeader) {
csvContent += self.snapshotHeaderToCSV();
csvContent += self.snapshotHeaderToCSV(fields);
}
angular.forEach(snapshot.docs, function (docs, queryId) {
const queryIdToMatch = parseInt(queryId, 10);
Expand All @@ -405,6 +417,11 @@
infoArray.push(stringifyField(matchingQueryText));
infoArray.push(stringifyField(doc.id));
infoArray.push(stringifyField(idx + 1));

angular.forEach(fields, function (field) {
infoArray.push(stringifyField(doc.fields[field]));
});

csvContent += infoArray.join(',') + EOL;
});
}
Expand All @@ -420,13 +437,36 @@
* @param aCase
*
*/

function formatDownloadFileName (fileName) {
var downloadFileName = fileName.replace(/ /g,'_').replace(/:/g,'_');

return downloadFileName;
}

function mergeArrays(...arrays) {
var mergedArray = [];

arrays.forEach(function(array) {
array.forEach(function(value) {
if (!mergedArray.includes(value)) {
mergedArray.push(value);
}
});
});

return mergedArray;
}

function arrayContains(containingArray, subsetArray){
subsetArray.forEach(function(value) {
if (!containingArray.includes(value)) {
return false;
}
});
return true;
}


}
]);
})();
18 changes: 16 additions & 2 deletions app/assets/javascripts/services/querySnapshotSvc.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -184,8 +184,22 @@ angular.module('QuepidApp')
}

var query = snapshot.queries[doc['Query Text']];

query.docs.push( { 'id': doc['Doc ID'], 'position': doc['Doc Position'] } );

var docPayload = { 'id': doc['Doc ID'], 'position': doc['Doc Position'] };

// Remove the properties of the doc that exist elsewhere.
delete doc['Doc ID'];
delete doc['Doc Position'];

delete doc['Snapshot Name'];
delete doc['Snapshot Time'];
delete doc['Case ID'];
delete doc['Query Text'];

// map any remaining properties of the doc as fields.
docPayload['fields'] = doc;

query.docs.push(docPayload );
});

function callApi (caseId, snapshotData) {
Expand Down
7 changes: 6 additions & 1 deletion app/views/api/v1/snapshots/_snapshot.json.jbuilder
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,12 @@ if with_docs
json.docs do
snapshot.snapshot_queries.each do |snapshot_query|
docs = snapshot_query.snapshot_docs.map do |doc|
{ id: doc.doc_id, explain: doc.explain, rated_only: doc.rated_only }
{
id: doc.doc_id,
explain: doc.explain.blank? ? nil : JSON.parse(doc.explain),
rated_only: doc.rated_only,
fields: doc.fields.blank? ? nil : JSON.parse(doc.fields),
}
end

json.set! snapshot_query.query_id, docs
Expand Down
32 changes: 16 additions & 16 deletions test/controllers/api/v1/snapshots_controller_test.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,12 @@ class SnapshotsControllerTest < ActionController::TestCase
name: 'New Snapshot',
docs: {
first_query.id => [
{ id: 'doc1', explain: '1' },
{ id: 'doc2', explain: '2' }
{ id: 'doc1', explain: 1 },
{ id: 'doc2', explain: 2 }
],
second_query.id => [
{ id: 'doc3', explain: '3' },
{ id: 'doc4', explain: '4' }
{ id: 'doc3', explain: 3 },
{ id: 'doc4', explain: 4 }
],
},
queries: {
Expand Down Expand Up @@ -79,8 +79,8 @@ class SnapshotsControllerTest < ActionController::TestCase
name: 'New Snapshot',
docs: {
first_query.id => [
{ id: 'doc1', explain: '1' },
{ id: 'doc2', explain: '2' }
{ id: 'doc1', explain: 1 },
{ id: 'doc2', explain: 2 }
],
# in Rails 4, we could do second_query.id => [] and getting the second_query in,
# but in Rails 5, the second_query doesn't show up because the array that is empty
Expand All @@ -107,8 +107,8 @@ class SnapshotsControllerTest < ActionController::TestCase

snapshot = response.parsed_body

assert_equal snapshot['name'], data[:snapshot][:name]
assert_equal snapshot['docs'].length, data[:snapshot][:docs].length
assert_equal data[:snapshot][:name], snapshot['name']
assert_equal data[:snapshot][:docs].length, snapshot['docs'].length

data_doc = data[:snapshot][:docs][first_query.id][0]
response_doc = snapshot['docs'][first_query.id.to_s][0]
Expand Down Expand Up @@ -148,12 +148,12 @@ class SnapshotsControllerTest < ActionController::TestCase
name: '',
docs: {
first_query.id => [
{ id: 'doc1', explain: '1' },
{ id: 'doc2', explain: '2' }
{ id: 'doc1', explain: 1 },
{ id: 'doc2', explain: 2 }
],
second_query.id => [
{ id: 'doc3', explain: '3' },
{ id: 'doc4', explain: '4' }
{ id: 'doc3', explain: 3 },
{ id: 'doc4', explain: 4 }
],
},
queries: {
Expand Down Expand Up @@ -181,12 +181,12 @@ class SnapshotsControllerTest < ActionController::TestCase
snapshot: {
docs: {
first_query.id => [
{ id: 'doc1', explain: '1' },
{ id: 'doc2', explain: '2' }
{ id: 'doc1', explain: 1 },
{ id: 'doc2', explain: 2 }
],
second_query.id => [
{ id: 'doc3', explain: '3' },
{ id: 'doc4', explain: '4' }
{ id: 'doc3', explain: 3 },
{ id: 'doc4', explain: 4 }
],
},
queries: {
Expand Down
Loading

0 comments on commit 5a4c579

Please sign in to comment.