Skip to content

Commit

Permalink
Preload data (#5667)
Browse files Browse the repository at this point in the history 
* Encode types of variables into their encoded names to preload data for each analysis

Implements jasp-stats/INTERNAL-jasp#2583

Has corresponding changes in jaspBase, jaspModuleEncoder and jaspDescriptives

fix some merge errors and bugs

add syntaxhighlighting to computecolumn R display

move focus to button on apply filter so text-entry is editFinished

columntype in filterconstructor now follows dataset changes

filterconstructor now has changeable columntypes!

force active focus on apply button in filter and comp col to focus out of constructor and trigger proper checks

remove last remnants of "keeping/converting columntypes"

started work on preloading data config in Description.qml

Add preloadData argument to analysisrunflow and decode to type

* set default off for preloadData

* fix annoying "computed column changed wanna save" popping up too much

* add some info on type changing in computed column help

* add some info on changing types to filter help
  • Loading branch information
@JorisGoosen
JorisGoosen authored Sep 30, 2024
1 parent 1d8fb2e commit 62b9544
Show file tree
Hide file tree
Showing 43 changed files with 495 additions and 367 deletions.
2 changes: 1 addition & 1 deletion Common/jaspColumnEncoder
Submodule jaspColumnEncoder updated 3 files
+120 −21 columnencoder.cpp
+15 −8 columnencoder.h
+6 −4 enumutilities.h
21 changes: 3 additions & 18 deletions CommonData/column.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ void Column::dbLoad(int id, bool getValues)
Json::Value emptyVals;

db().columnGetBasicInfo( _id, _name, _title, _description, _type, _revision, emptyVals, _autoSortByValue);
db().columnGetComputedInfo( _id, _analysisId, _invalidated, _forceTypes, _codeType, _rCode, _error, _constructorJson);
db().columnGetComputedInfo( _id, _analysisId, _invalidated, _codeType, _rCode, _error, _constructorJson);

_emptyValues->fromJson(emptyVals);

Expand Down Expand Up @@ -204,7 +204,7 @@ bool Column::setCustomEmptyValues(const stringset& customEmptyValues)

void Column::dbUpdateComputedColumnStuff()
{
db().columnSetComputedInfo(_id, _analysisId, _invalidated, _forceTypes, _codeType, _rCode, _error, constructorJsonStr());
db().columnSetComputedInfo(_id, _analysisId, _invalidated, _codeType, _rCode, _error, constructorJsonStr());
incRevision();
}

Expand All @@ -220,18 +220,6 @@ void Column::setInvalidated(bool invalidated)
incRevision(false);
}

void Column::setForceType(bool force)
{
JASPTIMER_SCOPE(Column::setForceType);

if(_forceTypes == force)
return;

_forceTypes = force;
db().columnSetForceSourceColType(_id, _forceTypes);
incRevision(false);
}

void Column::setCodeType(computedColumnType codeType)
{
JASPTIMER_SCOPE(Column::setCodeType);
Expand Down Expand Up @@ -361,7 +349,6 @@ void Column::setCompColStuff(bool invalidated, bool forceSourceColType, computed
JASPTIMER_SCOPE(Column::setCompColStuff);

_invalidated = invalidated;
_forceTypes = forceSourceColType;
_codeType = codeType;
_rCode = rCode;
_constructorJson = constructorJson;
Expand Down Expand Up @@ -1935,7 +1922,6 @@ Json::Value Column::serialize() const
json["constructorJson"] = _constructorJson;
json["autoSortByValue"] = _autoSortByValue;
json["description"] = _description;
json["forceTypes"] = _forceTypes;
json["codeType"] = int(_codeType);
json["error"] = _error;
json["type"] = int(_type);
Expand Down Expand Up @@ -2077,15 +2063,14 @@ void Column::deserialize(const Json::Value &json)
db().columnSetType(_id, _type);

_invalidated = json["invalidated"].asBool();
_forceTypes = json["forceTypes"].asBool();
_codeType = computedColumnType(json["codeType"].asInt());
_rCode = json["rCode"].asString();
_error = json["error"].asString();
_analysisId = json["analysisId"].asInt();
_constructorJson = json["constructorJson"];
_autoSortByValue = json["autoSortByValue"].asBool();

db().columnSetComputedInfo(_id, _analysisId, _invalidated, _forceTypes, _codeType, _rCode, _error, constructorJsonStr());
db().columnSetComputedInfo(_id, _analysisId, _invalidated, _codeType, _rCode, _error, constructorJsonStr());

deserializeLabelsForCopy(json["labels"]);

Expand Down
3 changes: 0 additions & 3 deletions CommonData/column.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,6 @@ class Column : public DataSetBaseNode
void setAnalysisId( int analysisId );
void setIndex( int index );
void setInvalidated( bool invalidated );
void setForceType( bool force );
void setCompColStuff( bool invalidated, bool forceSourceColType, computedColumnType codeType, const std::string & rCode, const std::string & error, const Json::Value & constructorJson);
void setDefaultValues( enum columnType columnType = columnType::unknown);

Expand All @@ -94,7 +93,6 @@ class Column : public DataSetBaseNode
bool isComputed() const { return _codeType != computedColumnType::notComputed && _codeType != computedColumnType::analysisNotComputed; }
bool invalidated() const { return _invalidated; }
bool autoSortByValue() const { return _autoSortByValue; }
bool forceTypes() const { return _forceTypes; }
computedColumnType codeType() const { return _codeType; }
const std::string & name() const { return _name; }
const std::string & title() const { return _title.empty() ? _name : _title; }
Expand Down Expand Up @@ -258,7 +256,6 @@ class Column : public DataSetBaseNode
doublevec _labelsTempDbls;
strintmap _labelsTempToIndex;
bool _invalidated = false,
_forceTypes = true, ///< If this is a computed column this means whether the source columns used in a computed columns calculation should be forcefully loaded as the desired type or just as their own.
_autoSortByValue;
computedColumnType _codeType = computedColumnType::notComputed;
std::string _name,
Expand Down
33 changes: 13 additions & 20 deletions CommonData/databaseinterface.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,12 @@ void DatabaseInterface::upgradeDBFromVersion(Version originalVersion)
if (!tableHasColumn("DataSets", "dataFileTimestamp"))
runStatements("ALTER TABLE DataSets ADD COLUMN dataFileTimestamp INT;");
}

if(originalVersion <= "0.19.2")
{
if (tableHasColumn("Columns", "forceSourceColType"))
runStatements("ALTER TABLE Columns DROP COLUMN forceSourceColType;");
}

transactionWriteEnd();
}
Expand Down Expand Up @@ -66,7 +72,7 @@ int DatabaseInterface::dataSetInsert(const std::string & dataFilePath, long data
std::function<void(sqlite3_stmt *stmt)> prepare = [&](sqlite3_stmt *stmt)
{
sqlite3_bind_text(stmt, 1, dataFilePath.c_str(), dataFilePath.length(), SQLITE_TRANSIENT);
sqlite3_bind_int(stmt, 2, dataFileTimestamp);
sqlite3_bind_int(stmt, 2, dataFileTimestamp);
sqlite3_bind_text(stmt, 3, description.c_str(), description.length(), SQLITE_TRANSIENT);
sqlite3_bind_text(stmt, 4, databaseJson.c_str(), databaseJson.length(), SQLITE_TRANSIENT);
sqlite3_bind_text(stmt, 5, emptyValuesJson.c_str(), emptyValuesJson.length(), SQLITE_TRANSIENT);
Expand Down Expand Up @@ -913,17 +919,6 @@ void DatabaseInterface::columnSetInvalidated(int columnId, bool invalidated)
});
}


void DatabaseInterface::columnSetForceSourceColType(int columnId, bool force)
{
JASPTIMER_SCOPE(DatabaseInterface::columnSetForceSourceColType);
runStatements("UPDATE Columns SET forceSourceColType=? WHERE id=?;", [&](sqlite3_stmt * stmt)
{
sqlite3_bind_int(stmt, 1, int(force));
sqlite3_bind_int(stmt, 2, columnId);
});
}

void DatabaseInterface::columnSetIndex(int columnId, int index)
{
JASPTIMER_SCOPE(DatabaseInterface::columnSetIndex);
Expand Down Expand Up @@ -1000,11 +995,11 @@ void DatabaseInterface::columnSetDescription(int columnId, const std::string & d
});
}

void DatabaseInterface::columnSetComputedInfo(int columnId, int analysisId, bool invalidated, bool forceSourceColType, computedColumnType codeType, const std::string & rCode, const std::string & error, const std::string & constructorJsonStr)
void DatabaseInterface::columnSetComputedInfo(int columnId, int analysisId, bool invalidated, computedColumnType codeType, const std::string & rCode, const std::string & error, const std::string & constructorJsonStr)
{
JASPTIMER_SCOPE(DatabaseInterface::columnSetComputedInfo);

runStatements("UPDATE Columns SET invalidated=?, codeType=?, rCode=?, error=?, constructorJson=?, analysisId=?, forceSourceColType=? WHERE id=?;", [&](sqlite3_stmt * stmt)
runStatements("UPDATE Columns SET invalidated=?, codeType=?, rCode=?, error=?, constructorJson=?, analysisId=? WHERE id=?;", [&](sqlite3_stmt * stmt)
{
std::string codeT = computedColumnTypeToString(codeType);

Expand All @@ -1014,8 +1009,7 @@ void DatabaseInterface::columnSetComputedInfo(int columnId, int analysisId, bool
sqlite3_bind_text(stmt, 4, error.c_str(), error.length(), SQLITE_TRANSIENT);
sqlite3_bind_text(stmt, 5, constructorJsonStr.c_str(), constructorJsonStr.length(), SQLITE_TRANSIENT);
sqlite3_bind_int(stmt, 6, analysisId);
sqlite3_bind_int(stmt, 7, int(forceSourceColType));
sqlite3_bind_int(stmt, 8, columnId);
sqlite3_bind_int(stmt, 7, columnId);
});
}

Expand Down Expand Up @@ -1060,7 +1054,7 @@ std::string DatabaseInterface::_wrap_sqlite3_column_text(sqlite3_stmt * stmt, in
return !col ? "" : std::string(reinterpret_cast<const char*>(col));
}

void DatabaseInterface::columnGetComputedInfo(int columnId, int &analysisId, bool &invalidated, bool & forceSourceColType, computedColumnType &codeType, std::string &rCode, std::string &error, Json::Value &constructorJson)
void DatabaseInterface::columnGetComputedInfo(int columnId, int &analysisId, bool &invalidated, computedColumnType &codeType, std::string &rCode, std::string &error, Json::Value &constructorJson)
{
JASPTIMER_SCOPE(DatabaseInterface::columnGetComputedInfo);

Expand All @@ -1073,15 +1067,14 @@ void DatabaseInterface::columnGetComputedInfo(int columnId, int &analysisId, boo
{
int colCount = sqlite3_column_count(stmt);

assert(colCount == 7);
assert(colCount == 6);

invalidated = sqlite3_column_int( stmt, 0);
std::string codeTypeStr = _wrap_sqlite3_column_text(stmt, 1);
rCode = _wrap_sqlite3_column_text(stmt, 2);
error = _wrap_sqlite3_column_text(stmt, 3);
std::string constructorJsonStr = _wrap_sqlite3_column_text(stmt, 4);
analysisId = sqlite3_column_int( stmt, 5);
forceSourceColType = sqlite3_column_int( stmt, 6);

codeType = computedColumnType::notComputed;
if (!codeTypeStr.empty())
Expand All @@ -1094,7 +1087,7 @@ void DatabaseInterface::columnGetComputedInfo(int columnId, int &analysisId, boo
Json::Reader().parse(constructorJsonStr, constructorJson);
};

runStatements("SELECT invalidated, codeType, rCode, error, constructorJson, analysisId, forceSourceColType FROM Columns WHERE id = ?;", prepare, processRow);
runStatements("SELECT invalidated, codeType, rCode, error, constructorJson, analysisId FROM Columns WHERE id = ?;", prepare, processRow);
}

void DatabaseInterface::labelsClear(int columnId)
Expand Down
5 changes: 2 additions & 3 deletions CommonData/databaseinterface.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,14 +126,13 @@ class DatabaseInterface
void columnSetType( int columnId, columnType colType);
void columnSetAutoSort( int columnId, bool sort);
void columnSetInvalidated( int columnId, bool invalidated);
void columnSetForceSourceColType(int columnId, bool force);
void columnSetName( int columnId, const std::string & name);
void columnSetTitle( int columnId, const std::string & title);
void columnSetEmptyVals( int columnId, const std::string & emptyValsJson);
void columnSetDescription( int columnId, const std::string & description);
void columnGetBasicInfo( int columnId, std::string & name, std::string & title, std::string & description, columnType & colType, int & revision, Json::Value & emptyValuesJson, bool & autoSort);
void columnSetComputedInfo( int columnId, int analysisId, bool invalidated, bool forceSourceColType, computedColumnType codeType, const std::string & rCode, const std::string & error, const std::string & constructorJson);
void columnGetComputedInfo( int columnId, int &analysisId, bool & invalidated, bool & forceSourceColType, computedColumnType & codeType, std::string & rCode, std::string & error, Json::Value & constructorJson);
void columnSetComputedInfo( int columnId, int analysisId, bool invalidated, computedColumnType codeType, const std::string & rCode, const std::string & error, const std::string & constructorJson);
void columnGetComputedInfo( int columnId, int &analysisId, bool & invalidated, computedColumnType & codeType, std::string & rCode, std::string & error, Json::Value & constructorJson);
void columnSetValues( int columnId, const intvec & ints, const doublevec & dbls);
void columnSetValue( int columnId, size_t row, int valueInt, double valueDbl);
intvec columnGetLabelIds( int columnId);
Expand Down
2 changes: 1 addition & 1 deletion Desktop/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ qt_add_executable(

set(
QML_IMPORT_PATH
"${PROJECT_SOURCE_DIR}/Desktop/components"
"${PROJECT_SOURCE_DIR}/Desktop/components"
"${PROJECT_SOURCE_DIR}/QMLComponents/components"
CACHE
PATH
Expand Down
1 change: 1 addition & 0 deletions Desktop/analysis/analysis.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -558,6 +558,7 @@ Json::Value Analysis::createAnalysisRequestJson()
json["typeRequest"] = engineStateToString(engineState::analysis);
json["id"] = int(id());
json["perform"] = performTypeToString(perform);
json["preloadData"] = _moduleData ? _moduleData->preloadData() : true;
json["revision"] = revision();
json["rfile"] = _moduleData == nullptr ? rfile() : "";
json["dynamicModuleCall"] = _moduleData == nullptr ? "" : _moduleData->getFullRCall();
Expand Down
39 changes: 4 additions & 35 deletions Desktop/components/JASP/Widgets/ComputeColumnWindow.qml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -162,16 +162,7 @@ FocusScope
anchors.fill: parent
anchors.leftMargin: 1
visible: !computedColumnsInterface.computeColumnUsesRCode
forceColumnInputs: !computedColumnsInterface.computeColumnForceType
? ""
: computedColumnsInterface.columnType === columnTypeScale
? "scale"
: computedColumnsInterface.columnType === columnTypeOrdinal
? "ordinal"
: computedColumnsInterface.columnType === columnTypeNominal
? "nominal"
: "scale"


showGeneratedRCode: false
KeyNavigation.tab: applyComputedColumnButton

Expand Down Expand Up @@ -298,40 +289,18 @@ FocusScope

onClicked: computedColumnConstructor.showGeneratedRCode = !computedColumnConstructor.showGeneratedRCode
}

JaspControls.RectangularButton
{
id: forceSourceColTypeButton

toolTip: qsTr("- Keeping types: use columns with their defined columntype.\n- Converting types: convert all used columns to the columntype of this computed column before use.\n\nThe button displays the *current setting*, not what will happen when you press it!")
text: !computedColumnsInterface.computeColumnForceType
? qsTr("Keeping types")
: qsTr("Converting types")

anchors.left: showGeneratedRCode.right
anchors.bottom: parent.bottom
anchors.top: helpButton.top

onClicked:
{
computedColumnsInterface.computeColumnForceType = !computedColumnsInterface.computeColumnForceType
computedColumnContainer.applyComputedColumn()
}
}




JaspControls.RectangularButton
{
id: applyComputedColumnButton

text: qsTr("Compute column")
anchors.left: forceSourceColTypeButton.right
anchors.left: showGeneratedRCode.right
anchors.right: helpButton.left
centerTextParent: true
anchors.bottom: parent.bottom
anchors.top: helpButton.top
onClicked: computedColumnContainer.applyComputedColumn()
onClicked: { forceActiveFocus(); computedColumnContainer.applyComputedColumn() }
toolTip: qsTr("Click to compute column")

}
Expand Down
25 changes: 14 additions & 11 deletions Desktop/components/JASP/Widgets/FilterConstructor/ColumnDrag.qml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,21 +2,24 @@ import QtQuick 2.0

DragGeneric
{
property string columnName: "?"
property string columnIcon: ""
property string columnName: "?"
property int columnTypeUser: -1

shownChild: showMe
dragKeys: showMe.dragKeys

property alias maxSize: showMe.maxSize
shownChild: showMe
dragKeys: showMe.dragKeys
property bool acceptsDrops: true
property alias maxSize: showMe.maxSize

toolTipText: acceptsDrops ? showMe.toolTip : ""

JASPColumn
{
id: showMe
columnName: parent.columnName
columnIcon: parent.columnIcon
id: showMe
columnName: parent.columnName
columnTypeUser: parent.columnTypeUser
changeTypeAllowed: parent.acceptsDrops

x: parent.dragX
y: parent.dragY
x: parent.dragX
y: parent.dragY
}
}
39 changes: 0 additions & 39 deletions Desktop/components/JASP/Widgets/FilterConstructor/ColumnsSelector.qml
View file

This file was deleted.

Loading

0 comments on commit 62b9544

Please sign in to comment.