From d47673a4cbbf5463bc006417f621f5f2115ce8e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Boull=C3=A9?= Date: Wed, 16 Oct 2024 10:24:55 +0200 Subject: [PATCH] Implement Khiops piloting using a json parameter file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reference - issue #230 Extending Khiops entries with a lightweight json structure for scenarios #230 - https://github.com/KhiopsML/khiops/issues/230 - cette issue contient les specification detaillees de la fonctionnalité, donc on pourra extraire la documentation - cf. commentaire "Extension du pilotage de Khiops via les scénarios" - https://github.com/KhiopsML/khiops/issues/230#issuecomment-2064075317 Choix d'implementation principaux - le fichier de parametre json est lu et traite en entier de facon prealable, avec une taille limitee - le fichier de commande est traite en flux, ce qui permet de n'avoir aucune limite de taille - toute ligne de commande peut etre commentee, y compris les lignes du langage de pilotage de type IF ou LOOP - les lignes d'un fichier de commande template n'ont pas besoin de se terminer par un commentaire - toute __key__ du fichier de commande doit se trouver dans le fichier json - toute key du fichier json doit etre utilise dans le fichier de commande - les __key__ de parametrage json ne peuvent concerner que la partie parametrage utilisateur d'une valeur - toute erreur ou incoherence dans les fichiers de commande et de parametrage json provoquent une erreur fatale CommandFile - methodes publiques - ReadInputCommand: refactoring et simplification - ReadWriteCommandFiles: pour ecrire le scenario en sortie sans rejouer les commandes - methodes privees principales de parsing des scenario et de leur langage - ResetParser - RecodeCurrentLineUsingJsonParameters - ParseInputCommand: analyse syntaxique des lignes d'un fichier de commande en entree - TokenizeInputCommand - GetFirstInputToken - variables de travail du parser - prefixees par parser (ex: nParserState, sParserBlockKey, nParserLineIndex...) - maintenues le temps d'une session d'analyse, pour gerer le parser de scenario en flux - contraintes - nMaxLineLength = 500: longueur max d'une ligne de fichier de commande - nLoopMaxLineNumber = 1000: Taille max d'un bloc d'instruction IF ou LOOP d'un fichier de commande - constantes: renommees et etendues pour harmonisation - sCommentPrefix - sJsonKeyDelimiter - sByteJsonKeyPrefix - ... JsonLex.lex - STRINGERROR: variante de STRINGVALUE dans le cas d'erreur d'encodage utf8 JsonYac.yac - regles impliquant STRINGERROR pour avoir des erreurs interpretable en cas de STRINGERROR UIObject: prise en compte de l'option -O, pour les scenarios en sortie sans rejouer les commandes - ParseMainParameters - CheckCommandLineOptions TextService: - parametrage avance de la conversion de lencodage Json vers un encodage C pas necessairemernet Utf8 pour le cas des rapport Khiops - Set|GetForceUnicodeToAnsi - Set|GetForceUtf8ToAnsi - modifie le comportement de la methode JsonToCString utilisee par le parser de Json - utilise uniquement dans CCCoclusteringReport LearningTestTool - kht_test.py - --nop-output-scenario: "create an output scenario nop_output_test.prm in results dir, without replaying commands" - _kht_families - ajout de la famille JsonParameters Ajout de tests unitaires: test\UnitTests\Norm\results.ref\base_TextService.txt Ajout de test CI/CD: test\LearningTest\TestKhiops\Standard\JsonSpliceJunction Batterie de tests LearningTest\TestKhiops\JsonParameters - une petite cinquantaine de test, les trois-quart concernant la detection des erreurs --- src/Learning/MODL/MODL.cpp | 3 +- .../CCCoclusteringReport.cpp | 37 +- .../MODL_Coclustering/MODL_Coclustering.cpp | 6 +- src/Norm/base/CommandFile.cpp | 1465 +++++++++++++++-- src/Norm/base/CommandFile.h | 252 ++- src/Norm/base/JSONTokenizer.cpp | 42 +- src/Norm/base/JSONTokenizer.h | 12 +- src/Norm/base/JsonLex.inc | 27 +- src/Norm/base/JsonLex.lex | 15 +- src/Norm/base/JsonYac.cpp | 318 ++-- src/Norm/base/JsonYac.hpp | 9 +- src/Norm/base/JsonYac.yac | 66 +- src/Norm/base/TextService.cpp | 130 +- src/Norm/base/TextService.h | 43 +- src/Norm/base/UIObject.cpp | 127 +- src/Norm/base/UIUnit.cpp | 46 +- src/Norm/base/UserInterface.h | 5 +- .../results.ref/C100_AnalysisResults.khj | 645 ++++++++ .../C100_AnalysisResults.model.kdic | 48 + .../JsonSpliceJunction/results.ref/err.txt | 11 + .../JsonSpliceJunction/results.ref/time.log | 1 + .../Standard/JsonSpliceJunction/test.json | 18 + .../Standard/JsonSpliceJunction/test.prm | 48 + test/LearningTestTool/README.md | 2 +- .../py/_kht_results_management.py | 2 +- test/LearningTestTool/py/kht_test.py | 12 + .../Norm/results.ref/base_TextService.txt | 4 +- 27 files changed, 2870 insertions(+), 524 deletions(-) create mode 100644 test/LearningTest/TestKhiops/Standard/JsonSpliceJunction/results.ref/C100_AnalysisResults.khj create mode 100644 test/LearningTest/TestKhiops/Standard/JsonSpliceJunction/results.ref/C100_AnalysisResults.model.kdic create mode 100644 test/LearningTest/TestKhiops/Standard/JsonSpliceJunction/results.ref/err.txt create mode 100644 test/LearningTest/TestKhiops/Standard/JsonSpliceJunction/results.ref/time.log create mode 100644 test/LearningTest/TestKhiops/Standard/JsonSpliceJunction/test.json create mode 100644 test/LearningTest/TestKhiops/Standard/JsonSpliceJunction/test.prm diff --git a/src/Learning/MODL/MODL.cpp b/src/Learning/MODL/MODL.cpp index e76e8fed..7a2de562 100644 --- a/src/Learning/MODL/MODL.cpp +++ b/src/Learning/MODL/MODL.cpp @@ -34,11 +34,12 @@ int main(int argc, char** argv) // Activation de la gestion des signaux via des erreurs, pour afficher des messages d'erreur explicites // A potentiellement commenter sur certains IDE lors des phases de debuggage - Global::ActivateSignalErrorManagement(); + //Global::ActivateSignalErrorManagement(); // Choix du repertoire de lancement pour le debugage sous Windows (a commenter apres fin du debug) // SetWindowsDebugDir("Standard", "IrisLight"); // SetWindowsDebugDir("Standard", "IrisU2D"); + // SetWindowsDebugDir("JsonParameters", "JsonSpliceJunctionAll"); // Parametrage des logs memoires depuis les variables d'environnement, pris en compte dans KWLearningProject // KhiopsMemStatsLogFileName, KhiopsMemStatsLogFrequency, KhiopsMemStatsLogToCollect diff --git a/src/Learning/MODL_Coclustering/CCCoclusteringReport.cpp b/src/Learning/MODL_Coclustering/CCCoclusteringReport.cpp index d0396c5b..b2a54024 100644 --- a/src/Learning/MODL_Coclustering/CCCoclusteringReport.cpp +++ b/src/Learning/MODL_Coclustering/CCCoclusteringReport.cpp @@ -51,7 +51,8 @@ boolean CCCoclusteringReport::ReadGenericReport(const ALString& sFileName, CCHie boolean bOk; int nFileFormat; ALString sKhiopsEncoding; - boolean bForceAnsi; + boolean bForceUnicodeToAnsi; + boolean bForceUtf8ToAnsi; require(sFileName != ""); require(coclusteringDataGrid != NULL); @@ -69,19 +70,19 @@ boolean CCCoclusteringReport::ReadGenericReport(const ALString& sFileName, CCHie // ansi if (sKhiopsEncoding == "") { - bForceAnsi = false; + bForceUnicodeToAnsi = false; AddWarning("The \"khiops_encoding\" field is missing in the read coclustering file. " "The coclustering file is deprecated, and may raise encoding problems " "in case of mixed ansi and utf8 chars " ": see the Khiops guide for more information."); } else if (sKhiopsEncoding == "ascii" or sKhiopsEncoding == "utf8") - bForceAnsi = false; + bForceUnicodeToAnsi = false; else if (sKhiopsEncoding == "ansi" or sKhiopsEncoding == "mixed_ansi_utf8") - bForceAnsi = true; + bForceUnicodeToAnsi = true; else if (sKhiopsEncoding == "colliding_ansi_utf8") { - bForceAnsi = false; + bForceUnicodeToAnsi = true; AddWarning("The \"khiops_encoding\" field is \"" + sKhiopsEncoding + "\" in the read coclustering file. " "This may raise encoding problems if the file has been modified outside of Khiops " @@ -89,7 +90,7 @@ boolean CCCoclusteringReport::ReadGenericReport(const ALString& sFileName, CCHie } else { - bForceAnsi = false; + bForceUnicodeToAnsi = false; AddWarning( "The value of the \"khiops_encoding\" field is \"" + sKhiopsEncoding + "\" in the read coclustering file. " @@ -98,10 +99,15 @@ boolean CCCoclusteringReport::ReadGenericReport(const ALString& sFileName, CCHie ": see the Khiops guide for more information."); } + // Parametrage de l'encodage ansi pour les caracteres Utf8 + bForceUtf8ToAnsi = sKhiopsEncoding == "ansi" or sKhiopsEncoding == "mixed_ansi_utf8"; + // Lecture du fichier en parametrant le json tokeniser correctement - JSONTokenizer::SetForceAnsi(bForceAnsi); + JSONTokenizer::SetForceUnicodeToAnsi(bForceUnicodeToAnsi); + JSONTokenizer::SetForceUtf8ToAnsi(bForceUtf8ToAnsi); bOk = ReadJSONReport(sFileName, coclusteringDataGrid); - JSONTokenizer::SetForceAnsi(false); + JSONTokenizer::SetForceUnicodeToAnsi(false); + JSONTokenizer::SetForceUtf8ToAnsi(false); } return bOk; } @@ -116,7 +122,8 @@ boolean CCCoclusteringReport::ReadGenericReportHeader(const ALString& sFileName, boolean bOk; int nFileFormat; ALString sKhiopsEncoding; - boolean bForceAnsi; + boolean bForceUnicodeToAnsi; + boolean bForceUtf8ToAnsi; require(sFileName != ""); require(coclusteringDataGrid != NULL); @@ -134,12 +141,16 @@ boolean CCCoclusteringReport::ReadGenericReportHeader(const ALString& sFileName, { // Analyse du type d'encodage pour determiner si on doit recoder les caracteres utf8 du fichier json en // ansi Pas de message pour cette lecture rapide - bForceAnsi = (sKhiopsEncoding == "ansi" or sKhiopsEncoding == "mixed_ansi_utf8"); + bForceUnicodeToAnsi = (sKhiopsEncoding == "ansi" or sKhiopsEncoding == "mixed_ansi_utf8" or + sKhiopsEncoding == "colliding_ansi_utf8"); + bForceUtf8ToAnsi = (sKhiopsEncoding == "ansi" or sKhiopsEncoding == "mixed_ansi_utf8"); // Lecture de l'entete du fichier en parametrant le json tokeniser correctement - JSONTokenizer::SetForceAnsi(bForceAnsi); + JSONTokenizer::SetForceUnicodeToAnsi(bForceUnicodeToAnsi); + JSONTokenizer::SetForceUtf8ToAnsi(bForceUtf8ToAnsi); bOk = ReadJSONReportHeader(sFileName, coclusteringDataGrid, nInstanceNumber, nCellNumber); - JSONTokenizer::SetForceAnsi(false); + JSONTokenizer::SetForceUnicodeToAnsi(false); + JSONTokenizer::SetForceUtf8ToAnsi(false); } return bOk; } @@ -371,7 +382,7 @@ int CCCoclusteringReport::DetectFileFormatAndEncoding(const ALString& sFileName, // Message d'erreur si pas de format reconnu if (nFileFormat == None) - AddError("Format of file " + sFileName + " should either " + + AddError("Format of file " + sFileName + " should be either " + CCCoclusteringReport::GetJSONReportSuffix() + " or " + CCCoclusteringReport::GetKhcReportSuffix()); diff --git a/src/Learning/MODL_Coclustering/MODL_Coclustering.cpp b/src/Learning/MODL_Coclustering/MODL_Coclustering.cpp index 5575afd1..fd707dc1 100644 --- a/src/Learning/MODL_Coclustering/MODL_Coclustering.cpp +++ b/src/Learning/MODL_Coclustering/MODL_Coclustering.cpp @@ -37,9 +37,9 @@ int main(int argc, char** argv) Global::ActivateSignalErrorManagement(); // Choix du repertoire de lancement pour le debugage sous Windows (a commenter apres fin du debug) - //SetWindowsDebugDir("y_CoclusteringIV_Standard", "IrisLight"); - //SetWindowsDebugDir("y_CoclusteringIV_Standard", "Iris"); - //SetWindowsDebugDir("Standard", "Iris"); + // SetWindowsDebugDir("y_CoclusteringIV_Standard", "IrisLight"); + // SetWindowsDebugDir("y_CoclusteringIV_Standard", "Iris"); + // SetWindowsDebugDir("Standard", "Iris"); // Point d'arret sur l'allocation d'un bloc memoire // MemSetAllocIndexExit(30406); diff --git a/src/Norm/base/CommandFile.cpp b/src/Norm/base/CommandFile.cpp index 228eeab2..00d9671d 100644 --- a/src/Norm/base/CommandFile.cpp +++ b/src/Norm/base/CommandFile.cpp @@ -6,29 +6,33 @@ CommandFile::CommandFile() { + bPrintOutputInConsole = false; fInputCommands = NULL; fOutputCommands = NULL; - bPrintOutputInConsole = false; + ResetParser(); + assert(nMaxLineLength < BUFFER_LENGTH); + assert(nMaxJsonKeyLength + nMaxStringValueLength < nMaxLineLength); } CommandFile::~CommandFile() { - require(not IsCommandFileOpened()); + require(not AreCommandFilesOpened()); + require(nParserState == TokenOther); } void CommandFile::Reset() { - require(not IsCommandFileOpened()); + require(not AreCommandFilesOpened()); SetInputCommandFileName(""); + SetInputParameterFileName(""); SetOutputCommandFileName(""); DeleteAllInputSearchReplaceValues(); - SetInputParameterFileName(""); } void CommandFile::SetInputCommandFileName(const ALString& sFileName) { - require(not IsCommandFileOpened()); + require(not AreCommandFilesOpened()); sInputCommandFileName = sFileName; } @@ -40,14 +44,14 @@ const ALString& CommandFile::GetInputCommandFileName() const void CommandFile::SetOutputCommandFileName(const ALString& sFileName) { - require(not IsCommandFileOpened()); + require(not AreCommandFilesOpened()); sOutputCommandFileName = sFileName; - // Si les fichier de sortie est /dev/stdout ou /dev/stderr, c'est un eredirection vers la console + // Si les fichier de sortie est /dev/stdout ou /dev/stderr, c'est une redirection vers la console bPrintOutputInConsole = false; #ifdef __linux_or_apple__ - if (sOutputCommandFileName == "/dev/stdout" or sLocalOutputCommandsFileName == "/dev/stderr") + if (sOutputCommandFileName == "/dev/stdout" or sOutputCommandFileName == "/dev/stderr") bPrintOutputInConsole = true; #endif } @@ -66,7 +70,7 @@ void CommandFile::AddInputSearchReplaceValues(const ALString& sSearchValue, cons { require(sSearchValue != ""); - require(not IsCommandFileOpened()); + require(not AreCommandFilesOpened()); // Ajout des valeurs svInputCommandSearchValues.Add(sSearchValue); @@ -99,7 +103,7 @@ void CommandFile::DeleteAllInputSearchReplaceValues() void CommandFile::SetInputParameterFileName(const ALString& sFileName) { - require(not IsCommandFileOpened()); + require(not AreCommandFilesOpened()); sInputParameterFileName = sFileName; } @@ -135,18 +139,12 @@ boolean CommandFile::OpenInputCommandFile() require(Check()); require(GetInputCommandFileName() != ""); require(not IsInputCommandFileOpened()); + require(nParserLineIndex == 0); + require(nParserState == TokenOther); // Copie depuis HDFS si necessaire bOk = PLRemoteFileService::BuildInputWorkingFile(sInputCommandFileName, sLocalInputCommandFileName); - // Fermeture du fichier si celui-ci est deja ouvert - // Ce cas peut arriver si on appelle plusieurs fois ParseParameters (notamment via MODL_dll) - if (fInputCommands != NULL) - { - fclose(fInputCommands); - fInputCommands = NULL; - } - // Ouverture du fichier en lecture if (bOk) fInputCommands = p_fopen(sLocalInputCommandFileName, "r"); @@ -162,7 +160,8 @@ boolean CommandFile::OpenInputCommandFile() bOk = LoadJsonParameters(); // Message d'erreur synthetique - AddInputParameterFileError("Unable to exploit json parameters"); + if (not bOk) + AddInputParameterFileError("Unable to exploit json parameters"); } return bOk; } @@ -211,13 +210,41 @@ boolean CommandFile::IsOutputCommandFileOpened() const return fOutputCommands != NULL; } -boolean CommandFile::IsCommandFileOpened() const +boolean CommandFile::AreCommandFilesOpened() const { return IsInputCommandFileOpened() or IsOutputCommandFileOpened(); } -void CommandFile::CloseCommandFiles() +void CommandFile::CloseInputCommandFile() { + static boolean bPendingFatalError = false; + boolean bOk; + boolean bIsParserOkBeforeEnd; + boolean bIsParserOkAfterEnd; + StringVector svIdentifierPath; + ALString sValue; + + // Lecture si necessaire de la fin du fichier pour avoir des diagnostiques d'erreur complets + // - bloc if non termine + // - diagnostique sur les cles du parametrage json non utilisees dans les commandes + bIsParserOkBeforeEnd = bParserOk; + if (GetInputParameterFileName() != "" and IsInputCommandFileOpened() and not IsInputCommandEnd() and + nParserLineIndex > 0 and bParserOk) + { + // Analyse des ligne jusqu'a la fin du fichier ou d'une erreur, qui mettra a jour bParserOk + bOk = true; + while (bOk) + { + bOk = ReadInputCommand(&svIdentifierPath, sValue); + svIdentifierPath.SetSize(0); + } + } + bIsParserOkAfterEnd = bParserOk; + + // Detection des membres non utilises du parametrage json + if (bIsParserOkAfterEnd) + bIsParserOkAfterEnd = DetectedUnusedJsonParameterMembers(); + // Fermeture du fichier d'entree if (fInputCommands != NULL) { @@ -228,6 +255,32 @@ void CommandFile::CloseCommandFiles() PLRemoteFileService::CleanInputWorkingFile(sInputCommandFileName, sLocalInputCommandFileName); } + // Nettoyage du parser et des parametre json (et donc de bParserOk) + ResetParser(); + jsonParameters.DeleteAll(); + + // Erreur fatal standard si erreur avant la fin de fichier + // On empeche une boucle infinie en cas d'erreur fatale, car la destruction des objets statiques + // et la fermeture des fichiers est appellee apres la fin du main en cas d'erreur fatale + if (not bPendingFatalError) + { + if (not bIsParserOkBeforeEnd) + { + AddInputCommandFileError("Analysis of input commands interrupted because of errors"); + bPendingFatalError = true, Global::AddFatalError("Command file", "", "Batch mode failure"); + } + // Erreur fatale en cas d'erreurs detectees apres la fermeture du fichier + else if (not bIsParserOkAfterEnd) + { + bPendingFatalError = true, + Global::AddFatalError("Command file", "", + "Batch mode failure detected when closing input command file"); + } + } +} + +void CommandFile::CloseOutputCommandFile() +{ // Fermeture du fichier de sortie, uniquement en mode batch (sinon, on enregistre toujours le scenario) if (fOutputCommands != NULL) { @@ -239,93 +292,192 @@ void CommandFile::CloseCommandFiles() } } +void CommandFile::CloseCommandFiles() +{ + // On ferme d'abord le fichier en sortie + CloseOutputCommandFile(); + + // Puis le fichier en entree, ce qui peut declencher potentiellement une erreur fatale + CloseInputCommandFile(); +} + boolean CommandFile::ReadInputCommand(StringVector* svIdentifierPath, ALString& sValue) { const char cDEL = (char)127; - int i; + boolean bOk; char sCharBuffer[1 + BUFFER_LENGTH]; - ALString sStringBuffer; + ALString sInputLine; + boolean bContinueParsing; + IntVector ivTokenTypes; + StringVector svTokenValues; + ALString sMessage; int nLength; - ALString sIdentifierPath; int nPosition; + int nToken; + ALString sEndLine; ALString sToken; + ALString sInterToken; + ALString sIdentifierPath; ALString sIdentifier; + int i; + ALString sTmp; require(svIdentifierPath != NULL); require(svIdentifierPath->GetSize() == 0); + require(GetInputSearchReplaceValueNumber() == 0 or GetInputParameterFileName() == ""); + require(nParserState == TokenOther or nParserState == TokenIf or nParserState == TokenLoop); // On arrete si pas de fichier ou si fin de fichier if (fInputCommands == NULL or feof(fInputCommands)) return false; - // Boucle de lecture pour ignorer les lignes vides - // ou ne comportant que des commentaires - while (sStringBuffer == "" and not feof(fInputCommands)) - { - // Lecture - StandardGetInputString(sCharBuffer, fInputCommands); + //////////////////////////////////////////////////////////////////////////////////////////// + // Recherche d'une ligne non vide a traiter - // Si erreur ou caractere fin de fichier, on ne renvoie rien - if (ferror(fInputCommands) or sCharBuffer[0] == '\0') + // On effectue le parsing uniquement si necessaire + // Dans le cas d'un bloc de type loop en cours de traitement, ce n'est pas necessaire + bOk = true; + if (oaParserLoopLinesTokenTypes.GetSize() == 0) + { + // Boucle de lecture pour ignorer les lignes vides ou ne comportant que des commentaires + while (sInputLine == "" and not feof(fInputCommands)) { - // Nettoyage - fclose(fInputCommands); - fInputCommands = NULL; - return false; - } + nParserLineIndex++; - // Suppression du premier '\n' - for (i = 0; i < BUFFER_LENGTH; i++) - { - if (sCharBuffer[i] == '\n') - sCharBuffer[i] = '\0'; - } + // Lecture + StandardGetInputString(sCharBuffer, fInputCommands); + sInputLine = sCharBuffer; - // Recherche /remplacement dans la partie valeur de la commande - sStringBuffer = ProcessSearchReplaceCommand(sCharBuffer); + // Si erreur ou caractere fin de fichier, on arrete sans message d'erreur + // Si on est pas en fin de fichier, on a forcement un caractere '\n' en fin de ligne + if (ferror(fInputCommands) or sInputLine.GetLength() == 0) + { + bOk = false; + break; + } - // Suppression des commentaires en partant de la fin - // Ainsi on ne supprime que le dernier commentaire, ce qui permet d'avoir - // des paires (IdentifierPath, valeur) avec valeur contenant des " //" - // si on a un commentaire est en fin de ligne - nLength = sStringBuffer.GetLength(); - for (i = nLength - 1; i >= 2; i--) - { - if (sStringBuffer.GetAt(i) == '/' and sStringBuffer.GetAt(i - 1) == '/' and - iswspace(sStringBuffer.GetAt(i - 2))) + // Erreur si ligne trop longue + if (bOk and sInputLine.GetLength() > nMaxLineLength) { - sStringBuffer = sStringBuffer.Left(i - 1); + bOk = false; + AddInputCommandFileError(sTmp + "line too long, with length " + + IntToString(sInputLine.GetLength()) + " > " + + IntToString(nMaxLineLength)); break; } + + // Suppression des blancs au debut et a la fin, donc du dernier caractere fin de ligne + sInputLine.TrimRight(); + sInputLine.TrimLeft(); + + // Supression de la ligne entiere si elle est entierement commentee + if (sInputLine.Find(sCommentPrefix) == 0) + sInputLine = ""; + // Sinon, suppression du commentaire de fin de ligne, ce qui permet d'avoir + // des paires (IdentifierPath, valeur) avec valeur contenant des " //" + else + { + nLength = sInputLine.GetLength(); + for (i = nLength - sCommentPrefix.GetLength(); i >= 0; i--) + { + if (sInputLine.GetAt(i) == sCommentPrefix.GetAt(0) and + sInputLine.Right(nLength - i).Find(sCommentPrefix) == 0) + { + sInputLine.GetBufferSetLength(i); + sInputLine.TrimRight(); + break; + } + } + } + + // Suppression si necessaire du dernier caractere s'il s'agit de cDEL + // (methode FromJstring qui utilise ce cartactere special en fin de chaine pour + // les conversions entre Java et C++ et l'ajoute dans les fichiers de commande en sortie) + if (sInputLine.GetLength() > 0 and sInputLine.GetAt(sInputLine.GetLength() - 1) == cDEL) + sInputLine.GetBufferSetLength(sInputLine.GetLength() - 1); + + // Tokenisation de la ligne en cours dans le cas d'un fichier de parametrage + // On continue tant que necessaire, c'est a dire jsuqu'a une commande parsee soit + // directement disponible, ou que l'on est prepare toutes les commandes d'un bloc loop + assert(bOk); + if (sInputLine != "" and GetInputParameterFileName() != "") + { + // Tokenisation de la ligne + bOk = ParseInputCommand(sInputLine, bContinueParsing); + + // Traitement pour ignorer des lignes en fonction l'etat du parser + // On force la poursuite duparsing si necessaire + if (bContinueParsing) + sInputLine = ""; + } + } + assert(sInputLine != "" or not bOk or feof(fInputCommands)); + + // Cas particulier de fin de fichier sans fin de bloc + // C'est le seul cas qui ne pas etre traite par le parsing en flux des lignes du fichier de commande + if (bOk and feof(fInputCommands) and GetInputParameterFileName() != "" and nParserState != TokenOther) + { + bOk = false; + AddInputCommandFileError("No " + sTokenEnd + " " + GetBlockType(nParserState) + + " found until the end of the file in current " + + GetBlockType(nParserState) + " " + sParserBlockKey + " block"); } - // Suppression des blancs au debut et a la fin - sStringBuffer.TrimRight(); - sStringBuffer.TrimLeft(); + // Arret si necessaire + if (not bOk or feof(fInputCommands)) + return false; + assert(sInputLine != ""); + } + + //////////////////////////////////////////////////////////////////////////////////////////// + // Pretraitement de la ligne - // Suppression si necessaire du dernier caractere s'il s'agit de cDEL - // (methode FromJstring qui utilise ce cartactere special en fin de chaine pour - // les conversions entre Java et C++) - if (sStringBuffer.GetLength() > 0 and sStringBuffer.GetAt(sStringBuffer.GetLength() - 1) == cDEL) - sStringBuffer.GetBufferSetLength(sStringBuffer.GetLength() - 1); + // Cas sans fichier de parametrage json: on verifie que l'on n'utilise pas + // des balises du langage de pilotage par fichier de parametre + if (bOk and GetInputParameterFileName() == "") + { + // Detection du premier token de la ligne + nToken = GetFirstInputToken(sInputLine, sToken, sInterToken, sEndLine); - // On supprime la ligne si elle commence par un commentaire - // Cela permet de commenter y compris les lignes ayant une valeur contenant des "//" - if (sStringBuffer.GetLength() >= 2 and sStringBuffer.GetAt(0) == '/' and sStringBuffer.GetAt(1) == '/') - sStringBuffer = ""; + // Arret si detection de token reserves au cas des fichiers de parametres json + // On ne peut pas aller plus loin dans l'analyse des tokens suivants, qui pourait + // correspondre a des valeurs dans les donnees (par exemple, un nom de variable "__improbable__") + if (nToken == TokenLoop or nToken == TokenIf or nToken == TokenEnd or nToken == TokenKey) + { + bOk = false; + AddInputCommandFileError(sTmp + "use of the \"" + GetPrintableValue(sToken) + + "\" value allowed only with a json parameter file"); + } } + // Cas avec fichier de parametrage json: analyse des lignes + if (bOk and GetInputParameterFileName() != "") + sInputLine = RecodeCurrentLineUsingJsonParameters(bOk); + // Cas du mode recherche/remplacement dans la partie valeur de la commande + else if (bOk and GetInputSearchReplaceValueNumber() > 0) + sInputLine = ProcessSearchReplaceCommand(sInputLine); + + // Arret si necessaire + if (not bOk) + return false; + + //////////////////////////////////////////////////////////////////////////////////////////// + // Analyse de la ligne a traiter + + // A ce stade, soit on a quite la methode, soit on a une ligne non vide a analyser + assert(sInputLine.GetLength() > 0); + // Recherche de l'IdentifierPath et de la valeur - nPosition = sStringBuffer.Find(' '); + nPosition = sInputLine.Find(' '); if (nPosition == -1) { - sIdentifierPath = sStringBuffer; + sIdentifierPath = sInputLine; sValue = ""; } else { - sIdentifierPath = sStringBuffer.Left(nPosition); - sValue = sStringBuffer.Right(sStringBuffer.GetLength() - nPosition - 1); + sIdentifierPath = sInputLine.Left(nPosition); + sValue = sInputLine.Right(sInputLine.GetLength() - nPosition - 1); sValue.TrimRight(); sValue.TrimLeft(); } @@ -371,6 +523,15 @@ boolean CommandFile::ReadInputCommand(StringVector* svIdentifierPath, ALString& return false; } +boolean CommandFile::IsInputCommandEnd() const +{ + require(GetInputCommandFileName() != ""); + require(IsInputCommandFileOpened()); + + // On teste s'il reste des ligne a lire dans le fichier en entree et des commande de bloc loop en cours + return feof(fInputCommands) and oaParserLoopLinesTokenTypes.GetSize() == 0; +} + void CommandFile::WriteOutputCommand(const ALString& sIdentifierPath, const ALString& sValue, const ALString& sLabel) { if (fOutputCommands != NULL) @@ -388,14 +549,15 @@ void CommandFile::WriteOutputCommand(const ALString& sIdentifierPath, const ALSt if (nCommandLength < 30) nCommandLength = 30; - // Impression + // Si redirection vers la console, on ajoute un prefixe if (bPrintOutputInConsole) - // Si redirection vers la console, on ajoute un prefixe fprintf(fOutputCommands, "Khiops.command\t"); + + // Impression if (sCommand == "" and sLabel == "") fprintf(fOutputCommands, "\n"); else if (sCommand == "") - fprintf(fOutputCommands, "// %s\n", (const char*)sLabel); + fprintf(fOutputCommands, sCommentPrefix + " %s\n", (const char*)sLabel); else fprintf(fOutputCommands, "%-*.*s // %s\n", nCommandLength, nCommandLength, (const char*)sCommand, (const char*)sLabel); @@ -403,6 +565,98 @@ void CommandFile::WriteOutputCommand(const ALString& sIdentifierPath, const ALSt } } +void CommandFile::WriteOutputCommandHeader() +{ + WriteOutputCommand("", "", "Output command file"); + WriteOutputCommand("", "", ""); + WriteOutputCommand("", "", "This file contains recorded commands, that can be replayed."); + WriteOutputCommand("", "", "Commands are based on user interactions:"); + WriteOutputCommand("", "", "\tfield update"); + WriteOutputCommand("", "", "\tlist item selection"); + WriteOutputCommand("", "", "\tmenu action"); + WriteOutputCommand("", "", "Every command can be commented, using ."); + WriteOutputCommand("", "", "For example, commenting the last Exit command will allow other"); + WriteOutputCommand("", "", "user interactions, after the commands have been replayed."); + WriteOutputCommand("", "", ""); +} + +boolean CommandFile::ReadWriteCommandFiles() +{ + boolean bOk = true; + StringVector svIdentifierPath; + ALString sValue; + ALString sIdentifierPath; + int i; + + require(Check()); + require(not AreCommandFilesOpened()); + require(GetInputCommandFileName() != ""); + require(GetOutputCommandFileName() != ""); + + // Ouverture des fichier de commandes + if (bOk) + bOk = OpenInputCommandFile(); + if (bOk) + bOk = OpenOutputCommandFile(); + + // Lecture/ecriture des lignes de commandes + while (bOk and not IsInputCommandEnd()) + { + bOk = ReadInputCommand(&svIdentifierPath, sValue); + + // Ecriture de la commande si ok + if (bOk) + { + // Construction de l'identifier de commande a partir des ses composant + sIdentifierPath = ""; + for (i = 0; i < svIdentifierPath.GetSize(); i++) + { + if (i > 0) + sIdentifierPath += '.'; + sIdentifierPath += svIdentifierPath.GetAt(i); + } + svIdentifierPath.SetSize(0); + + // Ecriture + WriteOutputCommand(sIdentifierPath, sValue, ""); + } + } + + // Fermeture des fichiers de commandes, declenchant potentiellement des erreur + CloseCommandFiles(); + return bOk; +} + +void CommandFile::AddInputCommandFileError(const ALString& sMessage) const +{ + ALString sLineLocalisation; + ALString sTmp; + + // On precise le numero de ligne si disponible + if (nParserLineIndex > 0) + sLineLocalisation = sTmp + ", line " + IntToString(nParserLineIndex); + Global::AddError("Input command file", sInputCommandFileName + sLineLocalisation, sMessage); + bParserOk = false; +} + +void CommandFile::AddInputParameterFileError(const ALString& sMessage) const +{ + Global::AddError("Input json parameter file", sInputParameterFileName, sMessage); +} + +void CommandFile::AddOutputCommandFileError(const ALString& sMessage) const +{ + Global::AddError("Input command file", sOutputCommandFileName, sMessage); +} + +const ALString CommandFile::GetPrintableValue(const ALString& sValue) const +{ + if (sValue.GetLength() <= nMaxPrintableLength) + return sValue; + else + return sValue.Left(nMaxPrintableLength) + "..."; +} + boolean CommandFile::LoadJsonParameters() { boolean bOk = true; @@ -452,17 +706,17 @@ boolean CommandFile::LoadJsonParameters() jsonMember = jsonParameters.GetMemberAt(i); // Verification de la cle - if (not CheckVariableName(jsonMember->GetKey(), sMessage)) + if (not CheckJsonKey(jsonMember->GetKey(), sMessage)) { AddInputParameterFileError("in main json object, " + sMessage); bOk = false; } // Verification une seule fois de la non-collision de la cle avec sa variante de type byte - else if (IsByteVariableName(jsonMember->GetKey()) and - jsonParameters.LookupMember(ToStandardVariableName(jsonMember->GetKey())) != NULL) + else if (IsByteJsonKey(jsonMember->GetKey()) and + jsonParameters.LookupMember(ToStandardJsonKey(jsonMember->GetKey())) != NULL) { AddInputParameterFileError("in main json object, the \"" + - ToStandardVariableName(jsonMember->GetKey()) + + ToStandardJsonKey(jsonMember->GetKey()) + "\" key is used twice, along with its \"" + jsonMember->GetKey() + "\" byte variant"); bOk = false; @@ -480,19 +734,19 @@ boolean CommandFile::LoadJsonParameters() bOk = false; } // Verification du type string dans le cas d'une cle avec sa variante de type byte - else if (IsByteVariableName(jsonMember->GetKey()) and + else if (IsByteJsonKey(jsonMember->GetKey()) and jsonMember->GetValueType() != JsonObject::StringValue) { AddInputParameterFileError( "in main json object, the " + jsonMember->GetValue()->TypeToString() + " type of value at " + BuildJsonPath(jsonMember, -1, NULL) + - " should be string, as the key prefix is \"" + sByteVariablePrefix + "\""); + " should be string, as the key prefix is \"" + sByteJsonKeyPrefix + "\""); bOk = false; } // Verification de de la longueur et de l'encodage base64 de la valeur string else if (jsonMember->GetValueType() == JsonObject::StringValue and not CheckStringValue(jsonMember->GetStringValue()->GetString(), - IsByteVariableName(jsonMember->GetKey()), sMessage)) + IsByteJsonKey(jsonMember->GetKey()), sMessage)) { AddInputParameterFileError("in main json object, at " + BuildJsonPath(jsonMember, -1, NULL) + ", " + sMessage); @@ -532,8 +786,7 @@ boolean CommandFile::LoadJsonParameters() jsonSubObjectMember = jsonSubObject->GetMemberAt(k); // Verification de la cle - if (not CheckVariableName(jsonSubObjectMember->GetKey(), - sMessage)) + if (not CheckJsonKey(jsonSubObjectMember->GetKey(), sMessage)) { AddInputParameterFileError( "in object value of array at " + @@ -543,15 +796,14 @@ boolean CommandFile::LoadJsonParameters() bIsArrayObjectValid = false; } // Verification de la non-collision de la cle avec sa variante de type byte - else if (IsByteVariableName(jsonSubObjectMember->GetKey()) and - jsonSubObject->LookupMember(ToStandardVariableName( + else if (IsByteJsonKey(jsonSubObjectMember->GetKey()) and + jsonSubObject->LookupMember(ToStandardJsonKey( jsonSubObjectMember->GetKey())) != NULL) { AddInputParameterFileError( "in object value of array at " + BuildJsonPath(jsonMember, j, NULL) + ", the \"" + - ToStandardVariableName( - jsonSubObjectMember->GetKey()) + + ToStandardJsonKey(jsonSubObjectMember->GetKey()) + "\" key is used twice, along with its \"" + jsonSubObjectMember->GetKey() + "\" byte variant"); bOk = false; @@ -559,15 +811,15 @@ boolean CommandFile::LoadJsonParameters() // Verification que la cle du sous-objet n'entre pas en collision avec une cle de l'objet principal else if (jsonParameters.LookupMember( jsonSubObjectMember->GetKey()) != NULL or - jsonParameters.LookupMember(ToVariantVariableName( + jsonParameters.LookupMember(ToVariantJsonKey( jsonSubObjectMember->GetKey())) != NULL) { // Message d'erreur avec precision dans le cas d'utilisation de variantes standard ou byte differents sLabelSuffix = ""; - if (jsonParameters.LookupMember(ToVariantVariableName( + if (jsonParameters.LookupMember(ToVariantJsonKey( jsonSubObjectMember->GetKey())) != NULL) - sLabelSuffix = " (in any standard or " - "byte variants)"; + sLabelSuffix = + " (in the standard or byte variants)"; AddInputParameterFileError( "in object value of array at " + BuildJsonPath(jsonMember, j, NULL) + ", the \"" + @@ -593,7 +845,7 @@ boolean CommandFile::LoadJsonParameters() bIsArrayObjectValid = false; } // Verification du type string dans le cas d'une cle avec sa variante de type byte - else if (IsByteVariableName(jsonSubObjectMember->GetKey()) and + else if (IsByteJsonKey(jsonSubObjectMember->GetKey()) and jsonSubObjectMember->GetValueType() != JsonObject::StringValue) { @@ -603,7 +855,7 @@ boolean CommandFile::LoadJsonParameters() " type of value at " + BuildJsonPath(jsonMember, j, jsonSubObjectMember) + " should be string, as the key prefix is \"" + - sByteVariablePrefix + "\""); + sByteJsonKeyPrefix + "\""); bOk = false; } // Verification de de la longueur et de l'encodage base64 de la valeur string @@ -611,7 +863,7 @@ boolean CommandFile::LoadJsonParameters() JsonObject::StringValue and not CheckStringValue( jsonSubObjectMember->GetStringValue()->GetString(), - IsByteVariableName(jsonSubObjectMember->GetKey()), + IsByteJsonKey(jsonSubObjectMember->GetKey()), sMessage)) { AddInputParameterFileError( @@ -659,7 +911,7 @@ boolean CommandFile::LoadJsonParameters() if (jsonSubObjectMember->GetKey() != firstJsonSubObjectMember->GetKey() and jsonSubObjectMember->GetKey() != - ToVariantVariableName( + ToVariantJsonKey( firstJsonSubObjectMember->GetKey())) { AddInputParameterFileError( @@ -706,7 +958,7 @@ boolean CommandFile::LoadJsonParameters() return bOk; } -boolean CommandFile::CheckVariableName(const ALString& sValue, ALString& sMessage) const +boolean CommandFile::CheckJsonKey(const ALString& sValue, ALString& sMessage) const { boolean bOk = true; ALString sTmp; @@ -723,25 +975,25 @@ boolean CommandFile::CheckVariableName(const ALString& sValue, ALString& sMessag // Test de la longueur max if (bOk) { - bOk = sValue.GetLength() <= nMaxVariableNameLength; + bOk = sValue.GetLength() <= nMaxJsonKeyLength; if (not bOk) - sMessage = sTmp + "overlengthy key \"" + GetPrintableValue(sValue) + "\", with length " + - IntToString(sValue.GetLength()) + " > " + IntToString(nMaxVariableNameLength); + sMessage = sTmp + "key \"" + GetPrintableValue(sValue) + "\" too long, with length " + + IntToString(sValue.GetLength()) + " > " + IntToString(nMaxJsonKeyLength); } // Test du format camelCase if (bOk) { - bOk = IsCamelCaseVariableName(sValue); + bOk = IsCamelCaseJsonKey(sValue); if (not bOk) - sMessage = "incorrect key \"" + GetPrintableValue(sValue) + "\", which should be camelCase"; + sMessage = "key \"" + GetPrintableValue(sValue) + "\" should be camelCase"; } ensure(not bOk or sMessage == ""); return bOk; } -boolean CommandFile::IsCamelCaseVariableName(const ALString& sValue) const +boolean CommandFile::IsCamelCaseJsonKey(const ALString& sValue) const { boolean bOk; char c; @@ -775,73 +1027,74 @@ boolean CommandFile::IsCamelCaseVariableName(const ALString& sValue) const return bOk; } -boolean CommandFile::IsByteVariableName(const ALString& sValue) const +boolean CommandFile::IsByteJsonKey(const ALString& sValue) const { boolean bOk; char c; - require(IsCamelCaseVariableName(sValue)); + require(IsCamelCaseJsonKey(sValue)); // Test si on est prefixe correctement, suivi d'un caractere alphabetique en majuscule - bOk = sValue.GetLength() > sByteVariablePrefix.GetLength(); + bOk = sValue.GetLength() > sByteJsonKeyPrefix.GetLength(); if (bOk) - bOk = sValue.Left(sByteVariablePrefix.GetLength()) == sByteVariablePrefix; + bOk = sValue.Left(sByteJsonKeyPrefix.GetLength()) == sByteJsonKeyPrefix; if (bOk) { - c = sValue.GetAt(sByteVariablePrefix.GetLength()); + c = sValue.GetAt(sByteJsonKeyPrefix.GetLength()); bOk = isalpha(c) and isupper(c); } return bOk; } -const ALString CommandFile::ToByteVariableName(const ALString& sValue) const +const ALString CommandFile::ToByteJsonKey(const ALString& sValue) const { - ALString sByteVariableName; + ALString sByteJsonKey; - require(IsCamelCaseVariableName(sValue)); + require(IsCamelCaseJsonKey(sValue)); - if (IsByteVariableName(sValue)) + if (IsByteJsonKey(sValue)) return sValue; else { - sByteVariableName = sValue; - sByteVariableName.SetAt(0, char(toupper(sByteVariableName.GetAt(0)))); - sByteVariableName = sByteVariablePrefix + sByteVariableName; - return sByteVariableName; + sByteJsonKey = sValue; + sByteJsonKey.SetAt(0, char(toupper(sByteJsonKey.GetAt(0)))); + sByteJsonKey = sByteJsonKeyPrefix + sByteJsonKey; + return sByteJsonKey; } } -const ALString CommandFile::ToStandardVariableName(const ALString& sValue) const +const ALString CommandFile::ToStandardJsonKey(const ALString& sValue) const { - ALString sStandardVariableName; + ALString sStandardJsonKey; - require(IsCamelCaseVariableName(sValue)); + require(IsCamelCaseJsonKey(sValue)); - if (IsByteVariableName(sValue)) + if (IsByteJsonKey(sValue)) { - sStandardVariableName = sValue.Right(sValue.GetLength() - sByteVariablePrefix.GetLength()); - sStandardVariableName.SetAt(0, char(tolower(sStandardVariableName.GetAt(0)))); - return sStandardVariableName; + sStandardJsonKey = sValue.Right(sValue.GetLength() - sByteJsonKeyPrefix.GetLength()); + sStandardJsonKey.SetAt(0, char(tolower(sStandardJsonKey.GetAt(0)))); + return sStandardJsonKey; } else return sValue; } -const ALString CommandFile::ToVariantVariableName(const ALString& sValue) const +const ALString CommandFile::ToVariantJsonKey(const ALString& sValue) const { - require(IsCamelCaseVariableName(sValue)); + require(IsCamelCaseJsonKey(sValue)); - if (IsByteVariableName(sValue)) - return ToStandardVariableName(sValue); + if (IsByteJsonKey(sValue)) + return ToStandardJsonKey(sValue); else - return ToByteVariableName(sValue); + return ToByteJsonKey(sValue); } boolean CommandFile::CheckStringValue(const ALString& sValue, boolean bCheckBase64Encoding, ALString& sMessage) const { boolean bOk = true; char* sBytes; + int nPos; ALString sTmp; // Test de la longueur max @@ -850,9 +1103,18 @@ boolean CommandFile::CheckStringValue(const ALString& sValue, boolean bCheckBase { bOk = sValue.GetLength() <= nMaxStringValueLength; if (not bOk) - sMessage = sTmp + "overlengthy string value \"" + GetPrintableValue(sValue) + - "\", with length " + IntToString(sValue.GetLength()) + " > " + - IntToString(nMaxStringValueLength); + sMessage = sTmp + "string value \"" + GetPrintableValue(sValue) + "\" too long, with length " + + IntToString(sValue.GetLength()) + " > " + IntToString(nMaxStringValueLength); + } + + // Test si champ multi-ligne + if (bOk) + { + nPos = sValue.Find('\n'); + bOk = nPos == -1; + if (not bOk) + sMessage = sTmp + "forbidden multi-line string value \"" + + GetPrintableValue(sValue.Left(nPos) + "...") + "\" "; } // Test de l'encodage base64 @@ -863,7 +1125,7 @@ boolean CommandFile::CheckStringValue(const ALString& sValue, boolean bCheckBase assert(nMaxStringValueLength <= BUFFER_LENGTH); // Test de l'encodage base64 - bOk = TextService::Base64StringToBytes(sValue, sBytes) != -1; + bOk = TextService::Base64StringToBytes(sValue, sBytes); if (not bOk) sMessage = "incorrect string value \"" + GetPrintableValue(sValue) + "\", which should be encoding using base64"; @@ -873,27 +1135,29 @@ boolean CommandFile::CheckStringValue(const ALString& sValue, boolean bCheckBase return bOk; } -const ALString CommandFile::GetPrintableValue(const ALString& sValue) const -{ - if (sValue.GetLength() <= nMaxPrintableLength) - return sValue; - else - return sValue.Left(nMaxPrintableLength) + "..."; -} - -void CommandFile::AddInputCommandFileError(const ALString& sMessage) const +ALString CommandFile::BuildJsonPath(JsonMember* member, int nArrayRank, JsonMember* arrayObjectmember) const { - Global::AddError("Input command file", sInputCommandFileName, sMessage); -} + ALString sJsonPath; -void CommandFile::AddInputParameterFileError(const ALString& sMessage) const -{ - Global::AddError("Input parameter file", sInputParameterFileName, sMessage); -} + require(member != NULL); + require(arrayObjectmember == NULL or nArrayRank >= 0); -void CommandFile::AddOutputCommandFileError(const ALString& sMessage) const -{ - Global::AddError("Input command file", sOutputCommandFileName, sMessage); + // Construction du chemin + sJsonPath = '"'; + sJsonPath += '/'; + sJsonPath += member->GetKey(); + if (nArrayRank >= 0) + { + sJsonPath += '/'; + sJsonPath += IntToString(nArrayRank); + } + if (arrayObjectmember != NULL) + { + sJsonPath += '/'; + sJsonPath += arrayObjectmember->GetKey(); + } + sJsonPath += '"'; + return sJsonPath; } const ALString CommandFile::ProcessSearchReplaceCommand(const ALString& sInputCommand) const @@ -907,6 +1171,8 @@ const ALString CommandFile::ProcessSearchReplaceCommand(const ALString& sInputCo int nSearchPosition; int i; + require(GetInputSearchReplaceValueNumber() > 0); + // Parcours de toutes les paires search/replace a appliquer sOutputCommand = sInputCommand; for (i = 0; i < GetInputSearchReplaceValueNumber(); i++) @@ -942,29 +1208,896 @@ const ALString CommandFile::ProcessSearchReplaceCommand(const ALString& sInputCo return sOutputCommand; } -ALString CommandFile::BuildJsonPath(JsonMember* member, int nArrayRank, JsonMember* arrayObjectmember) +const ALString& CommandFile::GetBlockType(int nToken) const { - ALString sJsonPath; + require(nToken == TokenIf or nToken == TokenLoop); + if (nToken == TokenIf) + return sTokenIf; + else + return sTokenLoop; +} - require(member != NULL); - require(arrayObjectmember == NULL or nArrayRank >= 0); +void CommandFile::ResetParser() +{ + nParserLineIndex = 0; + nParserState = TokenOther; + nParserCurrentLineState = None; + sParserBlockKey = ""; + ivParserTokenTypes.SetSize(0); + svParserTokenValues.SetSize(0); + bParserIfState = false; + parserLoopJsonArray = NULL; + oaParserLoopLinesTokenTypes.DeleteAll(); + oaParserLoopLinesTokenValues.DeleteAll(); + nParserLoopLineIndex = -1; + nParserLoopObjectIndex = -1; + bParserOk = true; + nkdParserUsedJsonParameterMembers.RemoveAll(); +} - // Construction du chemin - sJsonPath = '"'; - sJsonPath += '/'; - sJsonPath += member->GetKey(); - if (nArrayRank >= 0) +const ALString CommandFile::RecodeCurrentLineUsingJsonParameters(boolean& bOk) +{ + boolean bCleanLoopWorkingData; + ALString sRecodedLine; + JsonValue* jsonValue; + ALString sJsonKey; + boolean bIsByteJsonKey; + IntVector* ivCurrentTokenTypes; + StringVector* svCurrentTokenValues; + JsonObject* currentJsonObject; + char sCharBuffer[1 + BUFFER_LENGTH]; + int i; + int nToken; + + require(GetInputParameterFileName() != ""); + require(IsInputCommandFileOpened()); + require(nParserState == TokenOther or nParserState == TokenIf); + + // Initialisation + bOk = true; + + // Recherche du vecteur de token courant dans la boucle si elle est en cours de traitement + bCleanLoopWorkingData = false; + if (oaParserLoopLinesTokenTypes.GetSize() > 0) { - sJsonPath += '/'; - sJsonPath += IntToString(nArrayRank); + // Dans un bloc loop, on est revenu en etat standard une fois la fin de la boucle detectee + // et on doit alors traiter toutes les instruction memorisee autant de fois qu'il y d'objets json + assert(nParserState == TokenOther); + assert(parserLoopJsonArray != NULL); + assert(nParserLoopLineIndex >= 0); + assert(nParserLoopObjectIndex >= 0); + assert(nParserLoopLineIndex < oaParserLoopLinesTokenTypes.GetSize() or + nParserLoopObjectIndex < parserLoopJsonArray->GetValueNumber()); + + // Analyse de la ligne courante avec l'objet courant + ivCurrentTokenTypes = cast(IntVector*, oaParserLoopLinesTokenTypes.GetAt(nParserLoopLineIndex)); + svCurrentTokenValues = cast(StringVector*, oaParserLoopLinesTokenValues.GetAt(nParserLoopLineIndex)); + currentJsonObject = parserLoopJsonArray->GetValueAt(nParserLoopObjectIndex)->GetObjectValue(); + + // On passe a la suite + nParserLoopLineIndex++; + if (nParserLoopLineIndex == oaParserLoopLinesTokenTypes.GetSize()) + { + nParserLoopLineIndex = 0; + nParserLoopObjectIndex++; + } + + // On memorise la necessite de nettoyage si necessaire des donnees de travail du bloc loop + // Le nettoyage sera effectue plus tard, car il faut encore traiter la derniere instruction en cours + if (nParserLoopObjectIndex == parserLoopJsonArray->GetValueNumber()) + bCleanLoopWorkingData = true; } - if (arrayObjectmember != NULL) + // Sinon, on est sur une instruction standard a executer + else { - sJsonPath += '/'; - sJsonPath += arrayObjectmember->GetKey(); + assert(nParserState == TokenOther or nParserState == TokenIf); + assert(nParserState != TokenIf or bParserIfState); + ivCurrentTokenTypes = &ivParserTokenTypes; + svCurrentTokenValues = &svParserTokenValues; + currentJsonObject = NULL; } - sJsonPath += '"'; - return sJsonPath; + + // Recodage de la ligne en cours + for (i = 0; i < ivCurrentTokenTypes->GetSize(); i++) + { + nToken = ivCurrentTokenTypes->GetAt(i); + assert(nToken == TokenOther or nToken == TokenKey); + assert(nToken == TokenOther or i > 0); + + // On a necessairement un blanc avant le deuxieme token + if (i == 1) + sRecodedLine += ' '; + + // Prise en compte du token tel quel dans le cas standard + if (nToken == TokenOther) + sRecodedLine += svCurrentTokenValues->GetAt(i); + // Remplacement de la valeur dans le cas d'une cle json + else + { + sJsonKey = ExtractJsonKey(svCurrentTokenValues->GetAt(i)); + bIsByteJsonKey = false; + + // Recherche d'un valeur d'abord dans l'objet courant + jsonValue = NULL; + if (currentJsonObject != NULL) + { + // Recherche d'abord en mode standard + jsonValue = LookupJsonValue(currentJsonObject, sJsonKey); + + // Recherche sinon en mode byte + if (jsonValue == NULL) + { + jsonValue = LookupJsonValue(currentJsonObject, ToByteJsonKey(sJsonKey)); + if (jsonValue != NULL) + bIsByteJsonKey = true; + } + assert(jsonValue == NULL or + (LookupJsonValue(&jsonParameters, sJsonKey) == NULL and + LookupJsonValue(&jsonParameters, ToByteJsonKey(sJsonKey)) == NULL)); + } + + // Recherche dans l'objet principal si non trouve + if (jsonValue == NULL) + { + // Recherche d'abord en mode standard + jsonValue = LookupJsonValue(&jsonParameters, sJsonKey); + + // Recherche sinon en mode byte + if (jsonValue == NULL) + { + jsonValue = LookupJsonValue(&jsonParameters, ToByteJsonKey(sJsonKey)); + if (jsonValue != NULL) + bIsByteJsonKey = true; + } + } + + // Erreur si non trouve + if (jsonValue == NULL) + { + bOk = false; + AddInputCommandFileError("Value not found for key \"" + sJsonKey + + "\" in json parameters"); + } + // Erreur si type de valeur invalide + else if (jsonValue->GetType() != JsonValue::NumberValue and + jsonValue->GetType() != JsonValue::StringValue) + { + bOk = false; + AddInputCommandFileError( + "Value found for key \"" + sJsonKey + + "\" in json parameters should be of type number or string, instead of " + + jsonValue->TypeToString()); + } + // Sinon, on ajoute la valeur recodee + else + { + assert(jsonValue->GetType() == JsonValue::NumberValue or + jsonValue->GetType() == JsonValue::StringValue); + if (jsonValue->GetType() == JsonValue::NumberValue) + sRecodedLine += jsonValue->GetNumberValue()->WriteString(); + else + { + // Cas avec recodage base64 + if (bIsByteJsonKey) + { + assert(jsonValue->GetStringValue()->GetString().GetLength() <= + nMaxStringValueLength); + assert(nMaxStringValueLength < BUFFER_LENGTH); + TextService::Base64StringToBytes( + jsonValue->GetStringValue()->GetString(), sCharBuffer); + sRecodedLine += sCharBuffer; + } + // Cas standard + else + sRecodedLine += jsonValue->GetStringValue()->GetString(); + } + } + + // Arret si erreur + if (not bOk) + break; + } + } + + // Nettoyage si necessaire des donnees de travail du bloc loop + if (bCleanLoopWorkingData) + { + parserLoopJsonArray = NULL; + oaParserLoopLinesTokenTypes.DeleteAll(); + oaParserLoopLinesTokenValues.DeleteAll(); + nParserLoopLineIndex = 0; + nParserLoopObjectIndex = 0; + } + + // Nettoyage si erreur + if (not bOk) + sRecodedLine = ""; + return sRecodedLine; +} + +boolean CommandFile::ParseInputCommand(const ALString& sInputCommand, boolean& bContinueParsing) +{ + boolean bOk = true; + int nNextToken; + JsonValue* jsonValue; + + require(GetInputParameterFileName() != ""); + require(sInputCommand != ""); + require(IsValueTrimed(sInputCommand)); + require(sInputCommand.Find(sCommentPrefix) != 0); + require(nParserState == TokenOther or nParserState == TokenIf or nParserState == TokenLoop); + require(nParserState == TokenOther or sParserBlockKey != ""); + require(nParserState == TokenLoop or oaParserLoopLinesTokenTypes.GetSize() == 0); + require(oaParserLoopLinesTokenValues.GetSize() == oaParserLoopLinesTokenTypes.GetSize()); + + // Tokenisation de la ligne + bContinueParsing = false; + bOk = TokenizeInputCommand(sInputCommand, &ivParserTokenTypes, &svParserTokenValues); + + // Analyse des tokens + if (bOk) + { + nParserCurrentLineState = ivParserTokenTypes.GetAt(0); + nNextToken = None; + if (ivParserTokenTypes.GetSize() > 1) + nNextToken = ivParserTokenTypes.GetAt(1); + + ///////////////////////////////////////////////////////////////// + // Cas d'un debut de bloc IF ou LOOP + if (nParserCurrentLineState == TokenIf or nParserCurrentLineState == TokenLoop) + { + assert(nNextToken == TokenKey); + + // Cas valide uniquement l'etat du parser est instruction + if (nParserState == TokenOther) + { + nParserState = nParserCurrentLineState; + sParserBlockKey = svParserTokenValues.GetAt(1); + bParserIfState = false; + parserLoopJsonArray = NULL; + + // Recherche de la valeur json associee au bloc + jsonValue = LookupJsonValue(&jsonParameters, ExtractJsonKey(sParserBlockKey)); + + // Erreur si non trouve + if (jsonValue == NULL) + { + bOk = false; + AddInputCommandFileError("For beginning of " + svParserTokenValues.GetAt(0) + + " block, key " + svParserTokenValues.GetAt(1) + + " not found in json parameter file"); + } + // Cas du if + else if (nParserCurrentLineState == TokenIf) + { + // Erreur si type incorrect + if (jsonValue->GetType() != JsonObject::BooleanValue) + { + bOk = false; + AddInputCommandFileError("For beginning of " + + svParserTokenValues.GetAt(0) + " block, key " + + svParserTokenValues.GetAt(1) + + " found in json parameter file should be of " + "type boolean instead of " + + jsonValue->TypeToString()); + } + // Sinon, on memorise la valeur du booleen + else + { + bParserIfState = jsonValue->GetBooleanValue()->GetBoolean(); + + // Comme on est sur une structure de controle, il faut continuer le parsing + bContinueParsing = true; + } + } + // Cas du loop + else + { + assert(nParserCurrentLineState == TokenLoop); + + // Erreur si type incorrect + if (jsonValue->GetType() != JsonObject::ArrayValue) + { + bOk = false; + AddInputCommandFileError("For beginning of " + + svParserTokenValues.GetAt(0) + " block, key " + + svParserTokenValues.GetAt(1) + + " found in json parameter file should be of " + "type array instead of " + + jsonValue->TypeToString()); + } + // Sinon, on memorise la valeur du tableau + else + { + parserLoopJsonArray = jsonValue->GetArrayValue(); + + // Reinitialisation du tableau des vecteurs de tokens du bloc loop + oaParserLoopLinesTokenTypes.DeleteAll(); + oaParserLoopLinesTokenValues.DeleteAll(); + nParserLoopLineIndex = 0; + nParserLoopObjectIndex = 0; + + // Comme on est sur une structure de controle, il faut continuer le parsing + bContinueParsing = true; + } + } + } + // Commande invalide sinon + else + { + bOk = false; + AddInputCommandFileError("Unexpected beginning of a " + svParserTokenValues.GetAt(0) + + " block inside the current " + GetBlockType(nParserState) + + " " + sParserBlockKey + " block"); + } + } + ///////////////////////////////////////////////////////////////// + // Cas d'une fin de bloc END + else if (nParserCurrentLineState == TokenEnd) + { + assert(nNextToken == TokenIf or nNextToken == TokenLoop); + + // Cas valide uniquement si l'etat du parser est le bloc de bon type en cours + if ((nParserState == TokenIf or nParserState == TokenLoop) and nParserState == nNextToken) + { + nParserState = TokenOther; + sParserBlockKey = ""; + + // Il faut continuer le parsing dans le cas d'un bloc if ou d'un bloc loop vide + if (nParserState == TokenIf) + bContinueParsing = true; + else if (oaParserLoopLinesTokenTypes.GetSize() == 0) + bContinueParsing = true; + else if (parserLoopJsonArray->GetValueNumber() == 0) + { + bContinueParsing = true; + oaParserLoopLinesTokenTypes.DeleteAll(); + oaParserLoopLinesTokenValues.DeleteAll(); + } + else + bContinueParsing = false; + } + // Commande invalide sinon + else + { + bOk = false; + if (nParserState == TokenOther) + AddInputCommandFileError("Unexpected " + svParserTokenValues.GetAt(0) + " " + + svParserTokenValues.GetAt(1) + + " without beginning of the corresponding block"); + else + AddInputCommandFileError("Unexpected " + svParserTokenValues.GetAt(0) + " " + + svParserTokenValues.GetAt(1) + " inside the current " + + GetBlockType(nParserState) + " " + sParserBlockKey + + " block"); + } + } + ///////////////////////////////////////////////////////////////// + // Cas standard valide d'une instruction + else + { + // Memorisation dans le cas d'un bloc loop + if (nParserState == TokenLoop) + { + // Erreur si trop de ligne dans la boucle + if (oaParserLoopLinesTokenTypes.GetSize() > nLoopMaxLineNumber) + { + bOk = false; + AddInputCommandFileError("Too many lines in current " + sTokenLoop + " " + + sParserBlockKey + " block, with more than " + + IntToString(nLoopMaxLineNumber) + + " non-empty command lines"); + } + // Ajout des vecteur de tokens senon + else + { + oaParserLoopLinesTokenTypes.Add(ivParserTokenTypes.Clone()); + oaParserLoopLinesTokenValues.Add(svParserTokenValues.Clone()); + } + + // Il faut continuer le parsing tant que le bloc loop n'est pas fini, sans erreur + bContinueParsing = bOk; + } + // On ignore les ligne si necessaire dans le cas d'un bloc if + else if (nParserState == TokenIf) + bContinueParsing = not bParserIfState; + // Sinon, la ligne est prete a l'emploie + else + { + assert(nParserState == TokenOther); + bContinueParsing = false; + } + } + } + ensure(nParserState == TokenOther or nParserState == TokenIf or nParserState == TokenLoop); + ensure(oaParserLoopLinesTokenValues.GetSize() == oaParserLoopLinesTokenTypes.GetSize()); + return bOk; +} + +boolean CommandFile::TokenizeInputCommand(const ALString& sInputCommand, IntVector* ivTokenTypes, + StringVector* svTokenValues) const +{ + boolean bOk = true; + int nToken; + ALString sToken; + ALString sBuffer; + ALString sEndLine; + ALString sInterToken; + + require(sInputCommand != ""); + require(IsValueTrimed(sInputCommand)); + require(sInputCommand.Find(sCommentPrefix) != 0); + require(ivTokenTypes != NULL); + require(svTokenValues != NULL); + + // Initialisations + ivTokenTypes->SetSize(0); + svTokenValues->SetSize(0); + + // Recherche du premier Token + nToken = GetFirstInputToken(sInputCommand, sToken, sInterToken, sEndLine); + ivTokenTypes->Add(nToken); + svTokenValues->Add(sToken); + + // Cas IF ou LOOP + if (nToken == TokenIf or nToken == TokenLoop) + { + // Recherche du token suivant attendu + nToken = None; + if (sEndLine != "") + { + sBuffer = sEndLine; + nToken = GetFirstInputToken(sBuffer, sToken, sInterToken, sEndLine); + ivTokenTypes->Add(nToken); + svTokenValues->Add(sToken); + } + + // Erreur pas de token suivant + if (nToken == None) + { + bOk = false; + AddInputCommandFileError("the " + svTokenValues->GetAt(0) + + " command must be followed by a __key__"); + } + // Le token doit etre de type cle + else if (nToken != TokenKey) + { + bOk = false; + AddInputCommandFileError("the " + svTokenValues->GetAt(0) + + " command must be followed by a __key__ (instead of \"" + + GetPrintableValue(sToken) + "\")"); + } + // La cle doit etre valide + else + { + assert(nToken == TokenKey); + bOk = CheckTokenKey(sToken); + } + + // Il ne doit pas y en avoir d'autres tokens + if (bOk and sEndLine != "") + { + bOk = false; + AddInputCommandFileError("the " + svTokenValues->GetAt(0) + + " __key__ command is terminated (unexpected \"" + + GetPrintableValue(sEndLine) + "\" found afterwards)"); + } + } + // Cas END LOOP ou END IF + else if (nToken == TokenEnd) + { + // Recherche du token suivant attendu + nToken = None; + if (sEndLine != "") + { + sBuffer = sEndLine; + nToken = GetFirstInputToken(sBuffer, sToken, sInterToken, sEndLine); + ivTokenTypes->Add(nToken); + svTokenValues->Add(sToken); + } + + // Erreur pas de token suivant + if (nToken == None) + { + bOk = false; + AddInputCommandFileError("the " + svTokenValues->GetAt(0) + " command must be followed by " + + sTokenIf + " or " + sTokenLoop); + } + // Le token doit LOOP ou IF + else if (nToken != TokenIf and nToken != TokenLoop) + { + bOk = false; + AddInputCommandFileError("the " + sTokenEnd + " command must be followed by " + sTokenIf + + " or " + sTokenLoop + " (instead of \"" + GetPrintableValue(sToken) + + "\")"); + } + + // Il ne doit pas y en avoir d'autres tokens + if (bOk and sEndLine != "") + { + bOk = false; + AddInputCommandFileError("the " + svTokenValues->GetAt(0) + " " + sToken + + " command is terminated (unexpected \"" + GetPrintableValue(sEndLine) + + "\" found afterwards)"); + } + } + // Cas (|)* + else if (nToken == TokenOther) + { + // Analyse de la fin de la ligne pour en extraire les variables + while (sEndLine != "") + { + // Tokenisation de la fin de ligne + sBuffer = sEndLine; + nToken = GetFirstInputToken(sBuffer, sToken, sInterToken, sEndLine); + + // Analyse du token dans le cas d'une cle + if (nToken == TokenKey) + { + // Erreur si token cle en deuxieme position sans separateur + if (ivTokenTypes->GetSize() == 1) + { + if (svTokenValues->GetAt(0).Find(' ') == -1 and + sInputCommand.Find(sToken) == svTokenValues->GetAt(0).GetLength()) + { + bOk = false; + AddInputCommandFileError("incorrect use of key \"" + + GetPrintableValue(sToken) + + "\" after the command \"" + + GetPrintableValue(svTokenValues->GetAt(0)) + + "\" without a preceding space character"); + } + } + + // Verification du token + if (bOk) + bOk = CheckTokenKey(sToken); + + // Prise en compte si valide + if (bOk) + { + ivTokenTypes->Add(nToken); + svTokenValues->Add(sToken); + } + // Arret sinon + else + break; + + // A jout d'un token de type valeur si necessaire + if (sInterToken != "") + { + assert(sEndLine.GetLength() > 0); + ivTokenTypes->Add(TokenOther); + svTokenValues->Add(sInterToken); + } + } + // Sinon, ajout si necessaire du token en tant que TokenOther + else + { + // Ajout de la valeur inter-token + sToken += sInterToken; + + // Ajout du token si seul le token de commande est enregistre + if (ivTokenTypes->GetSize() == 1) + { + ivTokenTypes->Add(TokenOther); + svTokenValues->Add(sToken); + } + // Sinon, ajout du token si le precedent est de type cle + else if (ivTokenTypes->GetAt(ivTokenTypes->GetSize() - 1) == TokenKey) + { + ivTokenTypes->Add(TokenOther); + svTokenValues->Add(sToken); + } + // Completion du token existant sinon + else + { + assert(ivTokenTypes->GetSize() >= 2); + assert(ivTokenTypes->GetAt(ivTokenTypes->GetSize() - 1) == TokenOther); + sToken = svTokenValues->GetAt(ivTokenTypes->GetSize() - 1) + sToken; + svTokenValues->SetAt(ivTokenTypes->GetSize() - 1, sToken); + } + } + } + } + // Cas KEY ... + else + { + assert(nToken == TokenKey); + + // Erreur + bOk = false; + AddInputCommandFileError("incorrect use of key \"" + GetPrintableValue(sToken) + + "\" at the beginning of a command"); + } + + // Nettoyage si erreur + if (not bOk) + { + ivTokenTypes->SetSize(0); + svTokenValues->SetSize(0); + } + ensure(ivTokenTypes->GetSize() == svTokenValues->GetSize()); + ensure(not bOk or ivTokenTypes->GetSize() > 0); + return bOk; +} + +int CommandFile::GetFirstInputToken(const ALString& sInputCommand, ALString& sToken, ALString& sInterToken, + ALString& sEndLine) const +{ + int nOutputToken; + ALString sBuffer; + int nPosDelimiter; + int nPosSpace; + int nPosNextToken; + + require(sInputCommand != ""); + require(IsValueTrimed(sInputCommand)); + + // Cas d'un token de type cle json + nPosDelimiter = sInputCommand.Find(sJsonKeyDelimiter); + if (nPosDelimiter == 0) + { + nOutputToken = TokenKey; + + // Recherche de la fin de la cle json + sBuffer = sInputCommand.Right(sInputCommand.GetLength() - sJsonKeyDelimiter.GetLength()); + nPosDelimiter = sBuffer.Find(sJsonKeyDelimiter); + if (nPosDelimiter == -1) + { + sToken = sInputCommand; + sEndLine = ""; + } + else + { + sToken = sInputCommand.Left(nPosDelimiter + 2 * sJsonKeyDelimiter.GetLength()); + sEndLine = sInputCommand.Right(sInputCommand.GetLength() - sToken.GetLength()); + sEndLine.TrimLeft(); + } + } + // Analyse de la ligne sinon + else + { + assert(sInputCommand.Find(sJsonKeyDelimiter) != 0); + + // Recherche du premier token + nPosSpace = sInputCommand.Find(" "); + if (nPosSpace >= 0 and nPosDelimiter >= 0) + nPosNextToken = min(nPosSpace, nPosDelimiter); + else if (nPosSpace == -1) + nPosNextToken = nPosDelimiter; + else + nPosNextToken = nPosSpace; + + // Extraction du premier token + if (nPosNextToken == -1) + { + sToken = sInputCommand; + sEndLine = ""; + } + else + { + assert(nPosNextToken > 0); + sToken = sInputCommand.Left(nPosNextToken); + sEndLine = sInputCommand.Right(sInputCommand.GetLength() - nPosNextToken); + sEndLine.TrimLeft(); + } + + // Identification du type de token + if (sToken == sTokenLoop) + nOutputToken = TokenLoop; + else if (sToken == sTokenIf) + nOutputToken = TokenIf; + else if (sToken == sTokenEnd) + nOutputToken = TokenEnd; + else + nOutputToken = TokenOther; + } + + // Calcul de la valeur inter-token + // C'est necessaire pour ne pas pas perdre les caractere blancs dans les valeurs + sInterToken = sInputCommand.Mid(sToken.GetLength(), + sInputCommand.GetLength() - sToken.GetLength() - sEndLine.GetLength()); + + ensure(0 <= nOutputToken and nOutputToken < None); + ensure(IsValueTrimed(sToken)); + ensure(IsValueTrimed(sEndLine)); + ensure(sInputCommand.Find(sToken) == 0); + ensure(sToken.GetLength() + sInterToken.GetLength() + sEndLine.GetLength() == sInputCommand.GetLength()); + return nOutputToken; +} + +void CommandFile::WriteInputCommandTokens(ostream& ost, IntVector* ivTokenTypes, StringVector* svTokenValues) const +{ + int i; + int nToken; + + require(ivTokenTypes != NULL); + require(svTokenValues != NULL); + ensure(ivTokenTypes->GetSize() == svTokenValues->GetSize()); + + // Affichage avec mise en forme de la liste des tokens + for (i = 0; i < ivTokenTypes->GetSize(); i++) + { + nToken = ivTokenTypes->GetAt(i); + assert(0 <= nToken and nToken < None); + + // On a necessairement un blanc avant le deuxieme token + if (i == 1) + ost << ' '; + + // Affichage du token + ost << svTokenValues->GetAt(i); + } + ost << '\n'; +} + +boolean CommandFile::CheckTokenKey(const ALString& sToken) const +{ + boolean bOk; + ALString sJsonKey; + int nPos; + ALString sMessage; + + require(sToken != ""); + require(IsValueTrimed(sToken)); + require(sToken.Find(sJsonKeyDelimiter) == 0); + + // Initialisation du resultat + bOk = true; + + // Controle des delimiteurs, et suppression de ceux-ci + sJsonKey = sToken.Right(sToken.GetLength() - sJsonKeyDelimiter.GetLength()); + if (bOk) + { + nPos = sJsonKey.Find(sJsonKeyDelimiter); + if (nPos == -1) + { + bOk = false; + AddInputCommandFileError("missing trailing key delimiter \"" + sJsonKeyDelimiter + + "\" at the end of \"" + GetPrintableValue(sToken) + "\""); + } + + // Le token a tester doit etre extrait en s'arretant si possible au second delimiteur trouve + if (bOk) + { + sJsonKey = sJsonKey.Left(sJsonKey.GetLength() - sJsonKeyDelimiter.GetLength()); + + // On ne doit appeler la methode qu'avec un token candidat, ne au plus deux delimiteurs + assert(sJsonKey.Find(sJsonKeyDelimiter) == -1); + } + assert(not bOk or sJsonKey.GetLength() == sToken.GetLength() - 2 * sJsonKeyDelimiter.GetLength()); + } + + // Verification de syntaxe de la variable + if (bOk) + { + bOk = CheckJsonKey(sJsonKey, sMessage); + if (not bOk) + AddInputCommandFileError("incorrect __key__, " + sMessage); + } + + // Verification que l'on n'utilise par une variante de type byte dans un fichier de commande en entree + if (bOk and sJsonKey.Find(sByteJsonKeyPrefix) == 0) + { + bOk = false; + AddInputCommandFileError("prefix \"" + sByteJsonKeyPrefix + "\" used in \"" + + GetPrintableValue(sToken) + "\" not allowed in input command file"); + } + return bOk; +} + +const ALString CommandFile::ExtractJsonKey(const ALString& sTokenKey) const +{ + require(CheckTokenKey(sTokenKey)); + return sTokenKey.Mid(sJsonKeyDelimiter.GetLength(), sTokenKey.GetLength() - 2 * sJsonKeyDelimiter.GetLength()); +} + +JsonValue* CommandFile::LookupJsonValue(JsonObject* jsonObject, const ALString& sKey) const +{ + JsonValue* jsonValue; + JsonMember* jsonMember; + + debug(ALString sMessage); + + require(jsonObject != NULL); + debug(require(CheckJsonKey(sKey, sMessage))); + + // Recherche du membre + jsonMember = jsonObject->LookupMember(sKey); + + // Mise a jour de l'indicateur d'utilisation du membre + if (jsonMember != NULL) + nkdParserUsedJsonParameterMembers.SetAt(jsonMember, jsonMember); + + // Recherche de la valeur + jsonValue = NULL; + if (jsonMember != NULL) + jsonValue = jsonMember->GetValue(); + return jsonValue; +} + +boolean CommandFile::IsValueTrimed(const ALString& sValue) const +{ + if (sValue.GetLength() == 0) + return true; + else if (iswspace(sValue.GetAt(0))) + return false; + else if (iswspace(sValue.GetAt(sValue.GetLength() - 1))) + return false; + else + return true; +} + +boolean CommandFile::DetectedUnusedJsonParameterMembers() const +{ + boolean bOk = true; + int nMember; + int nObject; + int nSubMember; + JsonMember* jsonMember; + JsonArray* jsonArray; + JsonObject* jsonObject; + JsonMember* jsonSubMember; + JsonMember* usedMember; + + // Parcours des membres de l'objet principal de parametrage json + Global::ActivateErrorFlowControl(); + for (nMember = 0; nMember < jsonParameters.GetMemberNumber(); nMember++) + { + jsonMember = jsonParameters.GetMemberAt(nMember); + + // Recherche si le membre est utilise + usedMember = cast(JsonMember*, nkdParserUsedJsonParameterMembers.Lookup(jsonMember)); + assert(usedMember == NULL or usedMember == jsonMember); + + // Erreur si le membre n'est pas utlise + if (usedMember == NULL) + { + bOk = false; + AddInputParameterFileError("value at " + BuildJsonPath(jsonMember, -1, NULL) + + " not used in input command file"); + } + + // Cas d'un tableau + if (jsonMember->GetValueType() == JsonValue::ArrayValue) + { + jsonArray = jsonMember->GetArrayValue(); + + // Parcours des objets du tableau + for (nObject = 0; nObject < jsonArray->GetValueNumber(); nObject++) + { + jsonObject = jsonArray->GetValueAt(nObject)->GetObjectValue(); + + // Parcours des membres du sous-objets + for (nSubMember = 0; nSubMember < jsonObject->GetMemberNumber(); nSubMember++) + { + jsonSubMember = jsonObject->GetMemberAt(nSubMember); + + // Recherche si le membre est utilise + usedMember = + cast(JsonMember*, nkdParserUsedJsonParameterMembers.Lookup(jsonSubMember)); + assert(usedMember == NULL or usedMember == jsonSubMember); + + // Erreur si le membre n'est pas utlise + if (usedMember == NULL) + { + bOk = false; + AddInputParameterFileError( + "value at " + BuildJsonPath(jsonMember, nObject, jsonSubMember) + + " not used in input command file"); + } + } + } + } + } + Global::DesactivateErrorFlowControl(); + return bOk; } -const ALString CommandFile::sByteVariablePrefix = "byte"; +const ALString CommandFile::sCommentPrefix = "//"; +const ALString CommandFile::sTokenLoop = "LOOP"; +const ALString CommandFile::sTokenIf = "IF"; +const ALString CommandFile::sTokenEnd = "END"; +const ALString CommandFile::sJsonKeyDelimiter = "__"; +const ALString CommandFile::sByteJsonKeyPrefix = "byte"; diff --git a/src/Norm/base/CommandFile.h b/src/Norm/base/CommandFile.h index 2ed8920b..7ac55a6d 100644 --- a/src/Norm/base/CommandFile.h +++ b/src/Norm/base/CommandFile.h @@ -156,70 +156,177 @@ class CommandFile : public Object boolean IsOutputCommandFileOpened() const; // Indique si on est en train de traiter les commande, en entree ou en sortie - boolean IsCommandFileOpened() const; + boolean AreCommandFilesOpened() const; - // Ouverture des fichiers de commandes + // Fermeture des fichiers de commandes + // La fermeture peut provoquer un erreur fatale en cas d'erreur d'analyse des commandes + void CloseInputCommandFile(); + void CloseOutputCommandFile(); void CloseCommandFiles(); // Lecture d'une commande - // Renvoie false si pas de commande, sinon un vecteur de chaines de caracteres representant + // Renvoie false si pas de commande valide disponible, sinon un vecteur de chaines de caracteres representant // le parsing de IdentifierPath et une valeur optionnelle + // Cette methode peut etre appelee meme en l'absence de fichier de de commande en sortie boolean ReadInputCommand(StringVector* svIdentifierPath, ALString& sValue); + // Indique que l'on a fini de lire et traiter les commandes + boolean IsInputCommandEnd() const; + // Ecriture d'une commande + // Cette methode peut etre appelee meme en l'absence de fichier de de commande en sortie void WriteOutputCommand(const ALString& sIdentifierPath, const ALString& sValue, const ALString& sLabel); + // Ecriture d'une header de fichier de commande, consistant en lignes de commentaire en expliquant le fonctionnement + void WriteOutputCommandHeader(); + + // Mode lecture/ecriture d'un fichier de de commande, sans executer les commandes + // Cela permet de tester la validite des fichier de command eet de parametres en entree + // et d'effectuer les transformations en fichier de commande natif, sans parametres + boolean ReadWriteCommandFiles(); + + // Personnalisation des messages d'erreur + void AddInputCommandFileError(const ALString& sMessage) const; + void AddInputParameterFileError(const ALString& sMessage) const; + void AddOutputCommandFileError(const ALString& sMessage) const; + /////////////////////////////////////////////////////////////////////////////////////// ///// Implementation protected: + // Variante affichable d'une valeur, en completant si necessaire par des "..." + const ALString GetPrintableValue(const ALString& sValue) const; + + /////////////////////////////////////////////////////////////// + // Gestion du fichier de parametre json et de sa verification + // Chargement du fichier json en entree et verification de sa validite boolean LoadJsonParameters(); - // Test de la validite d'un nom de variable, avec creation si necessaire d'un message d'erreur + // Test de la validite d'une cle json, avec creation si necessaire d'un message d'erreur // complet, y compris la valeur testee - boolean CheckVariableName(const ALString& sValue, ALString& sMessage) const; + boolean CheckJsonKey(const ALString& sValue, ALString& sMessage) const; - // Test si une valeur correspond a un nom de variable au format camelCase - boolean IsCamelCaseVariableName(const ALString& sValue) const; + // Test si une valeur correspond a une cle json au format camelCase + boolean IsCamelCaseJsonKey(const ALString& sValue) const; - // Test si un nom de variable correspond a un contenu de type byte, donc encode au format base64 - // Un tel nom de variable au format camelCase doit etre prefixe par byte - boolean IsByteVariableName(const ALString& sValue) const; + // Test si une cle json correspond a un contenu de type byte, donc encode au format base64 + // Une telle cle json au format camelCase doit etre prefixe par byte + boolean IsByteJsonKey(const ALString& sValue) const; - // Transformation d'un nom de variable en sa variante byte ou standard + // Transformation d'une cle json en sa variante byte ou standard // On renvoie la valeur initiale si elle est deja dans sa bonne variante - const ALString ToByteVariableName(const ALString& sValue) const; - const ALString ToStandardVariableName(const ALString& sValue) const; + const ALString ToByteJsonKey(const ALString& sValue) const; + const ALString ToStandardJsonKey(const ALString& sValue) const; - // Transformation d'un nom de variable en sa variante opposee - const ALString ToVariantVariableName(const ALString& sValue) const; + // Transformation d'une cles json en sa variante opposee + const ALString ToVariantJsonKey(const ALString& sValue) const; // Test de la validite d'une valeur de type string, avec creation si necessaire d'un message d'erreur // complet, y compris la valeur testee boolean CheckStringValue(const ALString& sValue, boolean bCheckBase64Encoding, ALString& sMessage) const; - // Variante affichable d'une valeur, en completant si necessaire par des "..." - const ALString GetPrintableValue(const ALString& sValue) const; + // Construit d'un path json pour designer une valeur dans unse structure json + // Cf. https://jsonpatch.com/ + // On suit les element de structure valides dans le parametrage json + // La fin du parametrage peut etre non utilises (NULL ou -1) + ALString BuildJsonPath(JsonMember* member, int nArrayRank, JsonMember* arrayObjectmember) const; - // Personnalisation des messages d'erreur - void AddInputCommandFileError(const ALString& sMessage) const; - void AddInputParameterFileError(const ALString& sMessage) const; - void AddOutputCommandFileError(const ALString& sMessage) const; + /////////////////////////////////////////////////////////////// + // Gestion du fichier de commandes en entree // Application des recherche/remplacement de valeurs successivement sur une commande const ALString ProcessSearchReplaceCommand(const ALString& sInputCommand) const; - // Construit d'un path json pour designer une valeur dans unse structure json - // Cf. https://jsonpatch.com/ - // On suit les element de structure valides dans le parametrage json - // La fin du parametrage peut etre non utilises (NULL ou -1) - ALString BuildJsonPath(JsonMember* member, int nArrayRank, JsonMember* arrayObjectmember); + // Types de token possible + enum + { + TokenIf, + TokenLoop, + TokenEnd, + TokenKey, + TokenOther, + None + }; + + // Type de bloc pour un token de type bloc + const ALString& GetBlockType(int nToken) const; + + // Reinitialisation de la gestion du parser + void ResetParser(); + + // Recodage de la ligne de commande en cours en exploitant le parametrage json + // On renvoie la ligne recodee + // En cas d'erreur, le boolen en parametre est mis a false, avec emmission d'un message d'erreur + const ALString RecodeCurrentLineUsingJsonParameters(boolean& bOk); + + // Analyse d'une nouvelle ligne de commande pour mettre a jour l'etat du parser + // Etats possibles, gere par nParserState + // - TokenIf: en cours de traitement de bloc IF + // - TokenLoop: en cours de traitement de bloc LOOP + // - TokenOther: instruction standard + // Le parametre bContinueAnalysis en sortie indique qu'il faut continuer l'analyse + // du bloc en cours pour avoir une instruction executable disponible + // En cas d'erreur, on renvoie false, avec emmission d'un message d'erreur + boolean ParseInputCommand(const ALString& sInputCommand, boolean& bContinueAnalysis); + + // Tokenisation de la ligne de commande d'entree en une suite de tokens + // On renvoie la liste des types et valeur de tokens en sortie si la syntaxe est valide: + // TokenIf TokenKey: debut de bloc if + // TokenEnd TokenIf: fin de bloc if + // TokenLoop TokenKey: debut de bloc loop + // TokenEnd TokenLoop: fin de bloc loop + // TokenOther (TokenKey|TokenOther)*: instruction standard, avec commande suivi d'une eventuelle valeur + // En cas d'erreur, on renvoie false, avec emmission d'un message d'erreur + boolean TokenizeInputCommand(const ALString& sInputCommand, IntVector* ivTokenTypes, + StringVector* svTokenValues) const; + + // Decomposition de la ligne de commande d'entree en un premier token, + // suivi d'une valeur inter-token et de la fin de la ligne + // En sortie, on renvoie le type de token, et on indique la valeur du token la fin de ligne + // Il n'y a pas de message d'erreur a ce niveau. + // Les tokens de type TokenKey possedent a minimal leur delimiteur de debut, et peuvent ne pas etre valides + int GetFirstInputToken(const ALString& sInputCommand, ALString& sToken, ALString& sInterToken, + ALString& sEndLine) const; + + // Affichage d'un vecteur de token issu de l'analyse de la ligne de command + void WriteInputCommandTokens(ostream& ost, IntVector* ivTokenTypes, StringVector* svTokenValues) const; + + // Verification de la syntaxe d'un token de type cle, devant commencer par son delimiteur + // En cas d'erreur, on renvoie false, avec emmission d'un message d'erreur + boolean CheckTokenKey(const ALString& sToken) const; + + // Extraction de la cle d'un token de type cle valide entoure de ses delimiteurs + const ALString ExtractJsonKey(const ALString& sTokenKey) const; + + // Recherche de la valeur associe a une cle dans un objet json + // On renvoie NULL si non trouve + JsonValue* LookupJsonValue(JsonObject* jsonObject, const ALString& sKey) const; + + // Test si une valeur est trimee + boolean IsValueTrimed(const ALString& sValue) const; + + // Detection des membres non utilises du parametrage json, avec emission de messages d'erreur + // On renvoie true s'il y a au moins une erreur + boolean DetectedUnusedJsonParameterMembers() const; + + /////////////////////////////////////////////////////////////// + // Variables de specification des fichiers et parametres de commandes // Nom des fichiers ALString sInputCommandFileName; ALString sOutputCommandFileName; ALString sInputParameterFileName; + // Gestion des chaines des patterns a remplacer par des valeurs dans les fichiers d'input de scenario + StringVector svInputCommandSearchValues; + StringVector svInputCommandReplaceValues; + + /////////////////////////////////////////////////////////////// + // Variables de gestion des fichiers et parametres de commandes + + // Redirection de la sortie outputCommand vers la console + boolean bPrintOutputInConsole; + // Variante locale des noms de fichier de commande, dans le cas de fichiers HDFS ALString sLocalInputCommandFileName; ALString sLocalOutputCommandFileName; @@ -228,21 +335,89 @@ class CommandFile : public Object FILE* fInputCommands; FILE* fOutputCommands; - // Redirection de la sortie outputCommand vers la console - boolean bPrintOutputInConsole; - - // Gestion des chaines des patterns a remplacer par des valeurs dans les fichiers d'input de scenario - StringVector svInputCommandSearchValues; - StringVector svInputCommandReplaceValues; - // Object json pour les parametres en entree JsonObject jsonParameters; - // Prefixe des noms de variable ayant un contenu de type byte - static const ALString sByteVariablePrefix; + /////////////////////////////////////////////////////////////// + // Variables de gestion du parsing du fichier de commande en entree + // permettant sa gestion en flux + // + // Choix d'implementation principaux, avec impacts utilisateurs + // - le fichier de parametre json est lu et traite en entier de facon prealable, avec une taille limitee + // - le fichier de commande est traite en flux, ce qui permet de n'avoir aucune limite de taille + // - toute ligne de commande peut etre commentee, y compris les lignes du langage de pilotage de type IF ou LOOP + // - les lignes d'un fichier de commande template n'ont pas besoin de se terminer par un commentaire + // - toute __key__ du fichier de commande doit se trouver dans le fichier json + // - toute key du fichier json doit etre utilise dans le fichier de commande + // - les __key__ de parametrage json ne peuvent concerner que la partie parametrage utilisateur d'une valeur + // - toute erreur ou incoherence dans les fichiers de commande et de parametrage json provoquent une erreur fatale + + // Numero de ligne courant du fichier de commande en entree + int nParserLineIndex; + + // Gestion de l'etat courant du parser de commande + int nParserState; + + // Gestion de l'etat courant de la ligne en couurs de parsing + int nParserCurrentLineState; + + // Cle du bloc en cours de traitement + ALString sParserBlockKey; - // Longueur max d'un nom de variable - static const int nMaxVariableNameLength = 100; + // Vecteur des types et valeurs des tokens de la ligne en cours de traitement + IntVector ivParserTokenTypes; + StringVector svParserTokenValues; + + // Indicateur de traitement dans le cas d'un bloc de type if en cours + boolean bParserIfState; + + // Tableau en cours du parametrage json dans le cas d'un bloc loop en cours + JsonArray* parserLoopJsonArray; + + // Tableau des vecteur de type et valeurs de tokens pour les lignes de bloc loop en cours + ObjectArray oaParserLoopLinesTokenTypes; + ObjectArray oaParserLoopLinesTokenValues; + + // Index de la ligne de bloc en cours de traitement + int nParserLoopLineIndex; + + // Index de l'objet json du tableau en cours de traitement + int nParserLoopObjectIndex; + + // Indicateur d'erreur du parser dans l'analyse des commande + // Cet indicateur est mis a jour uniquement dans la methode d'ajout d'erreur + // Il permet de conditionner la fin de l'analyse des commandes lors de la fermeture + // du fichier de commande, pour detecter les erreurs de parsing de fin de fichier + mutable boolean bParserOk; + + // Dictionnaire des membres utilises des parametres json + // Ce dictionnaire est mise a jour lors de chaque recherche de membre par cle + // Cela permet de detecter les membres non utilises lors de la fermeture du fichier de commande + mutable NumericKeyDictionary nkdParserUsedJsonParameterMembers; + + /////////////////////////////////////////////////////////////// + // Constantes sur les mots cles du langage de commande en entree + // et sur les contrainte de taille des elements de langage + + // Valeurs des tokens du langage de pametrage des commande en entree + static const ALString sTokenLoop; + static const ALString sTokenIf; + static const ALString sTokenEnd; + + // Prefix de commentaire + static const ALString sCommentPrefix; + + // Delimiteur de cle json dans un fichier de commande + static const ALString sJsonKeyDelimiter; + + // Prefixe des cle json ayant un contenu de type byte + static const ALString sByteJsonKeyPrefix; + + // Longueur max d'une ligne de fichier de commande + static const int nMaxLineLength = 500; + + // Longueur max d'une cle json + static const int nMaxJsonKeyLength = 100; // Longueur max d'une valeur de type chaine de caracteres static const int nMaxStringValueLength = 300; @@ -250,6 +425,9 @@ class CommandFile : public Object // Longueur max affichee pour une valeur dans les messages d'erreur static const int nMaxPrintableLength = 30; + // Taille max d'un bloc d'instruction LOOP d'un fichier de commande, en nombre de lignes utiles + static const int nLoopMaxLineNumber = 1000; + // Taille max d'un fichier de parametrage static const longint lMaxInputParameterFileSize = lMB; }; diff --git a/src/Norm/base/JSONTokenizer.cpp b/src/Norm/base/JSONTokenizer.cpp index 242aad72..3365ec32 100644 --- a/src/Norm/base/JSONTokenizer.cpp +++ b/src/Norm/base/JSONTokenizer.cpp @@ -11,17 +11,6 @@ ALString JSONTokenizer::sLocalFileName; FILE* JSONTokenizer::fJSON = NULL; int JSONTokenizer::nLastToken = 0; JSONSTYPE JSONTokenizer::jsonLastTokenValue = {0}; -boolean JSONTokenizer::bForceAnsi = false; - -void JSONTokenizer::SetForceAnsi(boolean bValue) -{ - bForceAnsi = bValue; -} - -boolean JSONTokenizer::GetForceAnsi() -{ - return bForceAnsi; -} boolean JSONTokenizer::OpenForRead(const ALString& sErrorFamilyName, const ALString& sInputFileName) { @@ -95,7 +84,7 @@ boolean JSONTokenizer::CheckToken(int nToken) { if (nToken == '[' or nToken == ']' or nToken == '{' or nToken == '}' or nToken == ',' or nToken == ':' or nToken == 0 or nToken == String or nToken == Number or nToken == Boolean or nToken == Null or - nToken == Error) + nToken == StringError or nToken == Error) return true; else return false; @@ -108,22 +97,26 @@ ALString JSONTokenizer::GetTokenLabel(int nToken) require(CheckToken(nToken)); if (nToken == String) - sLabel = "String"; + sLabel = "string"; else if (nToken == Number) - sLabel = "Number"; + sLabel = "number"; else if (nToken == Boolean) - sLabel = "Boolean"; + sLabel = "boolean"; else if (nToken == Null) - sLabel = "Null"; + sLabel = "null"; + else if (nToken == StringError) + sLabel = "string error"; else if (nToken == Error) sLabel = "Error"; else if (nToken == 0) sLabel = "EndOfFile"; - else + else if (nToken < 256) { sLabel = "' '"; sLabel.SetAt(1, (char)nToken); } + else + sLabel = "unknown"; return sLabel; } @@ -140,13 +133,13 @@ int JSONTokenizer::ReadNextToken() // Lecture du token nLastToken = JsonObject::Lex(&jsonLastTokenValue); - // On force la conversion vers l'ansi si necessaire - if (bForceAnsi and nLastToken == String) + // On accepte avec un warning les erreur d'encodage de chaine de caracteres + if (nLastToken == StringError) { - sValueCopy = *jsonLastTokenValue.sValue; - TextService::CToCAnsiString(sValueCopy, *jsonLastTokenValue.sValue); + nLastToken = String; + AddParseWarning("Read token " + GetTokenLabel(nLastToken) + GetLastTokenValue() + + " with non-utf8 encoding"); } - return nLastToken; } @@ -611,6 +604,11 @@ boolean JSONTokenizer::DoubleToInt(double dValue, int& nValue) } } +void JSONTokenizer::AddParseWarning(const ALString& sLabel) +{ + Global::AddWarning(sErrorFamily, sFileName + " line " + IntToString(GetCurrentLineIndex()), sLabel); +} + void JSONTokenizer::AddParseError(const ALString& sLabel) { Global::AddError(sErrorFamily, sFileName + " line " + IntToString(GetCurrentLineIndex()), sLabel); diff --git a/src/Norm/base/JSONTokenizer.h b/src/Norm/base/JSONTokenizer.h index 3a868b00..7b044abe 100644 --- a/src/Norm/base/JSONTokenizer.h +++ b/src/Norm/base/JSONTokenizer.h @@ -24,11 +24,6 @@ class JSONTokenizer : public TextService ////////////////////////////////////////////////////////////////////////////// // Parametrage du fichier a analyser - // Parametrage de la lecture, en forcant ou non la conversion des caracteres utf8 de windows1252/iso8859-1 vers - // l'ansi etendu (defaut: false) - static void SetForceAnsi(boolean bValue); - static boolean GetForceAnsi(); - // Ouverture du fichier pour analyse // Le premier parametre sera utilise si necessaire pour parametrer les messages d'erreur static boolean OpenForRead(const ALString& sErrorFamilyName, const ALString& sInputFileName); @@ -53,6 +48,7 @@ class JSONTokenizer : public TextService Number = NUMBERVALUE, Boolean = BOOLEANVALUE, Null = NULLVALUE, + StringError = STRINGERROR, Error = ERROR, }; @@ -120,7 +116,8 @@ class JSONTokenizer : public TextService // Emet un message d'erreur si on est pas a la fin static boolean CheckObjectEnd(const ALString& sKey, boolean bIsEnd); - // Emission d'une erreur de parsing + // Emission d'un warning ou d'une erreur de parsing + static void AddParseWarning(const ALString& sLabel); static void AddParseError(const ALString& sLabel); ////////////////////////////////////////////////////////////////////////////// @@ -156,7 +153,4 @@ class JSONTokenizer : public TextService // Valeur du dernier token static JSONSTYPE jsonLastTokenValue; - - // Parametrage de la conversion vers l'ansi - static boolean bForceAnsi; }; diff --git a/src/Norm/base/JsonLex.inc b/src/Norm/base/JsonLex.inc index 41c47c27..2a9b4fc5 100644 --- a/src/Norm/base/JsonLex.inc +++ b/src/Norm/base/JsonLex.inc @@ -1104,18 +1104,29 @@ case 10: YY_RULE_SETUP #line 49 "JsonLex.lex" { + boolean bOk; ALString *sValue; + // Conversion sans les double-quote de debut et fin sValue = new ALString; yytext[yyleng-1] = '\0'; - TextService::JsonToCString((char*)&yytext[1], *sValue); + bOk = TextService::JsonToCString((char*)&yytext[1], *sValue); jsonlval.sValue = sValue; - return STRINGVALUE; + + // On retourne la chaine convertie en cas de succes + if (bOk) + return STRINGVALUE; + // Sinon, on retourne la chaine originelle + else + { + *sValue = (char*)&yytext[1]; + return STRINGERROR; + } } YY_BREAK case 11: YY_RULE_SETUP -#line 59 "JsonLex.lex" +#line 70 "JsonLex.lex" { char* endptr; double dValue; @@ -1128,12 +1139,12 @@ YY_RULE_SETUP case 12: /* rule 12 can match eol */ YY_RULE_SETUP -#line 68 "JsonLex.lex" +#line 79 "JsonLex.lex" {/*IGNORE*/} YY_BREAK case 13: YY_RULE_SETUP -#line 70 "JsonLex.lex" +#line 81 "JsonLex.lex" { ALString *sValue; @@ -1144,10 +1155,10 @@ YY_RULE_SETUP YY_BREAK case 14: YY_RULE_SETUP -#line 78 "JsonLex.lex" +#line 89 "JsonLex.lex" ECHO; YY_BREAK -#line 1150 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonLex.inc" +#line 1161 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonLex.inc" case YY_STATE_EOF(INITIAL): yyterminate(); @@ -2164,5 +2175,5 @@ void yyfree (void * ptr ) #define YYTABLES_NAME "yytables" -#line 78 "JsonLex.lex" +#line 89 "JsonLex.lex" diff --git a/src/Norm/base/JsonLex.lex b/src/Norm/base/JsonLex.lex index b6d63612..ce1c8c0f 100644 --- a/src/Norm/base/JsonLex.lex +++ b/src/Norm/base/JsonLex.lex @@ -47,13 +47,24 @@ null {return NULLVALUE;} {STRING} { + boolean bOk; ALString *sValue; + // Conversion sans les double-quote de debut et fin sValue = new ALString; yytext[yyleng-1] = '\0'; - TextService::JsonToCString((char*)&yytext[1], *sValue); + bOk = TextService::JsonToCString((char*)&yytext[1], *sValue); jsonlval.sValue = sValue; - return STRINGVALUE; + + // On retourne la chaine convertie en cas de succes + if (bOk) + return STRINGVALUE; + // Sinon, on retourne la chaine originelle + else + { + *sValue = (char*)&yytext[1]; + return STRINGERROR; + } } {NUMBER} { diff --git a/src/Norm/base/JsonYac.cpp b/src/Norm/base/JsonYac.cpp index 5a3ad677..c41d1a14 100644 --- a/src/Norm/base/JsonYac.cpp +++ b/src/Norm/base/JsonYac.cpp @@ -89,10 +89,12 @@ #include "JsonObject.h" /* Declaration du lexer utilise */ -void jsonerror(char const* fmt); -void jsonerrorWithLineCorrection(char const* fmt, int nDeltaLineNumber); int jsonlex(); +/* Methode de gestion des erreurs */ +void jsonerror(char const* fmt); +const ALString jsonPrintableValue(const ALString& sValue); + /* Objet json principal a utiliser pendant la lecture d'un fichier. */ /* Ce domaine est positionner par la methode ReadFile de JsonObject */ static JsonObject* mainReadJsonObject = NULL; @@ -119,7 +121,7 @@ static int nJsonFileParsingErrorNumber = 0; extern char *yyptok(int i); */ -#line 120 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" +#line 122 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" #ifndef YY_CAST #ifdef __cplusplus @@ -153,22 +155,23 @@ enum yysymbol_kind_t YYSYMBOL_STRINGVALUE = 3, /* STRINGVALUE */ YYSYMBOL_NUMBERVALUE = 4, /* NUMBERVALUE */ YYSYMBOL_BOOLEANVALUE = 5, /* BOOLEANVALUE */ - YYSYMBOL_ERROR = 6, /* ERROR */ - YYSYMBOL_NULLVALUE = 7, /* NULLVALUE */ - YYSYMBOL_8_ = 8, /* '{' */ - YYSYMBOL_9_ = 9, /* '}' */ - YYSYMBOL_10_ = 10, /* ',' */ - YYSYMBOL_11_ = 11, /* ':' */ - YYSYMBOL_12_ = 12, /* '[' */ - YYSYMBOL_13_ = 13, /* ']' */ - YYSYMBOL_YYACCEPT = 14, /* $accept */ - YYSYMBOL_jsonFile = 15, /* jsonFile */ - YYSYMBOL_value = 16, /* value */ - YYSYMBOL_object = 17, /* object */ - YYSYMBOL_members = 18, /* members */ - YYSYMBOL_member = 19, /* member */ - YYSYMBOL_array = 20, /* array */ - YYSYMBOL_values = 21 /* values */ + YYSYMBOL_STRINGERROR = 6, /* STRINGERROR */ + YYSYMBOL_ERROR = 7, /* ERROR */ + YYSYMBOL_NULLVALUE = 8, /* NULLVALUE */ + YYSYMBOL_9_ = 9, /* '{' */ + YYSYMBOL_10_ = 10, /* '}' */ + YYSYMBOL_11_ = 11, /* ',' */ + YYSYMBOL_12_ = 12, /* ':' */ + YYSYMBOL_13_ = 13, /* '[' */ + YYSYMBOL_14_ = 14, /* ']' */ + YYSYMBOL_YYACCEPT = 15, /* $accept */ + YYSYMBOL_jsonFile = 16, /* jsonFile */ + YYSYMBOL_value = 17, /* value */ + YYSYMBOL_object = 18, /* object */ + YYSYMBOL_members = 19, /* members */ + YYSYMBOL_member = 20, /* member */ + YYSYMBOL_array = 21, /* array */ + YYSYMBOL_values = 22 /* values */ }; typedef enum yysymbol_kind_t yysymbol_kind_t; @@ -473,21 +476,21 @@ union yyalloc #endif /* !YYCOPY_NEEDED */ /* YYFINAL -- State number of the termination state. */ -#define YYFINAL 10 +#define YYFINAL 11 /* YYLAST -- Last index in YYTABLE. */ -#define YYLAST 30 +#define YYLAST 34 /* YYNTOKENS -- Number of terminals. */ -#define YYNTOKENS 14 +#define YYNTOKENS 15 /* YYNNTS -- Number of nonterminals. */ #define YYNNTS 8 /* YYNRULES -- Number of rules. */ -#define YYNRULES 20 +#define YYNRULES 22 /* YYNSTATES -- Number of states. */ -#define YYNSTATES 30 +#define YYNSTATES 32 /* YYMAXUTOK -- Last valid token kind. */ -#define YYMAXUTOK 262 +#define YYMAXUTOK 263 /* YYTRANSLATE(TOKEN-NUM) -- Symbol number corresponding to TOKEN-NUM as returned by yylex, with out-of-bounds checking. */ @@ -497,19 +500,19 @@ union yyalloc /* YYTRANSLATE[TOKEN-NUM] -- Symbol number corresponding to TOKEN-NUM as returned by yylex. */ static const yytype_int8 yytranslate[] = { - 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 10, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 11, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 12, 2, 13, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 8, 2, 9, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 2, 3, 4, 5, 6, 7}; + 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 11, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 12, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 13, 2, 14, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 9, 2, 10, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 2, 3, 4, 5, 6, 7, 8}; #if JSONDEBUG /* YYRLINE[YYN] -- Source line where rule number YYN was defined. */ -static const yytype_int16 yyrline[] = {0, 76, 76, 95, 103, 104, 105, 117, 130, 143, 151, - 163, 165, 169, 182, 209, 221, 238, 240, 244, 257}; +static const yytype_int16 yyrline[] = {0, 79, 79, 98, 106, 107, 108, 120, 133, 146, 154, 164, + 180, 182, 186, 199, 226, 236, 252, 269, 271, 275, 288}; #endif /** Accessing symbol of state STATE. */ @@ -528,6 +531,7 @@ static const char* const yytname[] = {"\"end of file\"", "STRINGVALUE", "NUMBERVALUE", "BOOLEANVALUE", + "STRINGERROR", "ERROR", "NULLVALUE", "'{'", @@ -552,7 +556,7 @@ static const char* yysymbol_name(yysymbol_kind_t yysymbol) } #endif -#define YYPACT_NINF (-5) +#define YYPACT_NINF (-4) #define yypact_value_is_default(Yyn) ((Yyn) == YYPACT_NINF) @@ -562,40 +566,41 @@ static const char* yysymbol_name(yysymbol_kind_t yysymbol) /* YYPACT[STATE-NUM] -- Index in YYTABLE of the portion describing STATE-NUM. */ -static const yytype_int8 yypact[] = {18, -5, 15, 6, -5, -4, -5, -5, 13, -5, -5, 8, -5, 5, -5, - -5, -5, -5, -5, -3, -5, -5, -5, -5, -5, -5, 17, 8, -5, -5}; +static const yytype_int8 yypact[] = {18, -4, 23, 7, -4, 11, -4, -4, -4, -2, -4, -4, 9, -4, 22, -4, + -4, -4, -4, -4, -4, -3, -4, -4, -4, -4, -4, -4, 10, 9, -4, -4}; /* YYDEFACT[STATE-NUM] -- Default reduction number in state STATE-NUM. Performed when YYTABLE does not specify something else to do. Zero means the default is an error. */ -static const yytype_int8 yydefact[] = {0, 3, 0, 0, 2, 0, 15, 11, 0, 13, 1, 0, 12, 0, 6, - 7, 8, 10, 9, 0, 16, 4, 5, 14, 17, 19, 0, 0, 18, 20}; +static const yytype_int8 yydefact[] = {0, 3, 0, 0, 2, 0, 16, 17, 12, 0, 14, 1, 0, 13, 0, 6, + 7, 8, 10, 11, 9, 0, 18, 4, 5, 15, 19, 21, 0, 0, 20, 22}; /* YYPGOTO[NTERM-NUM]. */ -static const yytype_int8 yypgoto[] = {-5, -5, -2, 28, -5, 16, -5, -5}; +static const yytype_int8 yypgoto[] = {-4, -4, -1, 31, -4, 20, -4, -4}; /* YYDEFGOTO[NTERM-NUM]. */ -static const yytype_int8 yydefgoto[] = {0, 3, 20, 21, 8, 9, 22, 26}; +static const yytype_int8 yydefgoto[] = {0, 3, 22, 23, 9, 10, 24, 28}; /* YYTABLE[YYPACT[STATE-NUM]] -- What to do in state STATE-NUM. If positive, shift that token. If negative, reduce the rule whose number is the opposite. If YYTABLE_NINF, syntax error. */ -static const yytype_int8 yytable[] = {14, 15, 16, 17, 18, 2, 10, 11, 5, 19, 24, 14, 15, 16, 17, 18, - 2, 25, 5, 1, 19, 6, 12, 13, 7, 29, 2, 27, 4, 23, 28}; +static const yytype_int8 yytable[] = {15, 16, 17, 18, 19, 20, 2, 11, 13, 14, 21, 26, 15, 16, 17, 18, 19, 20, + 2, 1, 27, 29, 21, 12, 30, 5, 5, 2, 31, 6, 7, 4, 0, 8, 25}; -static const yytype_int8 yycheck[] = {3, 4, 5, 6, 7, 8, 0, 11, 3, 12, 13, 3, 4, 5, 6, 7, - 8, 19, 3, 1, 12, 6, 9, 10, 9, 27, 8, 10, 0, 13, 13}; +static const yytype_int8 yycheck[] = {3, 4, 5, 6, 7, 8, 9, 0, 10, 11, 13, 14, 3, 4, 5, 6, 7, 8, + 9, 1, 21, 11, 13, 12, 14, 3, 3, 9, 29, 6, 7, 0, -1, 10, 14}; /* YYSTOS[STATE-NUM] -- The symbol kind of the accessing symbol of state STATE-NUM. */ -static const yytype_int8 yystos[] = {0, 1, 8, 15, 17, 3, 6, 9, 18, 19, 0, 11, 9, 10, 3, - 4, 5, 6, 7, 12, 16, 17, 20, 19, 13, 16, 21, 10, 13, 16}; +static const yytype_int8 yystos[] = {0, 1, 9, 16, 18, 3, 6, 7, 10, 19, 20, 0, 12, 10, 11, 3, + 4, 5, 6, 7, 8, 13, 17, 18, 21, 20, 14, 17, 22, 11, 14, 17}; /* YYR1[RULE-NUM] -- Symbol kind of the left-hand side of rule RULE-NUM. */ -static const yytype_int8 yyr1[] = {0, 14, 15, 15, 16, 16, 16, 16, 16, 16, 16, 17, 17, 18, 18, 18, 19, 20, 20, 21, 21}; +static const yytype_int8 yyr1[] = {0, 15, 16, 16, 17, 17, 17, 17, 17, 17, 17, 17, + 18, 18, 19, 19, 19, 19, 20, 21, 21, 22, 22}; /* YYR2[RULE-NUM] -- Number of symbols on the right-hand side of rule RULE-NUM. */ -static const yytype_int8 yyr2[] = {0, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 3, 1, 3, 1, 3, 2, 3, 1, 3}; +static const yytype_int8 yyr2[] = {0, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 3, 1, 3, 1, 1, 3, 2, 3, 1, 3}; enum { @@ -776,93 +781,103 @@ static void yydestruct(const char* yymsg, yysymbol_kind_t yykind, YYSTYPE* yyval switch (yykind) { case YYSYMBOL_STRINGVALUE: /* STRINGVALUE */ -#line 69 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" +#line 72 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" { if (((*yyvaluep).sValue) != NULL) delete ((*yyvaluep).sValue); ((*yyvaluep).sValue) = NULL; } -#line 875 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" +#line 881 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" + break; + + case YYSYMBOL_STRINGERROR: /* STRINGERROR */ +#line 72 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" + { + if (((*yyvaluep).sValue) != NULL) + delete ((*yyvaluep).sValue); + ((*yyvaluep).sValue) = NULL; + } +#line 887 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" break; case YYSYMBOL_ERROR: /* ERROR */ -#line 69 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" +#line 72 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" { if (((*yyvaluep).sValue) != NULL) delete ((*yyvaluep).sValue); ((*yyvaluep).sValue) = NULL; } -#line 881 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" +#line 893 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" break; case YYSYMBOL_jsonFile: /* jsonFile */ -#line 69 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" +#line 72 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" { if (((*yyvaluep).jsonObject) != NULL) delete ((*yyvaluep).jsonObject); ((*yyvaluep).jsonObject) = NULL; } -#line 887 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" +#line 899 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" break; case YYSYMBOL_value: /* value */ -#line 69 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" +#line 72 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" { if (((*yyvaluep).jsonValue) != NULL) delete ((*yyvaluep).jsonValue); ((*yyvaluep).jsonValue) = NULL; } -#line 893 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" +#line 905 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" break; case YYSYMBOL_object: /* object */ -#line 69 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" +#line 72 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" { if (((*yyvaluep).jsonObject) != NULL) delete ((*yyvaluep).jsonObject); ((*yyvaluep).jsonObject) = NULL; } -#line 899 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" +#line 911 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" break; case YYSYMBOL_members: /* members */ -#line 69 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" +#line 72 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" { if (((*yyvaluep).jsonObject) != NULL) delete ((*yyvaluep).jsonObject); ((*yyvaluep).jsonObject) = NULL; } -#line 905 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" +#line 917 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" break; case YYSYMBOL_member: /* member */ -#line 69 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" +#line 72 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" { if (((*yyvaluep).jsonMember) != NULL) delete ((*yyvaluep).jsonMember); ((*yyvaluep).jsonMember) = NULL; } -#line 911 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" +#line 923 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" break; case YYSYMBOL_array: /* array */ -#line 69 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" +#line 72 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" { if (((*yyvaluep).jsonArray) != NULL) delete ((*yyvaluep).jsonArray); ((*yyvaluep).jsonArray) = NULL; } -#line 917 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" +#line 929 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" break; case YYSYMBOL_values: /* values */ -#line 69 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" +#line 72 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" { if (((*yyvaluep).jsonArray) != NULL) delete ((*yyvaluep).jsonArray); ((*yyvaluep).jsonArray) = NULL; } -#line 923 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" +#line 935 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" break; default: @@ -1112,7 +1127,7 @@ int yyparse(void) switch (yyn) { case 2: /* jsonFile: object */ -#line 77 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" +#line 80 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" { JsonObject* jsonObject; JsonMember* member; @@ -1131,36 +1146,36 @@ int yyparse(void) } jsonObject->RemoveAll(); } -#line 1210 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" +#line 1222 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" break; case 3: /* jsonFile: error */ -#line 96 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" +#line 99 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" { (yyval.jsonObject) = NULL; YYABORT; } -#line 1219 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" +#line 1231 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" break; case 4: /* value: object */ -#line 103 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" +#line 106 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" { (yyval.jsonValue) = (yyvsp[0].jsonObject); } -#line 1225 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" +#line 1237 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" break; case 5: /* value: array */ -#line 104 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" +#line 107 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" { (yyval.jsonValue) = (yyvsp[0].jsonArray); } -#line 1231 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" +#line 1243 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" break; case 6: /* value: STRINGVALUE */ -#line 106 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" +#line 109 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" { JsonString* jsonString; @@ -1172,11 +1187,11 @@ int yyparse(void) jsonString->SetString(*(yyvsp[0].sValue)); delete (yyvsp[0].sValue); } -#line 1247 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" +#line 1259 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" break; case 7: /* value: NUMBERVALUE */ -#line 118 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" +#line 121 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" { JsonNumber* jsonNumber; double dValue; @@ -1189,11 +1204,11 @@ int yyparse(void) dValue = (yyvsp[0].dValue); jsonNumber->SetNumber(dValue); } -#line 1264 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" +#line 1276 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" break; case 8: /* value: BOOLEANVALUE */ -#line 131 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" +#line 134 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" { JsonBoolean* jsonBoolean; boolean bValue; @@ -1206,11 +1221,11 @@ int yyparse(void) bValue = (yyvsp[0].bValue); jsonBoolean->SetBoolean(bValue); } -#line 1281 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" +#line 1293 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" break; case 9: /* value: NULLVALUE */ -#line 144 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" +#line 147 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" { JsonNull* jsonNull; @@ -1218,41 +1233,60 @@ int yyparse(void) jsonNull = new JsonNull; (yyval.jsonValue) = jsonNull; } -#line 1293 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" +#line 1305 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" break; - case 10: /* value: ERROR */ -#line 152 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" + case 10: /* value: STRINGERROR */ +#line 155 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" { // Message d'erreur - jsonerror("Invalid json value"); + jsonerror("Invalid json string \"" + jsonPrintableValue(*(yyvsp[0].sValue)) + + "\" with non-utf8 encoding"); + + // Nettoyage et sortie + delete (yyvsp[0].sValue); + (yyval.jsonValue) = NULL; + YYABORT; + } +#line 1319 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" + break; + + case 11: /* value: ERROR */ +#line 165 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" + { + ALString sValueError; + + // Message d'erreur, avec indication de la valeur sauf si elle se reduit au caractete '"' + if (*(yyvsp[0].sValue) != "\"") + sValueError = " \"" + jsonPrintableValue(*(yyvsp[0].sValue)) + "\""; + jsonerror("Invalid json value" + sValueError); // Nettoyage et sortie delete (yyvsp[0].sValue); (yyval.jsonValue) = NULL; YYABORT; } -#line 1307 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" +#line 1337 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" break; - case 11: /* object: '{' '}' */ -#line 164 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" + case 12: /* object: '{' '}' */ +#line 181 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" { (yyval.jsonObject) = new JsonObject; } -#line 1313 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" +#line 1343 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" break; - case 12: /* object: '{' members '}' */ -#line 166 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" + case 13: /* object: '{' members '}' */ +#line 183 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" { (yyval.jsonObject) = (yyvsp[-1].jsonObject); } -#line 1319 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" +#line 1349 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" break; - case 13: /* members: member */ -#line 170 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" + case 14: /* members: member */ +#line 187 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" { JsonObject* jsonObject; JsonMember* member; @@ -1265,11 +1299,11 @@ int yyparse(void) member = cast(JsonMember*, (yyvsp[0].jsonMember)); jsonObject->AddMember(member); } -#line 1336 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" +#line 1366 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" break; - case 14: /* members: members ',' member */ -#line 183 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" + case 15: /* members: members ',' member */ +#line 200 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" { JsonObject* jsonObject; JsonMember* member; @@ -1287,7 +1321,7 @@ int yyparse(void) else { TextService::CToJsonString(member->GetKey(), sJsonKey); - jsonerror("Duplicate key \"" + sJsonKey + "\" in json object"); + jsonerror("Duplicate key \"" + jsonPrintableValue(sJsonKey) + "\" in json object"); // Nettoyage et sortie delete member; @@ -1296,25 +1330,43 @@ int yyparse(void) YYABORT; } } -#line 1367 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" +#line 1397 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" break; - case 15: /* members: ERROR */ -#line 210 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" + case 16: /* members: STRINGERROR */ +#line 227 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" { // Message d'erreur - jsonerror("Invalid json key"); + jsonerror("Invalid json key \"" + jsonPrintableValue(*(yyvsp[0].sValue)) + "\" with non-utf8 encoding"); + + // Nettoyage et sortie + delete (yyvsp[0].sValue); + (yyval.jsonObject) = NULL; + YYABORT; + } +#line 1411 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" + break; + + case 17: /* members: ERROR */ +#line 237 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" + { + ALString sValueError; + + // Message d'erreur, avec indication de la valeur sauf si elle se reduit au caractete '"' + if (*(yyvsp[0].sValue) != "\"") + sValueError = " \"" + jsonPrintableValue(*(yyvsp[0].sValue)) + "\""; + jsonerror("Invalid json key" + sValueError); // Nettoyage et sortie delete (yyvsp[0].sValue); (yyval.jsonObject) = NULL; YYABORT; } -#line 1381 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" +#line 1429 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" break; - case 16: /* member: STRINGVALUE ':' value */ -#line 222 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" + case 18: /* member: STRINGVALUE ':' value */ +#line 253 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" { JsonMember* member; JsonValue* jsonValue; @@ -1329,27 +1381,27 @@ int yyparse(void) member->SetValue(jsonValue); delete (yyvsp[-2].sValue); } -#line 1400 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" +#line 1448 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" break; - case 17: /* array: '[' ']' */ -#line 239 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" + case 19: /* array: '[' ']' */ +#line 270 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" { (yyval.jsonArray) = new JsonArray; } -#line 1406 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" +#line 1454 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" break; - case 18: /* array: '[' values ']' */ -#line 241 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" + case 20: /* array: '[' values ']' */ +#line 272 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" { (yyval.jsonArray) = (yyvsp[-1].jsonArray); } -#line 1412 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" +#line 1460 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" break; - case 19: /* values: value */ -#line 245 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" + case 21: /* values: value */ +#line 276 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" { JsonArray* jsonArray; JsonValue* jsonValue; @@ -1362,11 +1414,11 @@ int yyparse(void) jsonValue = cast(JsonValue*, (yyvsp[0].jsonValue)); jsonArray->AddValue(jsonValue); } -#line 1429 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" +#line 1477 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" break; - case 20: /* values: values ',' value */ -#line 258 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" + case 22: /* values: values ',' value */ +#line 289 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" { JsonArray* jsonArray; JsonValue* jsonValue; @@ -1379,10 +1431,10 @@ int yyparse(void) jsonValue = cast(JsonValue*, (yyvsp[0].jsonValue)); jsonArray->AddValue(jsonValue); } -#line 1446 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" +#line 1494 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" break; -#line 1450 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" +#line 1498 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.cpp" default: break; @@ -1561,32 +1613,32 @@ int yyparse(void) return yyresult; } -#line 272 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" +#line 303 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" #include "JsonLex.inc" /* default yyerror for YACC and LEX */ void jsonerror(char const* fmt) -{ - jsonerrorWithLineCorrection(fmt, 0); -} - -/* Variante avec une correction du numero de ligne */ -void jsonerrorWithLineCorrection(char const* fmt, int nDeltaLineNumber) { char sErrorLine[20]; ALString sLabel; - int nLineNumber; nJsonFileParsingErrorNumber++; - nLineNumber = yylineno + nDeltaLineNumber; - if (nLineNumber <= 0) - nLineNumber = 1; - snprintf(sErrorLine, sizeof(sErrorLine), "Line %d", nLineNumber); + snprintf(sErrorLine, sizeof(sErrorLine), "Line %d", yylineno); sLabel = fmt; Global::AddError("Read json file", sErrorLine, sLabel); } +const ALString jsonPrintableValue(const ALString& sValue) +{ + const int nMaxPrintableLength = 30; + + if (sValue.GetLength() <= nMaxPrintableLength) + return sValue; + else + return sValue.Left(nMaxPrintableLength) + "..."; +} + int jsonparse(); /* Implementation de la methode de lecture de fichier de KWClassDomain */ diff --git a/src/Norm/base/JsonYac.hpp b/src/Norm/base/JsonYac.hpp index 7e032eab..5687c94b 100644 --- a/src/Norm/base/JsonYac.hpp +++ b/src/Norm/base/JsonYac.hpp @@ -69,8 +69,9 @@ enum jsontokentype STRINGVALUE = 258, /* STRINGVALUE */ NUMBERVALUE = 259, /* NUMBERVALUE */ BOOLEANVALUE = 260, /* BOOLEANVALUE */ - ERROR = 261, /* ERROR */ - NULLVALUE = 262 /* NULLVALUE */ + STRINGERROR = 261, /* STRINGERROR */ + ERROR = 262, /* ERROR */ + NULLVALUE = 263 /* NULLVALUE */ }; typedef enum jsontokentype jsontoken_kind_t; #endif @@ -79,7 +80,7 @@ typedef enum jsontokentype jsontoken_kind_t; #if !defined JSONSTYPE && !defined JSONSTYPE_IS_DECLARED union JSONSTYPE { -#line 44 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" +#line 46 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.yac" ALString* sValue; double dValue; @@ -89,7 +90,7 @@ union JSONSTYPE JsonArray* jsonArray; JsonMember* jsonMember; -#line 89 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.hpp" +#line 90 "C:/Applications/boullema/DevGit/khiops/src/Norm/base/JsonYac.hpp" }; typedef union JSONSTYPE JSONSTYPE; #define JSONSTYPE_IS_TRIVIAL 1 diff --git a/src/Norm/base/JsonYac.yac b/src/Norm/base/JsonYac.yac index 5bcf7b6c..e2c80fbb 100644 --- a/src/Norm/base/JsonYac.yac +++ b/src/Norm/base/JsonYac.yac @@ -7,10 +7,12 @@ #include "JsonObject.h" /* Declaration du lexer utilise */ -void jsonerror(char const *fmt); -void jsonerrorWithLineCorrection(char const *fmt, int nDeltaLineNumber); int jsonlex(); +/* Methode de gestion des erreurs */ +void jsonerror(char const *fmt); +const ALString jsonPrintableValue(const ALString& sValue); + /* Objet json principal a utiliser pendant la lecture d'un fichier. */ /* Ce domaine est positionner par la methode ReadFile de JsonObject */ static JsonObject* mainReadJsonObject=NULL; @@ -54,6 +56,7 @@ extern char *yyptok(int i); %token STRINGVALUE %token NUMBERVALUE %token BOOLEANVALUE +%token STRINGERROR %token ERROR %token NULLVALUE %type jsonFile @@ -148,10 +151,24 @@ value: object { $$ = $1; } jsonNull = new JsonNull; $$ = jsonNull; } - | ERROR + | STRINGERROR { // Message d'erreur - jsonerror("Invalid json value"); + jsonerror("Invalid json string \"" + jsonPrintableValue(*$1) + "\" with non-utf8 encoding"); + + // Nettoyage et sortie + delete $1; + $$ = NULL; + YYABORT; + } + | ERROR + { + ALString sValueError; + + // Message d'erreur, avec indication de la valeur sauf si elle se reduit au caractete '"' + if (*$1 != "\"") + sValueError = " \"" + jsonPrintableValue(*$1) + "\""; + jsonerror("Invalid json value" + sValueError); // Nettoyage et sortie delete $1; @@ -197,7 +214,7 @@ members: member else { TextService::CToJsonString(member->GetKey(), sJsonKey); - jsonerror("Duplicate key \"" + sJsonKey + "\" in json object"); + jsonerror("Duplicate key \"" + jsonPrintableValue(sJsonKey) + "\" in json object"); // Nettoyage et sortie delete member; @@ -206,10 +223,24 @@ members: member YYABORT; } } - | ERROR + | STRINGERROR { // Message d'erreur - jsonerror("Invalid json key"); + jsonerror("Invalid json key \"" + jsonPrintableValue(*$1) + "\" with non-utf8 encoding"); + + // Nettoyage et sortie + delete $1; + $$ = NULL; + YYABORT; + } + | ERROR + { + ALString sValueError; + + // Message d'erreur, avec indication de la valeur sauf si elle se reduit au caractete '"' + if (*$1 != "\"") + sValueError = " \"" + jsonPrintableValue(*$1) + "\""; + jsonerror("Invalid json key" + sValueError); // Nettoyage et sortie delete $1; @@ -276,28 +307,27 @@ values: value /* default yyerror for YACC and LEX */ void jsonerror(char const *fmt) -{ - jsonerrorWithLineCorrection(fmt, 0); -} - -/* Variante avec une correction du numero de ligne */ -void jsonerrorWithLineCorrection(char const *fmt, int nDeltaLineNumber) { char sErrorLine[20]; ALString sLabel; - int nLineNumber; nJsonFileParsingErrorNumber++; - nLineNumber = yylineno+nDeltaLineNumber; - if (nLineNumber <= 0) - nLineNumber = 1; - snprintf(sErrorLine, sizeof(sErrorLine), "Line %d", nLineNumber); + snprintf(sErrorLine, sizeof(sErrorLine), "Line %d", yylineno); sLabel = fmt; Global::AddError("Read json file", sErrorLine, sLabel); } +const ALString jsonPrintableValue(const ALString& sValue) +{ + const int nMaxPrintableLength = 30; + + if (sValue.GetLength() <= nMaxPrintableLength) + return sValue; + else + return sValue.Left(nMaxPrintableLength) + "..."; +} int jsonparse(); diff --git a/src/Norm/base/TextService.cpp b/src/Norm/base/TextService.cpp index d0eb3e00..cf5ac422 100644 --- a/src/Norm/base/TextService.cpp +++ b/src/Norm/base/TextService.cpp @@ -375,9 +375,32 @@ static int Utf32toUtf8String(unsigned int nCode, char* sUtf8Chars) } } -void TextService::JsonToCString(const char* sJsonString, ALString& sCString) +void TextService::SetForceUnicodeToAnsi(boolean bValue) { + bForceUnicodeToAnsi = bValue; +} + +boolean TextService::GetForceUnicodeToAnsi() +{ + return bForceUnicodeToAnsi; +} + +void TextService::SetForceUtf8ToAnsi(boolean bValue) +{ + bForceUtf8ToAnsi = bValue; +} + +boolean TextService::GetForceUtf8ToAnsi() +{ + return bForceUtf8ToAnsi; +} + +boolean TextService::JsonToCString(const char* sJsonString, ALString& sCString) +{ + boolean bOk = true; const unsigned char* sInputString; + int nAnsiCode; + boolean bContainsAnsiChars; unsigned int nCode; unsigned int nPotentialCode; int nBegin; @@ -387,9 +410,13 @@ void TextService::JsonToCString(const char* sJsonString, ALString& sCString) const char* sCharsToAdd; ALString sUnicodeChars; int nCharNumber; + ALString sCStringCopy; require(sJsonString != NULL); + // Initialisation de la chaine de caractere unicode a 4 caracteres + sUnicodeChars = " "; + // On passe par un format unsigned char pour le parsing sInputString = (const unsigned char*)sJsonString; nLength = (int)strlen(sJsonString); @@ -401,6 +428,7 @@ void TextService::JsonToCString(const char* sJsonString, ALString& sCString) nBegin = 0; nEnd = 0; sCharsToAdd = "?"; + bContainsAnsiChars = false; while (nEnd < nLength) { if (sInputString[nEnd] == '\\') @@ -439,29 +467,44 @@ void TextService::JsonToCString(const char* sJsonString, ALString& sCString) { nCode = 0; nEnd++; + // En principe, on doit avoir 4 caracteres suivant \u avec une chaine json correctement formee + // Sinon, on avance d'un caractere, avec une erreur + if (nEnd + 3 >= nLength) + { + bOk = false; + sCharsToAdd = "?"; + break; + } assert(nEnd < nLength); // Extraction des caracteres unicode - if (sUnicodeChars.GetLength() != 4) - sUnicodeChars.GetBufferSetLength(4); + assert(nEnd + 3 < nLength); + assert(sUnicodeChars.GetLength() == 4); sUnicodeChars.SetAt(0, sInputString[nEnd]); sUnicodeChars.SetAt(1, sInputString[nEnd + 1]); sUnicodeChars.SetAt(2, sInputString[nEnd + 2]); sUnicodeChars.SetAt(3, sInputString[nEnd + 3]); - // On tente d'abord le decodgage d'un caractere windows-1252 encode avec unicode - nCode = UnicodeHexToWindows1252(sUnicodeChars); - if (nCode != -1) + // On tente d'abord le decodgage d'un caractere windows-1252 encode avec unicode si on est en mode ForceAnsi + nAnsiCode = -1; + if (bForceUnicodeToAnsi) { - nEnd += 3; - nCharNumber = 1; - sUtf8Chars[0] = (char)nCode; - sUtf8Chars[1] = '\0'; - sCharsToAdd = sUtf8Chars; + nAnsiCode = UnicodeHexToWindows1252(sUnicodeChars); + if (nAnsiCode != -1) + { + bContainsAnsiChars = true; + nEnd += 3; + nCharNumber = 1; + sUtf8Chars[0] = (char)nAnsiCode; + sUtf8Chars[1] = '\0'; + sCharsToAdd = sUtf8Chars; + } } + // Cas general sinon - else + if (not bForceUnicodeToAnsi or nAnsiCode == -1) { + // Calcul du code du caractere unicode HexStringToCode(sInputString + nEnd, &nCode); nEnd += 3; assert(nEnd < nLength); @@ -483,6 +526,7 @@ void TextService::JsonToCString(const char* sJsonString, ALString& sCString) } else { + bOk = false; sCharsToAdd = "?"; break; } @@ -505,7 +549,11 @@ void TextService::JsonToCString(const char* sJsonString, ALString& sCString) } default: // En principe, impossible avec une chaine json correctement formee - assert(false); + // Dans ce cas, on avance d'un caractere, avec une erreur + bOk = false; + assert(nCharNumber == 1); + sCharsToAdd = "?"; + nEnd++; } if (nCharNumber == 1) sCString += sCharsToAdd[0]; @@ -520,6 +568,18 @@ void TextService::JsonToCString(const char* sJsonString, ALString& sCString) } } AppendSubString(sCString, sJsonString, nBegin, nEnd - nBegin); + + // Verification de l'encodage utf8, sauf si on a incorpore des caracteres ansi en mode ForceAnsi + if (bOk and not bContainsAnsiChars) + bOk = GetValidUTF8SubStringLength(sCString) == sCString.GetLength(); + + // Encode ansi des caracteres Utf8 si demande + if (bOk and bForceUtf8ToAnsi) + { + sCStringCopy = sCString; + TextService::CToCAnsiString(sCStringCopy, sCString); + } + return bOk; } void TextService::CToJsonString(const ALString& sCString, ALString& sJsonString) @@ -748,7 +808,7 @@ int TextService::UnicodeHexToWindows1252(const ALString& sUnicodeHexChars) return nCode; } -int TextService::Base64StringToBytes(const ALString& sBase64String, char* sBytes) +boolean TextService::Base64StringToBytes(const ALString& sBase64String, char* sBytes) { // Index des caracteres de l'alphabet base64, et -1 pour les caracteres n'en faisant pas partie static const unsigned char cBase64CharIndexes[] = { @@ -873,10 +933,12 @@ int TextService::Base64StringToBytes(const ALString& sBase64String, char* sBytes iOutput + (nTrailingEqualNumber > 0 ? 3 - nTrailingEqualNumber : 0) == nOutputByteNumber); } - // On renvoie -1 si erreur + // On renvoie une chaine termine par '\0' dans tous les cas if (not bOk) - nOutputByteNumber = -1; - return nOutputByteNumber; + sBytes[0] = '\0'; + else + sBytes[nOutputByteNumber] = '\0'; + return bOk; } void TextService::BytesToBase64String(const char* sBytes, int nByteNumber, ALString& sBase64String) @@ -1018,6 +1080,26 @@ int TextService::GetValidUTF8CharLengthAt(const ALString& sValue, int nStart) return nUtf8CharLength; } +int TextService::GetValidUTF8SubStringLength(const ALString& sValue) +{ + int nLength; + int nUTF8CharLength; + + // Parcours de la chaine jusqu'au premiere catactere non UTF8 + nLength = 0; + while (nLength < sValue.GetLength()) + { + nUTF8CharLength = GetValidUTF8CharLengthAt(sValue, nLength); + if (nUTF8CharLength > 0) + nLength += nUTF8CharLength; + else + break; + } + assert(nLength <= sValue.GetLength()); + assert(nLength == sValue.GetLength() or GetValidUTF8CharLengthAt(sValue, nLength) == 0); + return nLength; +} + void TextService::BuildTextSample(StringVector* svTextValues) { CharVector cvAscii; @@ -1142,18 +1224,16 @@ void TextService::Test() // Encodage/decodage de la valeur au format base64 sValue = svTextValues.GetAt(i); BytesToBase64String(sValue, sValue.GetLength(), sBase64String); - nLength = Base64StringToBytes(sBase64String, sBuffer); - assert(nLength >= 0 and nLength < BUFFER_LENGTH - 1); - sBuffer[nLength] = '\0'; + Base64StringToBytes(sBase64String, sBuffer); sRetrievedValue = sBuffer; assert(sRetrievedValue == sValue); // Verification de la non possibilite de decode une chaine avec encodage invalide // en ajoutant un a quatre caracteres invalides - assert(Base64StringToBytes(sValue + " ", sBuffer) == -1); - assert(Base64StringToBytes(sValue + " ", sBuffer) == -1); - assert(Base64StringToBytes(sValue + " ", sBuffer) == -1); - assert(Base64StringToBytes(sValue + " ", sBuffer) == -1); + assert(not Base64StringToBytes(sValue + " ", sBuffer)); + assert(not Base64StringToBytes(sValue + " ", sBuffer)); + assert(not Base64StringToBytes(sValue + " ", sBuffer)); + assert(not Base64StringToBytes(sValue + " ", sBuffer)); // Affichage cout << i << "\t"; @@ -1502,3 +1582,5 @@ StringVector TextService::svWindows1252UnicodeHexEncoding; StringVector TextService::svWindows1252Utf8HexEncoding; IntVectorSorter TextService::ivsWindows1252ControlCharUtf8CodeSorter; TextService TextService::textServiceGlobalInitializer; +boolean TextService::bForceUnicodeToAnsi = false; +boolean TextService::bForceUtf8ToAnsi = false; diff --git a/src/Norm/base/TextService.h b/src/Norm/base/TextService.h index 0853f85f..1c607d5c 100644 --- a/src/Norm/base/TextService.h +++ b/src/Norm/base/TextService.h @@ -76,11 +76,37 @@ class TextService : public Object /////////////////////////////////////////////////////////////////////////////// // Gestion de l'encodage des chaines ansi en utf8, le format supporte par json - // Les caracteres ansi 128 a 255 sont encodes avec iso-8859-1/windows-1252 - // L'encodage dans le json se fait avec le caractere d'echapement \uHHHH + // + // Cas avance, celui de l'encodage utuilise par Khiops pour gerer les caracteres ansi: + // - les caracteres ansi 128 a 255 sont encodes avec iso-8859-1/windows-1252 + // - l'encodage dans le json se fait avec le caractere d'echapement pour + // un encodage unicode \uHHHH, ce qui permet de le distinguer des memes + // caracteres encodes en Utf8 + // Exemple: cas du carectere e accent aigu + // - encodage ansi: E9 + // - encodage Utf8: C3A9 + // - encodage Unicode: U+00E0 + // Le parser de json utilise JsonToCString pour transformer les chaines de caracteres json + // en chaine de caracteres C, par defaut avec un encodage Utf8 + // Avec le mode ForceUnicodeToAnsi, on peut retrouver l'encodsage ansi dans le cas unicode + // Avec le mode ForceUtf8ToAnsi, on peut retrouver l'encodage ansi dans le cas Utf8 + + // En mode ForceUnicodeToAnsi (defaut: false), les caracteres de la plage de windows1252/iso8859-1 encodes + // via la syntaxe \uxxxx sont transformes en caracteres de l'ansi etendu, non Utf8 + static void SetForceUnicodeToAnsi(boolean bValue); + static boolean GetForceUnicodeToAnsi(); + + // En mode ForceUtf8ToAnsi (defaut: false), les caracteres de la plage de windows1252/iso8859-1 encodes + // via des caracteres Utf8 sont transformes en caracteres de l'ansi etendu, non Utf8 + static void SetForceUtf8ToAnsi(boolean bValue); + static boolean GetForceUtf8ToAnsi(); // Conversion d'une chaine Json valide vers une chaine C - static void JsonToCString(const char* sJsonString, ALString& sCString); + // On renvoie false en cas d'erreur d'encodage de la chaine a convertir, + // tout en encodant au mieux avec des caractere speciaux ou les caracteres tels quels + // En mode ForceUnicodeAnsi ou ForceUtf8ToAnsi, on renvoie true si la chaine Json est valide, + // bien que la chaine C ne soit pas avec un encodage UTF8 + static boolean JsonToCString(const char* sJsonString, ALString& sCString); // Encodage d'un chaine de caracteres C au format json, sans les double-quotes de debut et fin static void CToJsonString(const ALString& sCString, ALString& sJsonString); @@ -102,8 +128,8 @@ class TextService : public Object // Conversion d'une chaine base64 vers une chaine C // Le tableau de byte en sortie doit etre de taille au moins 3 * inputLength/4, // conformement aux besoins de l'encodage base64 - // On renvoie la longueur de la chaine encode en cas de succes, -1 sinon - static int Base64StringToBytes(const ALString& sBase64String, char* sBytes); + // On renvoie false en cas d'erreur d'encodage + static boolean Base64StringToBytes(const ALString& sBase64String, char* sBytes); // Encodage d'un tableau de bytes vers le format base64, sans les double-quotes de debut et fin // Le teableau en entree peut contenir n'importe quel byte, y comrpis des '\0' @@ -120,6 +146,9 @@ class TextService : public Object // Retourne 1 a 4 dans le cas d'un caractere valide, 0 sinon pour un caractere ANSI non encodable directement static int GetValidUTF8CharLengthAt(const ALString& sValue, int nStart); + // Longueur en bytes de la sous-partie d'une chaine encodee avec des caracteres UTF8 valide + static int GetValidUTF8SubStringLength(const ALString& sValue); + // Construction d'un echantillon de textes basiques pour des tests static void BuildTextSample(StringVector* svTextValues); @@ -192,6 +221,10 @@ class TextService : public Object // Ne pas declarer d'autres instances statiques de JSONFile, par exemple via d'autre classes, sinon cela pose // des problemes de memoire non liberee non diagnostiques par les outils de getsion de la memoire static TextService textServiceGlobalInitializer; + + // Parametrage de la conversion dees chaines json vers des chaines C + static boolean bForceUnicodeToAnsi; + static boolean bForceUtf8ToAnsi; }; //////////////////////////////////// diff --git a/src/Norm/base/UIObject.cpp b/src/Norm/base/UIObject.cpp index 19f24415..8bf69196 100644 --- a/src/Norm/base/UIObject.cpp +++ b/src/Norm/base/UIObject.cpp @@ -1801,20 +1801,27 @@ boolean UIObject::CheckCommandLineOptions(const ObjectArray& oaOptions) bOk = false; } - // Les option -j et -r sont exclusives + // Les options -j et -r sont exclusives if (odUsedOptions.Lookup("j") != NULL and odUsedOptions.Lookup("r") != NULL) { Global::AddError("Command line parameters", "", "-j and -r flags are exclusive"); bOk = false; } - // Les option -o et -O sont exclusives + // Les options -o et -O sont exclusives if (odUsedOptions.Lookup("o") != NULL and odUsedOptions.Lookup("O") != NULL) { Global::AddError("Command line parameters", "", "-o and -O flags are exclusive"); bOk = false; } + // L'option O n'est pas utilisable sans fichier de commande entree + if (odUsedOptions.Lookup("O") != NULL and odUsedOptions.Lookup("i") == NULL) + { + Global::AddError("Command line parameters", "", "-O flag must be used with -i"); + bOk = false; + } + // Retour au mode initial if (nCurrentUIMode == Graphic) SetUIMode(Graphic); @@ -1822,6 +1829,11 @@ boolean UIObject::CheckCommandLineOptions(const ObjectArray& oaOptions) return bOk; } +void UIObject::ExitHandlerCloseCommandFiles(int nExitCode) +{ + CleanCommandLineManagement(); +} + void UIObject::ParseMainParameters(int argc, char** argv) { static boolean bIsUserExitHandlerSet = false; @@ -1838,6 +1850,15 @@ void UIObject::ParseMainParameters(int argc, char** argv) ALString sToolName; ALString sGlobalHelp; + // Memorisation d'un handler de fermeture des fichiers en cas d'erreur fatale + // C'est indispensable pour ne pas faire planter le programme en release + // au moment de l'emission d'une erreur fatale + if (not bIsUserExitHandlerSet) + { + AddUserExitHandler(ExitHandlerCloseCommandFiles); + bIsUserExitHandlerSet = true; + } + // Aide globale sToolName = commandLineOptions.GetCommandName(); if (sToolName == "") @@ -1906,10 +1927,10 @@ void UIObject::ParseMainParameters(int argc, char** argv) oOutputScenarioNoReplay->AddDescriptionLine("same as -o option, but without replay"); oOutputScenarioNoReplay->SetParameterRequired(true); oOutputScenarioNoReplay->SetParameterDescription(CommandLineOption::sParameterFile); - oOutputScenarioNoReplay->SetMethod(OutputCommand); + oOutputScenarioNoReplay->SetMethod(OutputCommandNoReplay); commandLineOptions.AddOption(oOutputScenarioNoReplay); - // Option sur le parametrage des serach/replace + // Option sur le parametrage des search/replace oReplace = new CommandLineOption; oReplace->SetFlag('r'); oReplace->AddDescriptionLine("search and replace in the command file"); @@ -1934,17 +1955,11 @@ void UIObject::ParseMainParameters(int argc, char** argv) // Nettoyage initial CleanCommandLineManagement(); - // Nettoyage et notament fermeture des fichiers en cas d'erreur fatale - if (not bIsUserExitHandlerSet) - { - AddUserExitHandler(ExitHandlerCleanCommandLineManagement); - bIsUserExitHandlerSet = true; - } - // Parsing de la ligne de commande existante // pour detecter les options bVerboseCommandReplay = false; bBatchMode = false; + bOutputCommandNoReplay = false; // Analyse des option de la ligne de commande commandLineOptions.ParseMainParameters(argc, argv); @@ -1952,45 +1967,47 @@ void UIObject::ParseMainParameters(int argc, char** argv) // Initialisation des managers de message InitializeMessageManagers(); - // Ouverture du fichier de commande en entree - if (bOk and commandFile.GetInputCommandFileName() != "") - bOk = commandFile.OpenInputCommandFile(); - - // Ouverture du fichier de commande en sortie - if (bOk and commandFile.GetOutputCommandFileName() != "") + // Cas on on memorise les commandes dans le fichier en sortie sans les rejouer + if (bOutputCommandNoReplay) { - bOk = commandFile.OpenOutputCommandFile(); + commandFile.ReadWriteCommandFiles(); + CleanCommandLineManagement(); - // En mode Graphic on renvoie systematiquement true : on permet de lancer l'outil meme si on ne peut - // pas ecrire dans le fichier de log (les options -e et -o sont passees par defaut a l'outil dans les scripts de - // lancement bash ou cmd) En mode Textual on est plus strict (utilisation via python, java ou sur cluster) - if (not IsBatchMode()) - bOk = true; + // On sort du programme car a a fini tous les traitements, comme pour le -h + GlobalExitOnSuccess(); } + // Cas on on rejoue les commandes + else + { + // Ouverture du fichier de commande en entree + if (bOk and commandFile.GetInputCommandFileName() != "") + bOk = commandFile.OpenInputCommandFile(); - // Si il y a eu un pbm lors de l'excution d'une methode, on sort en erreur - if (not bOk) - GlobalExit(); + // Ouverture du fichier de commande en sortie + if (bOk and commandFile.GetOutputCommandFileName() != "") + { + bOk = commandFile.OpenOutputCommandFile(); - // Ecriture d'un en-tete dans le fichier des commandes - WriteOutputCommand("", "", CurrentTimestamp()); - WriteOutputCommand("", "", commandLineOptions.GetCommandName()); - if (commandFile.GetPrintOutputInConsole()) - { - sMessage = "Output command file\n"; - sMessage += "//\n"; - sMessage += "//This file contains recorded commands, that can be replayed.\n"; - sMessage += "//Commands are based on user interactions:\n"; - sMessage += "//\tfield update\n"; - sMessage += "//\tlist item selection\n"; - sMessage += "//\tmenu action\n"; - sMessage += "//Every command can be commented, using //.\n"; - sMessage += "//For example, commenting the last Exit command will allow other\n"; - sMessage += "//user interactions, after the commands have been replayed.\n"; - sMessage += "//\n"; - sMessage += "//"; + // En mode Graphic on renvoie systematiquement true : on permet de lancer l'outil meme si on ne peut + // pas ecrire dans le fichier de log (les options -e et -o sont passees par defaut a l'outil dans les scripts de + // lancement bash ou cmd) En mode Textual on est plus strict (utilisation via python, java ou sur cluster) + if (not IsBatchMode()) + bOk = true; + } + + // Si il y a eu un pbm lors de l'excution d'une methode, on sort en erreur + if (not bOk) + { + commandFile.CloseCommandFiles(); + Global::AddFatalError("Command file", "", "Batch mode failure"); + } + + // Ecriture d'un en-tete dans le fichier des commandes + WriteOutputCommand("", "", CurrentTimestamp()); + WriteOutputCommand("", "", commandLineOptions.GetCommandName()); + if (not commandFile.GetPrintOutputInConsole()) + commandFile.WriteOutputCommandHeader(); } - WriteOutputCommand("", "", sMessage); // Choix de l'interface if (IsBatchMode()) @@ -2000,10 +2017,9 @@ void UIObject::ParseMainParameters(int argc, char** argv) bOk = SetUIMode(UIObject::Graphic); if (not bOk and not GetTextualInteractiveModeAllowed()) { - Global::AddError("", "", - "Unable to load GUI, use -b and -i flags to launch khiops in batch mode " - "(khiops -h for help)"); - GlobalExit(); + Global::AddFatalError("", "", + "Unable to load GUI, use -b and -i flags to launch khiops in batch mode " + "(khiops -h for help)"); } } } @@ -2045,6 +2061,13 @@ boolean UIObject::OutputCommand(const ALString& sFileName) return true; } +boolean UIObject::OutputCommandNoReplay(const ALString& sFileName) +{ + commandFile.SetOutputCommandFileName(sFileName); + bOutputCommandNoReplay = true; + return true; +} + boolean UIObject::JsonCommand(const ALString& sFileName) { commandFile.SetInputParameterFileName(sFileName); @@ -2111,7 +2134,7 @@ boolean UIObject::TaskProgressionCommand(const ALString& sTaskFile) void UIObject::CleanCommandLineManagement() { - // Par defaut, on rehoue les commande + // Par defaut, on rejoue les commande bNoReplayMode = false; // Fermeture des fichiers de commande en entree et sortie @@ -2131,11 +2154,6 @@ void UIObject::CleanCommandLineManagement() } } -void UIObject::ExitHandlerCleanCommandLineManagement(int nExitCode) -{ - CleanCommandLineManagement(); -} - boolean UIObject::ReadInputCommand(StringVector* svIdentifierPath, ALString& sValue) { return commandFile.ReadInputCommand(svIdentifierPath, sValue); @@ -2204,6 +2222,7 @@ int UIObject::nUIMode = Textual; int UIObject::nInstanceNumber = 0; boolean UIObject::bVerboseCommandReplay = false; boolean UIObject::bBatchMode = false; +boolean UIObject::bOutputCommandNoReplay = false; boolean UIObject::bFastExitMode = true; boolean UIObject::bTextualInteractiveModeAllowed = false; ObjectDictionary UIObject::odListIndexCommands; diff --git a/src/Norm/base/UIUnit.cpp b/src/Norm/base/UIUnit.cpp index 076a89c0..e8f748ba 100644 --- a/src/Norm/base/UIUnit.cpp +++ b/src/Norm/base/UIUnit.cpp @@ -75,7 +75,7 @@ void UIUnit::Open() // On envoie un log des commandes rejouees avec les erreurs eventuelles // La politique est tres tolerante aux erreurs: on se contente de les signaler bActionExit = false; - while (bActionExit != true and ReadInputCommand(&svIdentifierPath, sValue) == true) + while (not bActionExit and ReadInputCommand(&svIdentifierPath, sValue) == true) { // Recherche de l'unite d'interface concernee assert(svIdentifierPath.GetSize() > 0); @@ -99,13 +99,16 @@ void UIUnit::Open() if (uiAction->GetIdentifier() == "Exit" or uiAction->GetParameters() == "Exit") bActionExit = true; if (bVerboseCommandReplay) - AddSimpleMessage("Replay user action\t: " + sCommand); + AddSimpleMessage("Replay user action: " + sCommand); WriteOutputCommand(sCommand, sValue, uiAction->GetUnformattedLabel()); uiUnit->ExecuteUserActionAt(uiAction->GetIdentifier()); // Sortie en mode fast exit if (bBatchMode and bFastExitMode and Global::IsAtLeastOneError()) + { + commandFile.CloseCommandFiles(); GlobalExitOnSuccess(); + } break; } @@ -125,7 +128,7 @@ void UIUnit::Open() uiField = cast(UIElement*, uiData); values = uiUnit->oaUnitValues.GetAt(nIndex); if (bVerboseCommandReplay) - AddSimpleMessage("Replay field update\t: " + sCommand + " " + + AddSimpleMessage("Replay field update: " + sCommand + " " + sValue); WriteOutputCommand(sCommand, sValue, uiField->GetUnformattedLabel()); uiField->SetFieldStringValue(values, uiUnit->nCurrentItemIndex, sValue); @@ -139,7 +142,10 @@ void UIUnit::Open() // Sortie en mode fast exit, pouvant etre declenchee par une erreur de refresh if (bBatchMode and bFastExitMode and Global::IsAtLeastOneError()) + { + commandFile.CloseCommandFiles(); GlobalExitOnSuccess(); + } } break; } @@ -147,11 +153,10 @@ void UIUnit::Open() { assert(uiUnit->GetDataType() == List); // Arret sinon - Global::AddError("Command file", "", - "Replay failure\t: " + sCommand + " " + sValue + - " at index " + - IntToString(uiUnit->nCurrentItemIndex)); - CleanCommandLineManagement(); + commandFile.AddInputCommandFileError( + "Incorrect index in command: " + sCommand + " " + sValue + + " at index " + IntToString(uiUnit->nCurrentItemIndex)); + commandFile.CloseCommandFiles(); if (bBatchMode) Global::AddFatalError("Command file", "", "Batch mode failure"); break; @@ -188,8 +193,8 @@ void UIUnit::Open() // Changement d'index si OK if (bVerboseCommandReplay) AddSimpleMessage( - "Replay list item selection\t: " + - sCommand + " " + sValue); + "Replay list item selection: " + sCommand + + " " + sValue); WriteOutputCommand(sCommand, sValue, "List item selection"); uiList->SetSelectedItemIndex(nListIndex); @@ -205,8 +210,8 @@ void UIUnit::Open() // Changement d'index if (bVerboseCommandReplay) AddSimpleMessage( - "Replay list item selection\t: " + - sCommand + " " + sValue); + "Replay list item selection: " + sCommand + + " " + sValue); WriteOutputCommand(sCommand, sValue, "List item selection"); uiList->SetSelectedItemKey(sValue); @@ -220,8 +225,8 @@ void UIUnit::Open() } // Arret sinon - Global::AddError("Command file", "", "Replay failure\t: " + sCommand + " " + sValue); - CleanCommandLineManagement(); + commandFile.AddInputCommandFileError("Incorrect command: " + sCommand + " " + sValue); + commandFile.CloseCommandFiles(); if (bBatchMode) Global::AddFatalError("Command file", "", "Batch mode failure"); break; @@ -234,7 +239,7 @@ void UIUnit::Open() svIdentifierPath.SetSize(0); // Ouverture de la fenetre, selon le mode (inutile si action Exit rejouee) - if (bActionExit == false) + if (not bActionExit) { // Mode graphique if (GetUIMode() == Graphic) @@ -272,10 +277,13 @@ void UIUnit::Open() else // Sinon, en mode textuel, on n'autorise pas d'interactions sans scenario { - if (commandFile.IsInputCommandFileOpened()) - AddFatalError("Unexpected end of file in the input commands file"); - else - AddFatalError("Missing input commands file"); + // Erreur fatale + // Remarque: dans certains cas, l'erreur fatale pourra etre emise + // directement lors de la fermeture du fichier de commande en entree + commandFile.CloseCommandFiles(); + commandFile.AddInputCommandFileError( + "Analysis of input commands interrupted because of errors"); + Global::AddFatalError("Command file", "", "Batch mode failure"); } } } diff --git a/src/Norm/base/UserInterface.h b/src/Norm/base/UserInterface.h index 7736985c..c420e992 100644 --- a/src/Norm/base/UserInterface.h +++ b/src/Norm/base/UserInterface.h @@ -396,8 +396,8 @@ class UIObject : public Object // Nettoyage de la gestion de la ligne de commande, fermeture des fichiers ouverts... static void CleanCommandLineManagement(); - // Fonction de sortie utilisateur pour fermer les fichiers d'entre-sortie de commande et d'erreur - static void ExitHandlerCleanCommandLineManagement(int nExitCode); + // Fonction de sortie utilisateur pour fermer les fichiers d'entre-sortie de commande + static void ExitHandlerCloseCommandFiles(int nExitCode); // Lecture d'une commande // renvoie false si pas de commande, sinon un vecteur de chaines de caracteres representant le @@ -431,6 +431,7 @@ class UIObject : public Object static int nInstanceNumber; static boolean bVerboseCommandReplay; static boolean bBatchMode; + static boolean bOutputCommandNoReplay; static boolean bFastExitMode; static boolean bTextualInteractiveModeAllowed; static ALString sIconImageJarPath; diff --git a/test/LearningTest/TestKhiops/Standard/JsonSpliceJunction/results.ref/C100_AnalysisResults.khj b/test/LearningTest/TestKhiops/Standard/JsonSpliceJunction/results.ref/C100_AnalysisResults.khj new file mode 100644 index 00000000..cf906df5 --- /dev/null +++ b/test/LearningTest/TestKhiops/Standard/JsonSpliceJunction/results.ref/C100_AnalysisResults.khj @@ -0,0 +1,645 @@ +{ + "tool": "Khiops", + "version": "10.5.5-b.0", + "shortDescription": "", + "modelingReport": { + "reportType": "Modeling", + "summary": { + "dictionary": "SpliceJunction", + "database": "..\/..\/..\/MTdatasets\/SpliceJunction\/SpliceJunction.txt", + "samplePercentage": 10, + "samplingMode": "Include sample", + "selectionVariable": "", + "selectionValue": "", + "learningTask": "Classification analysis", + "targetVariable": "Class" + }, + "trainedPredictors": [ + { + "rank": "R1", + "type": "Classifier", + "family": "Selective Naive Bayes", + "name": "Selective Naive Bayes", + "variables": 7 + } + ], + "trainedPredictorsDetails": { + "R1": { + "selectedVariables": [ + { + "preparedName": "PMode(DNA.Char) where Pos <= 30.5", + "name": "Mode(DNA.Char) where Pos <= 30.5", + "level": 0.0744365, + "weight": 0.363281, + "importance": 0.164443 + }, + { + "preparedName": "PMode(DNA.Char) where Pos in ]15.5, 30.5]", + "name": "Mode(DNA.Char) where Pos in ]15.5, 30.5]", + "level": 0.0508826, + "weight": 0.464844, + "importance": 0.153794 + }, + { + "preparedName": "PMode(DNA.Char)", + "name": "Mode(DNA.Char)", + "level": 0.0461059, + "weight": 0.472656, + "importance": 0.147622 + }, + { + "preparedName": "PMedian(DNA.Pos) where Char = G", + "name": "Median(DNA.Pos) where Char = G", + "level": 0.0225808, + "weight": 0.433594, + "importance": 0.0989488 + }, + { + "preparedName": "PMean(DNA.Pos) where Char = G", + "name": "Mean(DNA.Pos) where Char = G", + "level": 0.0274408, + "weight": 0.285156, + "importance": 0.0884586 + }, + { + "preparedName": "PCount(DNA) where Char = G", + "name": "Count(DNA) where Char = G", + "level": 0.0160697, + "weight": 0.332031, + "importance": 0.0730455 + }, + { + "preparedName": "PMedian(DNA.Pos) where Char = C", + "name": "Median(DNA.Pos) where Char = C", + "level": 0.0101423, + "weight": 0.300781, + "importance": 0.0552325 + } + ] + } + } + }, + "trainEvaluationReport": { + "reportType": "Evaluation", + "evaluationType": "Train", + "summary": { + "dictionary": "SpliceJunction", + "database": "..\/..\/..\/MTdatasets\/SpliceJunction\/SpliceJunction.txt", + "samplePercentage": 10, + "samplingMode": "Include sample", + "selectionVariable": "", + "selectionValue": "", + "instances": 303, + "learningTask": "Classification analysis", + "targetVariable": "Class" + }, + "predictorsPerformance": [ + { + "rank": "R1", + "type": "Classifier", + "family": "Selective Naive Bayes", + "name": "Selective Naive Bayes", + "accuracy": 0.636964, + "compression": 0.266223, + "auc": 0.787892 + } + ], + "predictorsDetailedPerformance": { + "R1": { + "confusionMatrix": { + "values": ["EI","IE","N"], + "matrix": [ + [8,1,4], + [5,42,20], + [54,26,143] + ] + } + } + }, + "liftCurves": [ + { + "targetValue": "EI", + "curves": [ + { + "classifier": "Optimal", + "values": [0,0.452239,0.904478,1.35672,1.80896,2.26119,2.71343,3.16567,3.61791,4.07015,4.52239,4.97463,5.42687,5.8791,6.33134,6.78358,7.23582,7.68806,8.1403,8.59254,9.04478,9.49701,9.94925,10.4015,10.8537,11.306,11.7582,12.2104,12.6627,13.1149,13.5672,14.0194,14.4716,14.9239,15.3761,15.8284,16.2806,16.7328,17.1851,17.6373,18.0896,18.5418,18.994,19.4463,19.8985,20.3507,20.803,21.2552,21.7075,22.1597,22.6119,23.0642,23.5164,23.9687,24.4209,24.8731,25.3254,25.7776,26.2299,26.6821,27.1343,27.5866,28.0388,28.491,28.9433,29.3955,29.8478,30.3,30.7522,31.2045,31.6567,32.109,32.5612,33.0134,33.4657,33.9179,34.3701,34.8224,35.2746,35.7269,36.1791,36.6313,37.0836,37.5358,37.9881,38.4403,38.8925,39.3448,39.797,40.2493,40.7015,41.1537,41.606,42.0582,42.5104,42.9627,43.4149,43.8672,44.3194,44.7716,45.2239,45.6761,46.1284,46.5806,47.0328,47.4851,47.9373,48.3896,48.8418,49.294,49.7463,50.1985,50.6507,51.103,51.5552,52.0075,52.4597,52.9119,53.3642,53.8164,54.2687,54.7209,55.1731,55.6254,56.0776,56.5299,56.9821,57.4343,57.8866,58.3388,58.791,59.2433,59.6955,60.1478,60.6,61.0522,61.5045,61.9567,62.409,62.8612,63.3134,63.7657,64.2179,64.6701,65.1224,65.5746,66.0269,66.4791,66.9313,67.3836,67.8358,68.2881,68.7403,69.1925,69.6448,70.097,70.5493,71.0015,71.4537,71.906,72.3582,72.8104,73.2627,73.7149,74.1672,74.6194,75.0716,75.5239,75.9761,76.4284,76.8806,77.3328,77.7851,78.2373,78.6896,79.1418,79.594,80.0463,80.4985,80.9507,81.403,81.8552,82.3075,82.7597,83.2119,83.6642,84.1164,84.5687,85.0209,85.4731,85.9254,86.3776,86.8299,87.2821,87.7343,88.1866,88.6388,89.091,89.5433,89.9955,90.4478,90.9,91.3522,91.8045,92.2567,92.709,93.1612,93.6134,94.0657,94.5179,94.9701,95.4224,95.8746,96.3269,96.7791,97.2313,97.6836,98.1358,98.5881,99.0403,99.4925,99.9448,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100] + }, + { + "classifier": "Selective Naive Bayes", + "values": [0,0.452239,0.904478,1.35672,1.80896,2.26119,2.71343,3.16567,3.61791,4.07015,4.49254,4.64328,4.79403,4.94478,5.09552,5.24627,5.39701,5.54776,5.69851,5.84925,6.01493,6.24104,6.46716,6.69328,6.9194,7.14552,7.37164,7.50771,7.58308,7.65846,7.73383,7.8092,7.88458,7.95995,8.03532,8.1107,8.18607,8.26144,8.33682,8.41219,8.48756,8.56294,8.63831,8.71368,8.78905,8.86443,8.9398,9.13507,9.36119,9.58731,9.81343,10.0396,10.2657,10.5006,10.7719,11.0433,11.3146,11.586,11.8573,12.1287,12.4,12.6713,12.9427,13.214,13.4854,13.7567,14.0281,14.2994,14.5707,14.8421,15.1343,15.4358,15.7373,16.0388,16.3403,16.6418,16.9433,17.2448,17.5463,17.8478,18.2687,18.7209,19.1731,19.6254,20.0776,20.5299,20.9388,21.1649,21.391,21.6172,21.8433,22.0694,22.2955,22.5216,22.7478,22.9739,23.2,23.4261,23.6522,23.8784,23.8806,23.8806,23.8806,24.0106,24.199,24.3874,24.5759,24.7643,24.9527,25.1412,25.3296,25.518,25.7065,25.8949,26.0833,26.2718,26.4602,26.6486,26.8371,27.0255,27.2139,27.4024,27.5908,27.7792,27.9677,28.1561,28.3445,28.533,28.7214,28.9098,29.0983,29.2867,29.4751,29.6636,29.852,30.0404,30.2289,30.4173,30.6057,30.7942,30.9826,31.171,31.3627,31.5888,31.8149,32.041,32.2672,32.4933,32.7194,32.8358,32.8358,32.8358,32.9194,33.3716,33.8239,34.2761,34.595,34.8965,35.198,35.4995,35.801,36.1025,36.404,36.7055,37.007,37.3085,37.3134,37.3134,37.3134,37.3134,37.3134,37.3134,37.3134,37.3134,37.3134,37.3134,37.3134,37.3134,37.3134,37.6672,38.1194,38.5716,39.0239,39.4761,39.9284,40.2985,40.2985,40.2985,40.2985,40.2985,40.2985,40.2985,40.2985,40.2985,40.2985,40.3461,40.5157,40.6853,40.8549,41.0244,41.194,41.3636,41.5332,41.7028,41.8724,42.042,42.2116,42.3812,42.5507,42.7203,42.8899,43.0595,43.2291,43.3987,43.5683,43.7379,43.9075,44.0771,44.2466,44.4162,44.5858,44.7554,45.1731,45.6254,46.0776,46.2687,46.2687,46.2687,46.3313,46.5575,46.7836,47.0097,47.2358,47.4619,47.6881,47.7612,47.7612,47.7612,47.7612,47.7612,47.7612,47.7759,47.9697,48.1635,48.3574,48.5512,48.745,48.9388,49.1326,49.3264,49.5203,49.7141,49.9079,50.1017,50.2955,50.4893,50.6832,50.877,51.0708,51.2646,51.4584,51.6522,51.8461,52.0399,52.2337,52.6791,53.1313,53.5836,53.7313,53.7313,53.7313,53.7313,53.7313,53.7313,53.7313,53.7313,53.7313,53.7313,53.7313,53.7313,53.7313,53.7313,53.7313,53.7313,53.7313,53.7313,53.7313,53.7313,53.7313,53.7313,53.7313,53.7313,53.7313,53.7313,53.8129,54.1144,54.4159,54.7174,55.0189,55.3204,55.6219,55.9234,56.2249,56.5264,56.8,57.0261,57.2522,57.4784,57.7045,57.9306,58.1567,58.5567,59.009,59.4612,59.7015,59.7015,59.7015,59.7015,59.7015,59.7015,59.7015,59.8978,60.1239,60.35,60.5761,60.8022,61.0284,61.2545,61.4806,61.7067,61.9328,62.159,62.3851,62.6112,62.9881,63.4403,63.8925,64.1791,64.1791,64.1791,64.194,64.4201,64.6463,64.8724,65.0985,65.3246,65.5507,65.8821,66.3343,66.7866,67.1642,67.1642,67.1642,67.1642,67.1642,67.1642,67.1642,67.1925,67.2428,67.293,67.3433,67.3935,67.4438,67.494,67.5443,67.5945,67.6448,67.695,67.7453,67.7955,67.8458,67.896,67.9463,67.9965,68.0468,68.097,68.1473,68.1975,68.2478,68.298,68.3483,68.3985,68.4488,68.499,68.5493,68.5995,68.6498,68.6567,68.6567,68.6567,68.6567,68.6567,68.6567,68.6567,68.6567,68.6567,68.6567,68.6567,68.6567,68.6567,68.6567,68.6567,68.6567,68.6567,68.6567,68.6567,68.6567,68.6567,68.6567,68.6567,68.6567,68.6567,68.6567,68.6567,68.6567,68.6567,68.6567,68.6567,68.6567,68.6567,68.8507,69.0769,69.303,69.5291,69.7552,69.9813,70.1493,70.1493,70.1493,70.1493,70.1493,70.1493,70.1493,70.1493,70.1493,70.1493,70.3104,70.7627,71.2149,71.6672,72.1194,72.5716,73.0239,73.4761,73.9284,74.3806,74.6857,74.8149,74.9441,75.0733,75.2026,75.3318,75.461,75.5902,75.7194,75.8486,75.9778,76.107,76.2362,76.3655,76.4947,76.6239,76.7531,76.8823,77.0115,77.1407,77.2699,77.3991,77.5284,77.6119,77.6119,77.6119,77.6119,77.6119,77.6119,77.6119,77.6119,77.6119,77.6119,77.6119,77.6119,77.6119,77.6119,77.6119,77.6119,77.6119,77.7403,77.891,78.0418,78.1925,78.3433,78.494,78.6448,78.7955,78.9463,79.097,79.2478,79.3985,79.5493,79.7,79.8507,80.0015,80.1522,80.303,80.4537,80.6045,80.7552,80.906,81.0567,81.2075,81.3582,81.509,81.6597,81.8104,81.9612,82.1567,82.609,83.0612,83.5134,83.6588,83.7493,83.8397,83.9301,84.0206,84.111,84.2015,84.2919,84.3824,84.4728,84.5633,84.6537,84.7442,84.8346,84.9251,85.0155,85.0746,85.0746,85.0746,85.0851,85.3112,85.5373,85.7634,85.9896,86.2157,86.4418,86.6679,86.894,87.1201,87.3463,87.5724,87.7985,88.0246,88.0597,88.0597,88.0597,88.0597,88.0597,88.0597,88.0597,88.0597,88.0597,88.0597,88.0597,88.0597,88.0597,88.0961,88.1526,88.2091,88.2657,88.3222,88.3787,88.4353,88.4918,88.5483,88.6049,88.6614,88.7179,88.7744,88.831,88.8875,88.944,89.0006,89.0571,89.1136,89.1701,89.2267,89.2832,89.3397,89.3963,89.4528,89.5093,89.5522,89.5522,89.5522,89.5522,89.5522,89.5522,89.5522,89.5522,89.5522,89.5522,89.5522,89.5522,89.5522,89.5542,89.6045,89.6547,89.705,89.7552,89.8055,89.8557,89.906,89.9562,90.0065,90.0567,90.107,90.1572,90.2075,90.2577,90.308,90.3582,90.4085,90.4587,90.509,90.5592,90.6095,90.6597,90.71,90.7602,90.8104,90.8607,90.9109,90.9612,91.0114,91.0448,91.0448,91.0448,91.0448,91.0448,91.0448,91.0448,91.0633,91.0884,91.1135,91.1386,91.1638,91.1889,91.214,91.2391,91.2643,91.2894,91.3145,91.3396,91.3648,91.3899,91.415,91.4401,91.4653,91.4904,91.5155,91.5406,91.5658,91.5909,91.616,91.6411,91.6663,91.6914,91.7165,91.7416,91.7667,91.7919,91.817,91.8421,91.8672,91.8924,91.9175,91.9426,91.9677,91.9929,92.018,92.0431,92.0682,92.0934,92.1185,92.1436,92.1687,92.1939,92.219,92.2441,92.2692,92.2944,92.3195,92.3446,92.3697,92.3949,92.42,92.4451,92.4702,92.4954,92.5205,92.5373,92.5373,92.5373,92.5373,92.5373,92.5373,92.5373,92.5373,92.5373,92.5373,92.5373,92.5373,92.5373,92.5373,92.5373,92.5373,92.5373,92.9119,93.3642,93.8164,94.2687,94.7209,95.1731,95.5224,95.5224,95.5224,95.5224,95.5224,95.5224,95.5224,95.5224,95.5224,95.5224,95.5224,95.5224,95.5224,95.5224,95.5224,95.5224,95.5224,95.5224,95.5224,95.5224,95.5455,95.5998,95.654,95.7083,95.7626,95.8168,95.8711,95.9254,95.9796,96.0339,96.0882,96.1424,96.1967,96.251,96.3053,96.3595,96.4138,96.4681,96.5223,96.5766,96.6309,96.6851,96.7394,96.7937,96.8479,96.9022,96.9565,97.0107,97.065,97.1193,97.1736,97.2278,97.2821,97.3364,97.3906,97.4449,97.4992,97.5534,97.6077,97.662,97.7162,97.7705,97.8248,97.879,97.9333,97.9876,98.0419,98.0961,98.1504,98.2047,98.2589,98.3132,98.3675,98.4217,98.476,98.5303,98.5845,98.6388,98.6931,98.7473,98.8016,98.8559,98.9101,98.9644,99.0187,99.073,99.1272,99.1815,99.2358,99.29,99.3443,99.3986,99.4528,99.5071,99.5614,99.6156,99.6699,99.7242,99.7784,99.8327,99.887,99.9413,99.9955,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100] + } + ] + }, + { + "targetValue": "IE", + "curves": [ + { + "classifier": "Optimal", + "values": [0,0.43913,0.878261,1.31739,1.75652,2.19565,2.63478,3.07391,3.51304,3.95217,4.3913,4.83043,5.26957,5.7087,6.14783,6.58696,7.02609,7.46522,7.90435,8.34348,8.78261,9.22174,9.66087,10.1,10.5391,10.9783,11.4174,11.8565,12.2957,12.7348,13.1739,13.613,14.0522,14.4913,14.9304,15.3696,15.8087,16.2478,16.687,17.1261,17.5652,18.0043,18.4435,18.8826,19.3217,19.7609,20.2,20.6391,21.0783,21.5174,21.9565,22.3957,22.8348,23.2739,23.713,24.1522,24.5913,25.0304,25.4696,25.9087,26.3478,26.787,27.2261,27.6652,28.1043,28.5435,28.9826,29.4217,29.8609,30.3,30.7391,31.1783,31.6174,32.0565,32.4957,32.9348,33.3739,33.813,34.2522,34.6913,35.1304,35.5696,36.0087,36.4478,36.887,37.3261,37.7652,38.2043,38.6435,39.0826,39.5217,39.9609,40.4,40.8391,41.2783,41.7174,42.1565,42.5957,43.0348,43.4739,43.913,44.3522,44.7913,45.2304,45.6696,46.1087,46.5478,46.987,47.4261,47.8652,48.3043,48.7435,49.1826,49.6217,50.0609,50.5,50.9391,51.3783,51.8174,52.2565,52.6957,53.1348,53.5739,54.013,54.4522,54.8913,55.3304,55.7696,56.2087,56.6478,57.087,57.5261,57.9652,58.4043,58.8435,59.2826,59.7217,60.1609,60.6,61.0391,61.4783,61.9174,62.3565,62.7957,63.2348,63.6739,64.113,64.5522,64.9913,65.4304,65.8696,66.3087,66.7478,67.187,67.6261,68.0652,68.5043,68.9435,69.3826,69.8217,70.2609,70.7,71.1391,71.5783,72.0174,72.4565,72.8957,73.3348,73.7739,74.213,74.6522,75.0913,75.5304,75.9696,76.4087,76.8478,77.287,77.7261,78.1652,78.6043,79.0435,79.4826,79.9217,80.3609,80.8,81.2391,81.6783,82.1174,82.5565,82.9957,83.4348,83.8739,84.313,84.7522,85.1913,85.6304,86.0696,86.5087,86.9478,87.387,87.8261,88.2652,88.7043,89.1435,89.5826,90.0217,90.4609,90.9,91.3391,91.7783,92.2174,92.6565,93.0957,93.5348,93.9739,94.413,94.8522,95.2913,95.7304,96.1696,96.6087,97.0478,97.487,97.9261,98.3652,98.8043,99.2435,99.6826,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100] + }, + { + "classifier": "Selective Naive Bayes", + "values": [0,0.351304,0.702609,1.05391,1.40522,1.75652,2.10783,2.45913,2.81043,3.16174,3.51304,3.86435,4.21565,4.56696,4.91826,5.26957,5.62087,5.93716,6.2182,6.49925,6.78029,7.06133,7.34238,7.62342,7.90446,8.18551,8.46655,8.74759,9.02864,9.30968,9.59072,9.87177,10.1528,10.4339,10.7149,10.9959,11.277,11.558,11.8391,12.1201,12.4012,12.6822,12.9632,13.2443,13.5253,13.8064,14.0874,14.3685,14.6495,14.9306,15.2116,15.4926,15.7737,16.0547,16.3358,16.6168,16.8979,17.1789,17.4599,17.741,18.022,18.3031,18.5841,18.8652,19.1462,19.4272,19.7083,19.9893,20.2704,20.5514,20.8325,21.1135,21.3946,21.6756,21.9566,22.2377,22.5187,22.7998,23.0808,23.3619,23.6429,23.9239,24.205,24.486,24.7671,25.0481,25.3292,25.6102,25.8912,26.1723,26.4533,26.7344,27.0154,27.2965,27.5775,27.8586,28.1396,28.4206,28.7017,28.9827,29.2754,29.5681,29.8609,30.1536,30.4464,30.7391,31.0319,31.3246,31.6174,31.9036,32.1232,32.3428,32.5623,32.7819,33.0014,33.221,33.5478,33.987,34.4261,34.8652,35.3043,35.7435,36.1826,36.6217,37.0609,37.5,37.9391,38.3783,38.8174,39.2145,39.5072,39.8,40.0928,40.3855,40.6783,40.971,41.2638,41.5565,41.8493,42.1562,42.4855,42.8149,43.1442,43.4736,43.8029,44.1322,44.4616,44.7909,45.1203,45.4496,45.779,46.1083,46.4219,46.6659,46.9098,47.1538,47.3977,47.6417,47.8857,48.1296,48.3736,48.6176,48.8615,49.1055,49.3494,49.5934,49.8374,50.0813,50.3253,50.5692,50.8132,51.0572,51.3011,51.5451,51.789,52.033,52.277,52.5209,52.7649,53.0089,53.2528,53.4968,53.7407,53.9847,54.2287,54.4726,54.7166,54.9605,55.2045,55.4485,55.6924,55.9364,56.1804,56.4243,56.6683,56.9122,57.1562,57.4002,57.6441,57.8881,58.132,58.376,58.62,58.8639,59.1079,59.3519,59.5958,59.8398,60.0837,60.3277,60.5717,60.8156,61.2116,61.6507,62.0899,62.4502,62.7246,62.9991,63.2736,63.548,63.8225,64.0969,64.3714,64.6458,64.9203,65.1947,65.4692,65.7437,66.0181,66.2926,66.567,66.8415,67.1159,67.3904,67.6649,67.9393,68.2138,68.4882,68.7627,69.0371,69.3116,69.5819,69.8014,70.021,70.2406,70.4601,70.6797,70.8993,71.0145,71.0145,71.0145,71.0145,71.0145,71.0145,71.0145,71.0145,71.0145,71.0145,71.1826,71.4754,71.7681,72.0609,72.3536,72.6464,72.9391,73.2319,73.5246,73.8174,74.1101,74.4029,74.6957,74.9884,75.2812,75.5739,75.8667,76.1594,76.4522,76.7449,77.1507,77.5899,78.029,78.3852,78.6487,78.9122,79.1757,79.4391,79.7026,79.9661,80.2296,80.493,80.7565,81.02,81.2835,81.547,81.8104,82.0739,82.3374,82.6009,82.7152,82.825,82.9348,83.0446,83.1543,83.2641,83.3739,83.4837,83.5935,83.7033,83.813,83.9228,84.0326,84.058,84.058,84.058,84.1266,84.2729,84.4193,84.5657,84.7121,84.8585,85.0048,85.1512,85.2976,85.444,85.5072,85.5072,85.5072,85.5333,85.6309,85.7285,85.8261,85.9237,86.0213,86.1188,86.2164,86.314,86.4116,86.5092,86.6068,86.7043,86.8019,86.8995,86.9971,87.0947,87.1923,87.2899,87.3874,87.485,87.5826,87.6802,87.7778,87.8754,87.9729,88.0705,88.1681,88.2657,88.3633,88.4058,88.4058,88.4058,88.4058,88.4058,88.4058,88.4058,88.4058,88.4058,88.4058,88.4058,88.4058,88.4058,88.4058,88.4058,88.4058,88.4058,88.4058,88.4058,88.4058,88.4058,88.4058,88.4058,88.5072,88.7268,88.9464,89.1659,89.3855,89.6051,89.8246,90.0442,90.2638,90.4833,90.7029,90.9225,91.142,91.3043,91.3043,91.3043,91.3043,91.3043,91.3043,91.3043,91.3043,91.3043,91.3043,91.3043,91.3043,91.3043,91.3043,91.3043,91.3043,91.3043,91.6377,92.0768,92.5159,92.7536,92.7536,92.7536,92.7614,92.8101,92.8589,92.9077,92.9565,93.0053,93.0541,93.1029,93.1517,93.2005,93.2493,93.2981,93.3469,93.3957,93.4444,93.4932,93.542,93.5908,93.6396,93.6884,93.7372,93.786,93.8348,93.8836,93.9324,93.9812,94.03,94.0787,94.1275,94.1763,94.2029,94.2029,94.2029,94.2029,94.2029,94.2029,94.2029,94.3906,94.6101,94.8297,95.0493,95.2688,95.4884,95.708,95.9275,96.1471,96.3667,96.5862,96.8058,97.0254,97.1014,97.1014,97.1014,97.1014,97.1014,97.1014,97.1014,97.1014,97.1014,97.1014,97.1014,97.1014,97.1014,97.1014,97.1014,97.1014,97.1014,97.1014,97.1014,97.1014,97.1014,97.1014,97.1014,97.1014,97.1014,97.1014,97.1014,97.1014,97.1014,97.1014,97.1014,97.1014,97.1014,97.1014,97.1014,97.1014,97.1014,97.1014,97.1014,97.1014,97.1014,97.1014,97.1014,97.1014,97.1014,97.1014,97.1014,97.1014,97.1014,97.1014,97.1014,97.1014,97.1014,97.1014,97.1014,97.1014,97.1816,97.328,97.4744,97.6208,97.7671,97.9135,98.0599,98.2063,98.3527,98.499,98.5507,98.5507,98.5507,98.5507,98.5507,98.5507,98.5507,98.5507,98.5507,98.5507,98.5507,98.5507,98.5507,98.5507,98.5507,98.5507,98.5507,98.5507,98.5507,98.5507,98.5507,98.5507,98.5507,98.5507,98.5507,98.5507,98.5507,98.5507,98.5507,98.5507,98.5507,98.5507,98.5507,98.5507,98.5507,98.5507,98.5884,98.6982,98.808,98.9178,99.0275,99.1373,99.2471,99.3569,99.4667,99.5764,99.6862,99.796,99.9058,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100] + } + ] + }, + { + "targetValue": "N", + "curves": [ + { + "classifier": "Optimal", + "values": [0,0.181437,0.362874,0.544311,0.725749,0.907186,1.08862,1.27006,1.4515,1.63293,1.81437,1.99581,2.17725,2.35868,2.54012,2.72156,2.90299,3.08443,3.26587,3.44731,3.62874,3.81018,3.99162,4.17305,4.35449,4.53593,4.71737,4.8988,5.08024,5.26168,5.44311,5.62455,5.80599,5.98743,6.16886,6.3503,6.53174,6.71317,6.89461,7.07605,7.25749,7.43892,7.62036,7.8018,7.98323,8.16467,8.34611,8.52754,8.70898,8.89042,9.07186,9.25329,9.43473,9.61617,9.7976,9.97904,10.1605,10.3419,10.5234,10.7048,10.8862,11.0677,11.2491,11.4305,11.612,11.7934,11.9749,12.1563,12.3377,12.5192,12.7006,12.882,13.0635,13.2449,13.4263,13.6078,13.7892,13.9707,14.1521,14.3335,14.515,14.6964,14.8778,15.0593,15.2407,15.4222,15.6036,15.785,15.9665,16.1479,16.3293,16.5108,16.6922,16.8737,17.0551,17.2365,17.418,17.5994,17.7808,17.9623,18.1437,18.3251,18.5066,18.688,18.8695,19.0509,19.2323,19.4138,19.5952,19.7766,19.9581,20.1395,20.321,20.5024,20.6838,20.8653,21.0467,21.2281,21.4096,21.591,21.7725,21.9539,22.1353,22.3168,22.4982,22.6796,22.8611,23.0425,23.224,23.4054,23.5868,23.7683,23.9497,24.1311,24.3126,24.494,24.6754,24.8569,25.0383,25.2198,25.4012,25.5826,25.7641,25.9455,26.1269,26.3084,26.4898,26.6713,26.8527,27.0341,27.2156,27.397,27.5784,27.7599,27.9413,28.1228,28.3042,28.4856,28.6671,28.8485,29.0299,29.2114,29.3928,29.5743,29.7557,29.9371,30.1186,30.3,30.4814,30.6629,30.8443,31.0257,31.2072,31.3886,31.5701,31.7515,31.9329,32.1144,32.2958,32.4772,32.6587,32.8401,33.0216,33.203,33.3844,33.5659,33.7473,33.9287,34.1102,34.2916,34.4731,34.6545,34.8359,35.0174,35.1988,35.3802,35.5617,35.7431,35.9246,36.106,36.2874,36.4689,36.6503,36.8317,37.0132,37.1946,37.376,37.5575,37.7389,37.9204,38.1018,38.2832,38.4647,38.6461,38.8275,39.009,39.1904,39.3719,39.5533,39.7347,39.9162,40.0976,40.279,40.4605,40.6419,40.8234,41.0048,41.1862,41.3677,41.5491,41.7305,41.912,42.0934,42.2749,42.4563,42.6377,42.8192,43.0006,43.182,43.3635,43.5449,43.7263,43.9078,44.0892,44.2707,44.4521,44.6335,44.815,44.9964,45.1778,45.3593,45.5407,45.7222,45.9036,46.085,46.2665,46.4479,46.6293,46.8108,46.9922,47.1737,47.3551,47.5365,47.718,47.8994,48.0808,48.2623,48.4437,48.6251,48.8066,48.988,49.1695,49.3509,49.5323,49.7138,49.8952,50.0766,50.2581,50.4395,50.621,50.8024,50.9838,51.1653,51.3467,51.5281,51.7096,51.891,52.0725,52.2539,52.4353,52.6168,52.7982,52.9796,53.1611,53.3425,53.524,53.7054,53.8868,54.0683,54.2497,54.4311,54.6126,54.794,54.9754,55.1569,55.3383,55.5198,55.7012,55.8826,56.0641,56.2455,56.4269,56.6084,56.7898,56.9713,57.1527,57.3341,57.5156,57.697,57.8784,58.0599,58.2413,58.4228,58.6042,58.7856,58.9671,59.1485,59.3299,59.5114,59.6928,59.8743,60.0557,60.2371,60.4186,60.6,60.7814,60.9629,61.1443,61.3257,61.5072,61.6886,61.8701,62.0515,62.2329,62.4144,62.5958,62.7772,62.9587,63.1401,63.3216,63.503,63.6844,63.8659,64.0473,64.2287,64.4102,64.5916,64.7731,64.9545,65.1359,65.3174,65.4988,65.6802,65.8617,66.0431,66.2246,66.406,66.5874,66.7689,66.9503,67.1317,67.3132,67.4946,67.676,67.8575,68.0389,68.2204,68.4018,68.5832,68.7647,68.9461,69.1275,69.309,69.4904,69.6719,69.8533,70.0347,70.2162,70.3976,70.579,70.7605,70.9419,71.1234,71.3048,71.4862,71.6677,71.8491,72.0305,72.212,72.3934,72.5749,72.7563,72.9377,73.1192,73.3006,73.482,73.6635,73.8449,74.0263,74.2078,74.3892,74.5707,74.7521,74.9335,75.115,75.2964,75.4778,75.6593,75.8407,76.0222,76.2036,76.385,76.5665,76.7479,76.9293,77.1108,77.2922,77.4737,77.6551,77.8365,78.018,78.1994,78.3808,78.5623,78.7437,78.9251,79.1066,79.288,79.4695,79.6509,79.8323,80.0138,80.1952,80.3766,80.5581,80.7395,80.921,81.1024,81.2838,81.4653,81.6467,81.8281,82.0096,82.191,82.3725,82.5539,82.7353,82.9168,83.0982,83.2796,83.4611,83.6425,83.824,84.0054,84.1868,84.3683,84.5497,84.7311,84.9126,85.094,85.2754,85.4569,85.6383,85.8198,86.0012,86.1826,86.3641,86.5455,86.7269,86.9084,87.0898,87.2713,87.4527,87.6341,87.8156,87.997,88.1784,88.3599,88.5413,88.7228,88.9042,89.0856,89.2671,89.4485,89.6299,89.8114,89.9928,90.1743,90.3557,90.5371,90.7186,90.9,91.0814,91.2629,91.4443,91.6257,91.8072,91.9886,92.1701,92.3515,92.5329,92.7144,92.8958,93.0772,93.2587,93.4401,93.6216,93.803,93.9844,94.1659,94.3473,94.5287,94.7102,94.8916,95.0731,95.2545,95.4359,95.6174,95.7988,95.9802,96.1617,96.3431,96.5246,96.706,96.8874,97.0689,97.2503,97.4317,97.6132,97.7946,97.976,98.1575,98.3389,98.5204,98.7018,98.8832,99.0647,99.2461,99.4275,99.609,99.7904,99.9719,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100] + }, + { + "classifier": "Selective Naive Bayes", + "values": [0,0.181437,0.362874,0.544311,0.725749,0.907186,1.08862,1.27006,1.4515,1.63293,1.81437,1.99581,2.17725,2.35868,2.54012,2.72156,2.90299,3.07313,3.23189,3.39064,3.5494,3.70816,3.86692,4.02567,4.18443,4.34319,4.50195,4.6607,4.81946,4.97822,5.13698,5.29573,5.45449,5.61325,5.77201,5.93076,6.08952,6.24828,6.40704,6.56579,6.72455,6.88331,7.04207,7.18563,7.18563,7.18563,7.18563,7.18563,7.18563,7.18563,7.27545,7.45689,7.63832,7.81976,8.0012,8.18263,8.36407,8.54551,8.72695,8.90838,9.08982,9.27126,9.45269,9.63413,9.81557,9.99701,10.1784,10.3599,10.5413,10.7228,10.9042,11.0856,11.2671,11.4485,11.6299,11.8114,11.9928,12.1743,12.3557,12.5371,12.6775,12.8071,12.9367,13.0663,13.1959,13.3255,13.4551,13.5847,13.7143,13.8439,13.9735,14.1031,14.2327,14.3623,14.4919,14.6215,14.7511,14.8807,15.0103,15.1399,15.2695,15.3991,15.5287,15.6627,15.7988,15.9349,16.071,16.207,16.3431,16.4792,16.6153,16.7513,16.8874,17.0235,17.1596,17.2957,17.4539,17.6353,17.8168,17.9811,18.0719,18.1626,18.2533,18.344,18.4347,18.5254,18.6695,18.8509,19.0323,19.208,19.3693,19.5305,19.6918,19.8531,20.0144,20.1756,20.3369,20.4982,20.6595,20.8208,20.982,21.1433,21.3046,21.4659,21.6271,21.7884,21.9497,22.111,22.2723,22.4335,22.5948,22.7561,22.9174,23.0786,23.2399,23.4012,23.5625,23.7238,23.885,24.0581,24.2395,24.421,24.6024,24.7838,24.9653,25.1467,25.3281,25.5096,25.691,25.8725,26.0539,26.2353,26.4168,26.5982,26.7796,26.9611,27.1425,27.324,27.5054,27.6868,27.8683,28.0497,28.2311,28.4126,28.594,28.7754,28.9569,29.1383,29.3198,29.3413,29.3413,29.3413,29.4467,29.6281,29.8096,29.9656,30.0563,30.147,30.2377,30.3284,30.4192,30.5099,30.6006,30.6913,30.782,30.8728,30.9635,31.0542,31.1521,31.3335,31.515,31.6964,31.8072,31.8979,31.9886,32.0793,32.1701,32.2608,32.3461,32.4066,32.4671,32.5275,32.588,32.6485,32.709,32.7695,32.8299,32.8904,32.9844,33.1659,33.3473,33.5287,33.7102,33.8916,34.0731,34.2545,34.4359,34.6174,34.7988,34.9802,35.1617,35.3431,35.5246,35.706,35.8874,36.0689,36.2503,36.4317,36.6132,36.7946,36.976,37.1575,37.3389,37.5204,37.7018,37.8832,38.0647,38.2461,38.4275,38.609,38.7904,38.9506,39.0542,39.1579,39.2616,39.3653,39.4689,39.5726,39.6763,39.78,39.8837,39.9873,40.091,40.1947,40.2984,40.4021,40.5057,40.6094,40.7131,40.8168,40.9204,41.0241,41.1278,41.2315,41.3485,41.5299,41.7114,41.8928,42.0743,42.2557,42.4371,42.6186,42.8,42.9814,43.1629,43.3443,43.5257,43.7072,43.8886,44.0701,44.2515,44.3722,44.4629,44.5536,44.6443,44.735,44.8257,44.9228,45.1042,45.2856,45.4671,45.6485,45.8299,46.0114,46.1645,46.2854,46.4064,46.5273,46.6483,46.7693,46.8902,47.0112,47.1321,47.2531,47.4084,47.5898,47.7713,47.9419,48.083,48.2242,48.3653,48.5064,48.6475,48.7886,48.9297,49.0709,49.212,49.3531,49.4942,49.6353,49.7764,49.9176,50.0587,50.1998,50.3409,50.482,50.6232,50.7643,50.9054,51.0465,51.1876,51.3287,51.4699,51.611,51.7521,51.8932,52.0343,52.1982,52.3796,52.5611,52.7246,52.8379,52.9513,53.0647,53.1781,53.2915,53.4049,53.5183,53.6317,53.7451,53.8585,53.9719,54.0853,54.1987,54.3121,54.4255,54.5389,54.6523,54.7657,54.8791,54.9925,55.1059,55.2193,55.3327,55.4461,55.5595,55.6729,55.6886,55.6886,55.6886,55.7904,55.9719,56.1533,56.3347,56.5162,56.6976,56.879,56.8862,56.8862,56.8862,57.006,57.1874,57.3689,57.5503,57.7317,57.9132,58.0838,58.0838,58.0838,58.0838,58.2216,58.403,58.5844,58.7659,58.9473,59.1287,59.3102,59.4916,59.6731,59.8545,60.0359,60.2174,60.3988,60.5802,60.7617,60.9431,61.0934,61.1539,61.2144,61.2749,61.3353,61.3958,61.4563,61.5168,61.5772,61.6377,61.7198,61.8407,61.9617,62.0826,62.2036,62.3246,62.4455,62.5665,62.6874,62.8084,62.9569,63.1383,63.3198,63.5012,63.6826,63.8641,64.0455,64.0719,64.0719,64.0719,64.1222,64.2129,64.3036,64.3943,64.485,64.5757,64.6665,64.7716,64.8774,64.9833,65.0891,65.195,65.3008,65.4066,65.5125,65.6183,65.7242,65.83,65.9358,66.0417,66.1475,66.2533,66.3592,66.465,66.5709,66.6767,66.7825,66.8884,66.9942,67.1,67.2059,67.3117,67.4176,67.5234,67.6292,67.7351,67.8409,67.9468,68.0526,68.1584,68.2643,68.3701,68.4759,68.5818,68.6876,68.7935,68.9257,69.1072,69.2886,69.4611,69.4611,69.4611,69.4611,69.597,69.7784,69.9599,70.0599,70.0599,70.0599,70.0599,70.0599,70.0599,70.0599,70.0599,70.0599,70.0599,70.1593,70.3407,70.5222,70.6587,70.6587,70.6587,70.6587,70.6587,70.6587,70.6587,70.6587,70.6587,70.6587,70.6901,70.7808,70.8716,70.9623,71.053,71.1437,71.2344,71.3251,71.4159,71.5066,71.5973,71.688,71.7787,71.8668,71.9394,72.012,72.0846,72.1571,72.2297,72.3023,72.3749,72.4474,72.52,72.5926,72.6651,72.7377,72.8103,72.8829,72.9554,73.028,73.0539,73.0539,73.0539,73.0816,73.1622,73.2428,73.3235,73.4041,73.4848,73.5654,73.646,73.7267,73.8073,73.888,73.9686,74.0492,74.1299,74.2105,74.2912,74.3718,74.4524,74.5331,74.6137,74.6943,74.775,74.8556,74.9363,75.0169,75.0975,75.1782,75.2588,75.3395,75.4201,75.5459,75.6971,75.8483,75.9995,76.1507,76.3019,76.4531,76.6043,76.7555,76.9067,77.0579,77.2091,77.3603,77.5115,77.6627,77.8139,77.9651,78.1163,78.2675,78.4187,78.5192,78.6099,78.7006,78.7913,78.882,78.9728,79.0635,79.1542,79.2449,79.3356,79.4263,79.5171,79.6078,79.6792,79.7397,79.8002,79.8607,79.9212,79.9816,80.0421,80.1026,80.1631,80.2236,80.284,80.3445,80.405,80.4655,80.5259,80.5864,80.6469,80.7074,80.7679,80.8283,80.9898,81.1713,81.3527,81.4371,81.4371,81.4371,81.4371,81.4371,81.4371,81.4371,81.5219,81.6126,81.7033,81.794,81.8847,81.9754,82.0762,82.1972,82.3182,82.4391,82.5601,82.681,82.802,82.923,83.0439,83.1649,83.2728,83.3635,83.4542,83.5449,83.6356,83.7263,83.8171,83.8889,83.957,84.025,84.093,84.1611,84.2291,84.2972,84.3652,84.4332,84.5013,84.5693,84.6374,84.7054,84.7734,84.8415,84.9095,84.9775,85.0456,85.1136,85.1817,85.2497,85.3177,85.3858,85.4538,85.5219,85.5899,85.6287,85.6287,85.6287,85.6287,85.6287,85.6287,85.6287,85.779,85.9605,86.1419,86.2275,86.2275,86.2275,86.2275,86.2275,86.2275,86.2275,86.2836,86.3441,86.4046,86.4651,86.5255,86.586,86.6465,86.707,86.7675,86.8263,86.8263,86.8263,86.8263,86.8593,86.9046,86.95,86.9954,87.0407,87.0861,87.1314,87.1768,87.2222,87.2675,87.3129,87.3582,87.4036,87.4251,87.4251,87.4251,87.4333,87.4696,87.5059,87.5422,87.5784,87.6147,87.651,87.6873,87.7236,87.7599,87.7962,87.8325,87.8687,87.905,87.9413,87.9776,88.0139,88.024,88.024,88.024,88.024,88.024,88.024,88.0461,88.2275,88.409,88.5904,88.7719,88.9533,89.1347,89.2531,89.3136,89.3741,89.4345,89.495,89.5555,89.616,89.6764,89.7369,89.7974,89.8204,89.8204,89.8204,89.8204,89.8204,89.8204,89.8204,89.8204,89.8204,89.8204,89.8204,89.8204,89.8204,89.8204,89.8204,89.8204,89.8275,89.888,89.9485,90.009,90.0695,90.1299,90.1904,90.2509,90.3114,90.3719,90.429,90.4744,90.5198,90.5651,90.6105,90.6558,90.7012,90.7466,90.7919,90.8373,90.8826,90.928,90.9734,91.0191,91.0897,91.1602,91.2308,91.3014,91.3719,91.4425,91.513,91.5836,91.6542,91.7247,91.7953,91.8658,91.9364,92.007,92.0775,92.1481,92.2186,92.2892,92.3597,92.4303,92.5009,92.5714,92.642,92.7125,92.7831,92.8537,92.9242,92.9948,93.0653,93.1359,93.2065,93.277,93.3476,93.4181,93.4887,93.5592,93.6298,93.7004,93.7709,93.8415,93.912,93.9826,94.0532,94.1237,94.1943,94.2648,94.3354,94.406,94.4765,94.5471,94.6176,94.6882,94.7587,94.8293,94.8999,94.9704,95.041,95.1115,95.1821,95.265,95.3557,95.4464,95.5371,95.6278,95.7186,95.8087,95.845,95.8813,95.9176,95.9539,95.9902,96.0265,96.0628,96.099,96.1353,96.1716,96.2079,96.2442,96.2805,96.3168,96.3531,96.3893,96.4293,96.4729,96.5164,96.56,96.6035,96.647,96.6906,96.7341,96.7777,96.8212,96.8648,96.9083,96.9519,96.9954,97.0389,97.0825,97.126,97.1696,97.2131,97.2567,97.3002,97.3438,97.3873,97.4309,97.4744,97.5179,97.5615,97.605,97.6486,97.6921,97.7357,97.7792,97.8228,97.8663,97.9098,97.9534,97.9969,98.0405,98.084,98.1276,98.1711,98.2147,98.2582,98.3017,98.3453,98.3888,98.4324,98.4759,98.5195,98.563,98.6066,98.6501,98.6937,98.7372,98.7807,98.8243,98.8678,98.9114,98.9549,98.9985,99.042,99.0856,99.1291,99.1726,99.2162,99.2597,99.3033,99.3468,99.3904,99.4339,99.4775,99.521,99.5646,99.6081,99.6516,99.6952,99.7387,99.7823,99.8258,99.8694,99.9129,99.9565,100] + } + ] + } + ] + }, + "testEvaluationReport": { + "reportType": "Evaluation", + "evaluationType": "Test", + "summary": { + "dictionary": "SpliceJunction", + "database": "..\/..\/..\/MTdatasets\/SpliceJunction\/SpliceJunction.txt", + "samplePercentage": 10, + "samplingMode": "Exclude sample", + "selectionVariable": "", + "selectionValue": "", + "instances": 2875, + "learningTask": "Classification analysis", + "targetVariable": "Class" + }, + "predictorsPerformance": [ + { + "rank": "R1", + "type": "Classifier", + "family": "Selective Naive Bayes", + "name": "Selective Naive Bayes", + "accuracy": 0.607652, + "compression": 0.166957, + "auc": 0.755468 + } + ], + "predictorsDetailedPerformance": { + "R1": { + "confusionMatrix": { + "values": ["EI","IE","N"], + "matrix": [ + [101,12,52], + [31,327,117], + [562,354,1319] + ] + } + } + }, + "liftCurves": [ + { + "targetValue": "EI", + "curves": [ + { + "classifier": "Optimal", + "values": [0,0.414265,0.82853,1.2428,1.65706,2.07133,2.48559,2.89986,3.31412,3.72839,4.14265,4.55692,4.97118,5.38545,5.79971,6.21398,6.62824,7.04251,7.45677,7.87104,8.2853,8.69957,9.11383,9.5281,9.94236,10.3566,10.7709,11.1852,11.5994,12.0137,12.428,12.8422,13.2565,13.6707,14.085,14.4993,14.9135,15.3278,15.7421,16.1563,16.5706,16.9849,17.3991,17.8134,18.2277,18.6419,19.0562,19.4705,19.8847,20.299,20.7133,21.1275,21.5418,21.9561,22.3703,22.7846,23.1988,23.6131,24.0274,24.4416,24.8559,25.2702,25.6844,26.0987,26.513,26.9272,27.3415,27.7558,28.17,28.5843,28.9986,29.4128,29.8271,30.2414,30.6556,31.0699,31.4841,31.8984,32.3127,32.7269,33.1412,33.5555,33.9697,34.384,34.7983,35.2125,35.6268,36.0411,36.4553,36.8696,37.2839,37.6981,38.1124,38.5267,38.9409,39.3552,39.7695,40.1837,40.598,41.0122,41.4265,41.8408,42.255,42.6693,43.0836,43.4978,43.9121,44.3264,44.7406,45.1549,45.5692,45.9834,46.3977,46.812,47.2262,47.6405,48.0548,48.469,48.8833,49.2976,49.7118,50.1261,50.5403,50.9546,51.3689,51.7831,52.1974,52.6117,53.0259,53.4402,53.8545,54.2687,54.683,55.0973,55.5115,55.9258,56.3401,56.7543,57.1686,57.5829,57.9971,58.4114,58.8256,59.2399,59.6542,60.0684,60.4827,60.897,61.3112,61.7255,62.1398,62.554,62.9683,63.3826,63.7968,64.2111,64.6254,65.0396,65.4539,65.8682,66.2824,66.6967,67.111,67.5252,67.9395,68.3537,68.768,69.1823,69.5965,70.0108,70.4251,70.8393,71.2536,71.6679,72.0821,72.4964,72.9107,73.3249,73.7392,74.1535,74.5677,74.982,75.3963,75.8105,76.2248,76.639,77.0533,77.4676,77.8818,78.2961,78.7104,79.1246,79.5389,79.9532,80.3674,80.7817,81.196,81.6102,82.0245,82.4388,82.853,83.2673,83.6816,84.0958,84.5101,84.9244,85.3386,85.7529,86.1671,86.5814,86.9957,87.4099,87.8242,88.2385,88.6527,89.067,89.4813,89.8955,90.3098,90.7241,91.1383,91.5526,91.9669,92.3811,92.7954,93.2097,93.6239,94.0382,94.4524,94.8667,95.281,95.6952,96.1095,96.5238,96.938,97.3523,97.7666,98.1808,98.5951,99.0094,99.4236,99.8379,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100] + }, + { + "classifier": "Selective Naive Bayes", + "values": [0,0.310699,0.621398,0.923091,1.19236,1.46164,1.73091,2.00018,2.26945,2.53872,2.82421,3.15562,3.48703,3.81844,4.14986,4.48127,4.81268,5.12644,5.39986,5.67327,5.94669,6.2201,6.49352,6.76693,7.04035,7.31376,7.58718,7.86059,8.13401,8.40742,8.68084,8.95425,9.22767,9.50108,9.7745,9.87632,10.0144,10.1525,10.2906,10.4287,10.5187,10.5187,10.6098,10.7594,10.909,11.0586,11.2082,11.3578,11.5074,11.657,11.8066,11.9562,12.1057,12.2553,12.4207,12.7522,13.0644,13.3405,13.6167,13.8929,14.1691,14.4452,14.7135,14.8984,15.0834,15.2683,15.4532,15.6382,15.8231,16.0081,16.193,16.3779,16.5629,16.7478,16.9328,17.1177,17.3026,17.4876,17.6725,17.8575,18.0424,18.2273,18.4258,18.6329,18.8401,19.0472,19.2543,19.4615,19.6686,19.9638,20.306,20.6483,20.9905,21.3327,21.6749,22.0171,22.3594,22.5684,22.7065,22.8446,22.9827,23.1208,23.2589,23.493,23.6311,23.7064,23.7818,23.8571,23.9769,24.3084,24.5677,24.7334,24.8581,25.0717,25.2852,25.4988,25.7123,25.9258,26.1394,26.3529,26.5664,26.78,26.9935,27.2071,27.4206,27.6341,27.8477,28.0612,28.2748,28.4883,28.7018,28.9154,29.1289,29.3424,29.556,29.7695,29.9831,30.1966,30.4101,30.6237,30.8372,31.0508,31.2643,31.4778,31.6914,31.9049,32.2406,32.6171,32.8577,33.0982,33.3388,33.5793,33.8198,34.0604,34.3009,34.5415,34.782,35.0225,35.2486,35.4527,35.5908,35.5908,35.5908,35.6481,35.9117,36.1753,36.439,36.6054,36.7435,36.9645,37.1854,37.4063,37.6273,37.8482,37.9163,37.9623,38.0083,38.0614,38.1304,38.1844,38.2153,38.2745,38.3337,38.3929,38.452,38.4726,38.6707,38.9013,39.1499,39.3984,39.647,39.8955,40.1441,40.3927,40.6412,40.8898,41.1023,41.2599,41.3545,41.476,41.7867,41.8282,41.8696,41.911,41.9765,42.0639,42.1514,42.2388,42.3263,42.4137,42.5012,42.5887,42.6761,42.7636,42.851,42.9385,43.0259,43.1134,43.2008,43.2883,43.3758,43.4632,43.5507,43.6381,43.7256,43.813,43.9005,43.988,44.0754,44.1629,44.2503,44.3378,44.4252,44.5127,44.6001,44.7586,44.8454,44.8831,44.9207,44.9568,45.1009,45.377,45.5331,45.5331,45.5331,45.7385,46.0014,46.1861,46.2536,46.2536,46.2536,46.2536,46.2824,46.4481,46.6318,46.83,46.926,47.1542,47.2839,47.3667,47.431,47.4784,47.5257,47.5731,47.6204,47.6678,47.7151,47.7625,47.8098,47.8571,47.9045,47.9518,48.0548,48.1654,48.2246,48.3055,48.4649,48.6242,48.7835,48.9429,49.0562,49.1494,49.2426,49.3358,49.429,49.5222,49.6155,49.7087,49.8019,49.8951,49.9883,50.0815,50.1747,50.2679,50.3963,50.5343,50.5764,50.7835,50.9906,51.1347,51.3329,51.485,51.577,51.6691,51.7291,51.7291,51.7291,51.7291,51.7381,51.7899,51.8417,51.9813,52.1614,52.2875,52.4293,52.5588,52.6882,52.8177,52.9471,53.0766,53.2061,53.3355,53.465,53.5944,53.7239,53.7464,53.8528,54.0976,54.3424,54.5872,54.832,55.0768,55.3216,55.5664,55.7817,55.9888,56.196,56.4031,56.6102,56.8174,57.0245,57.2316,57.4388,57.6459,57.853,58.0602,58.2673,58.4744,58.6816,58.8887,58.9607,59.2714,59.5821,59.9244,60.3386,60.3851,60.3967,60.4082,60.4197,60.4312,60.4427,60.4542,60.4657,60.4772,60.4887,60.5002,60.5117,60.5608,60.6682,60.7756,60.883,60.9904,61.0978,61.2052,61.3126,61.42,61.5274,61.6348,61.7422,61.8496,61.957,62.0644,62.1718,62.2792,62.3866,62.494,62.5865,62.6693,62.8963,63.1394,63.2637,63.3465,63.4671,63.6583,63.8495,64.0407,64.2319,64.3912,64.6548,64.9185,65.1821,65.4359,65.6844,65.8006,65.9042,66.0077,66.1113,66.2149,66.3112,66.3941,66.4769,66.5598,66.6427,66.7255,66.8084,66.8912,66.9741,67.0569,67.1398,67.2226,67.3055,67.3883,67.4712,67.554,67.6369,67.7197,67.8026,67.8854,67.9683,68.0512,68.134,68.2169,68.2997,68.5257,68.7516,68.9776,69.2036,69.4295,69.6555,69.8815,70.0288,70.1369,70.3026,70.4683,70.6052,70.6952,70.8043,70.9133,71.0223,71.1313,71.2403,71.4157,71.5238,71.6138,71.6138,71.6138,71.6138,71.6138,71.6138,71.6333,71.6679,71.7024,71.7369,71.7953,71.8909,71.9865,72.0821,72.1777,72.2703,72.3623,72.4544,72.6825,72.8386,72.9422,73.0457,73.1808,73.1988,73.1988,73.2238,73.2557,73.2875,73.3194,73.4011,73.6242,73.8473,74.0703,74.2934,74.4251,74.5245,74.6239,74.7233,74.8228,74.9222,75.0216,75.121,75.2205,75.3199,75.4193,75.5187,75.6182,75.7176,75.817,75.9164,76.0159,76.1469,76.3372,76.5276,76.7179,76.9082,77.0986,77.2889,77.4793,77.6696,77.8599,78.0503,78.2406,78.431,78.5303,78.5303,78.5303,78.5303,78.5618,78.6653,78.7689,78.8725,78.976,79.0796,79.1832,79.2867,79.3903,79.4939,79.5974,79.701,79.8046,79.9081,80.0117,80.1153,80.2188,80.3224,80.426,80.5295,80.6331,80.7367,80.8402,80.9438,81.0474,81.1239,81.1239,81.1239,81.1239,81.1908,81.2763,81.3618,81.4473,81.5328,81.6182,81.7037,81.7892,81.8747,81.9602,82.0457,82.1311,82.2166,82.3021,82.3876,82.4731,82.5586,82.644,82.7295,82.815,82.9005,82.986,83.0607,83.1338,83.2069,83.28,83.3531,83.4262,83.6275,83.8347,84.013,84.0544,84.0958,84.1372,84.1787,84.2201,84.2615,84.2939,84.3594,84.4348,84.5101,84.5854,84.6607,84.736,84.8114,84.8867,84.962,85.0373,85.1127,85.188,85.2633,85.3386,85.4139,85.5403,85.7061,85.9533,86.1086,86.264,86.3767,86.4709,86.565,86.6592,86.7533,86.8475,86.9416,87.0349,87.1086,87.1824,87.2561,87.3298,87.4035,87.4773,87.551,87.6247,87.6984,87.7722,87.8459,87.9196,87.9933,88.0671,88.1408,88.2145,88.2882,88.362,88.4357,88.5094,88.5831,88.6569,88.7306,88.8043,88.878,88.9518,89.0255,89.0992,89.1729,89.2467,89.3204,89.3941,89.4678,89.5416,89.6153,89.689,89.7627,89.8365,89.9102,89.9839,90.0576,90.2786,90.4995,90.7205,90.9414,91.1623,91.2104,91.2104,91.2424,91.3345,91.4265,91.506,91.54,91.5741,91.6081,91.6422,91.6762,91.7103,91.7443,91.7784,91.8124,91.8465,91.8805,91.9146,91.9486,91.9826,92.0167,92.0507,92.0848,92.1188,92.1529,92.1869,92.221,92.255,92.2891,92.3231,92.3572,92.3631,92.428,92.5094,92.5589,92.6085,92.6581,92.7076,92.7572,92.8068,92.8564,92.9059,92.9555,93.0051,93.0546,93.1042,93.1538,93.2033,93.2529,93.3025,93.3521,93.4016,93.4512,93.5008,93.5503,93.5999,93.6495,93.699,93.7486,93.7982,93.8478,93.8973,93.9469,93.9965,94.046,94.0956,94.1452,94.1947,94.2443,94.2939,94.3435,94.393,94.4426,94.4922,94.5425,94.5943,94.6461,94.6686,94.6686,94.6821,94.7857,94.8127,94.8127,94.8127,94.8127,94.8127,94.8127,94.9268,95.0648,95.2029,95.341,95.389,95.389,95.389,95.389,95.3904,95.4008,95.4111,95.4215,95.4318,95.4422,95.4525,95.4629,95.4733,95.4836,95.494,95.5043,95.5147,95.525,95.5331,95.5786,95.6659,95.7531,95.8403,95.9275,96.0147,96.1019,96.1275,96.1472,96.167,96.1867,96.2064,96.2262,96.2459,96.2591,96.2681,96.2771,96.2861,96.2951,96.3041,96.3131,96.3221,96.3311,96.3401,96.3491,96.3581,96.3672,96.3762,96.3852,96.3942,96.4032,96.4122,96.4212,96.4302,96.4392,96.4482,96.4572,96.4662,96.4752,96.4842,96.4932,96.5022,96.5112,96.5203,96.5293,96.5383,96.5473,96.5563,96.5653,96.5743,96.5833,96.5923,96.6013,96.6103,96.6193,96.6283,96.6373,96.6463,96.6553,96.6643,96.6733,96.6824,96.6859,96.7444,96.8338,96.8556,96.8774,96.8992,96.921,96.9428,96.9646,97.0033,97.0551,97.1069,97.1306,97.1466,97.1625,97.1784,97.1944,97.2103,97.2262,97.2422,97.2581,97.274,97.29,97.3059,97.3218,97.3378,97.3537,97.3696,97.3856,97.4015,97.4107,97.417,97.4233,97.4295,97.4358,97.4421,97.4484,97.4546,97.4609,97.4672,97.4735,97.4798,97.486,97.4923,97.4986,97.5049,97.5111,97.5174,97.5237,97.53,97.5362,97.5425,97.5488,97.551,97.5639,97.5769,97.5898,97.6028,97.6157,97.6287,97.6416,97.6546,97.6675,97.6805,97.6934,97.7733,97.8596,97.9459,98.0322,98.1185,98.2049,98.2912,98.3775,98.4296,98.4555,98.4814,98.5073,98.5332,98.5591,98.6281,98.6972,98.7662,98.8353,98.8473,98.8473,98.8473,98.8473,98.8473,98.8473,98.8563,98.8722,98.8881,98.9041,98.92,98.9359,98.9519,98.9678,98.9837,98.9997,99.0156,99.0315,99.0475,99.0634,99.0793,99.0953,99.1112,99.1271,99.1431,99.159,99.1749,99.1909,99.2068,99.2227,99.2387,99.2546,99.2705,99.2942,99.3281,99.3619,99.3957,99.4295,99.4633,99.4971,99.531,99.5648,99.5986,99.6324,99.6662,99.7001,99.7339,99.7677,99.8015,99.8353,99.8585,99.8652,99.8719,99.8786,99.8852,99.8919,99.8986,99.9053,99.912,99.9187,99.9253,99.932,99.9387,99.9454,99.9521,99.9587,99.9654,99.9721,99.9788,99.9855,99.9922,99.9988,100,100,100,100,100,100,100,100,100,100,100,100] + } + ] + }, + { + "targetValue": "IE", + "curves": [ + { + "classifier": "Optimal", + "values": [0,0.414863,0.829726,1.24459,1.65945,2.07431,2.48918,2.90404,3.3189,3.73377,4.14863,4.56349,4.97835,5.39322,5.80808,6.22294,6.63781,7.05267,7.46753,7.8824,8.29726,8.71212,9.12698,9.54185,9.95671,10.3716,10.7864,11.2013,11.6162,12.031,12.4459,12.8608,13.2756,13.6905,14.1053,14.5202,14.9351,15.3499,15.7648,16.1797,16.5945,17.0094,17.4242,17.8391,18.254,18.6688,19.0837,19.4986,19.9134,20.3283,20.7431,21.158,21.5729,21.9877,22.4026,22.8175,23.2323,23.6472,24.062,24.4769,24.8918,25.3066,25.7215,26.1364,26.5512,26.9661,27.381,27.7958,28.2107,28.6255,29.0404,29.4553,29.8701,30.285,30.6999,31.1147,31.5296,31.9444,32.3593,32.7742,33.189,33.6039,34.0188,34.4336,34.8485,35.2633,35.6782,36.0931,36.5079,36.9228,37.3377,37.7525,38.1674,38.5823,38.9971,39.412,39.8268,40.2417,40.6566,41.0714,41.4863,41.9012,42.316,42.7309,43.1457,43.5606,43.9755,44.3903,44.8052,45.2201,45.6349,46.0498,46.4646,46.8795,47.2944,47.7092,48.1241,48.539,48.9538,49.3687,49.7835,50.1984,50.6133,51.0281,51.443,51.8579,52.2727,52.6876,53.1025,53.5173,53.9322,54.347,54.7619,55.1768,55.5916,56.0065,56.4214,56.8362,57.2511,57.6659,58.0808,58.4957,58.9105,59.3254,59.7403,60.1551,60.57,60.9848,61.3997,61.8146,62.2294,62.6443,63.0592,63.474,63.8889,64.3038,64.7186,65.1335,65.5483,65.9632,66.3781,66.7929,67.2078,67.6227,68.0375,68.4524,68.8672,69.2821,69.697,70.1118,70.5267,70.9416,71.3564,71.7713,72.1861,72.601,73.0159,73.4307,73.8456,74.2605,74.6753,75.0902,75.5051,75.9199,76.3348,76.7496,77.1645,77.5794,77.9942,78.4091,78.824,79.2388,79.6537,80.0685,80.4834,80.8983,81.3131,81.728,82.1429,82.5577,82.9726,83.3874,83.8023,84.2172,84.632,85.0469,85.4618,85.8766,86.2915,86.7063,87.1212,87.5361,87.9509,88.3658,88.7807,89.1955,89.6104,90.0253,90.4401,90.855,91.2698,91.6847,92.0996,92.5144,92.9293,93.3442,93.759,94.1739,94.5887,95.0036,95.4185,95.8333,96.2482,96.6631,97.0779,97.4928,97.9076,98.3225,98.7374,99.1522,99.5671,99.982,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100] + }, + { + "classifier": "Selective Naive Bayes", + "values": [0,0.390459,0.780918,1.17138,1.56184,1.9523,2.3373,2.66498,2.99266,3.32034,3.64802,3.9757,4.30339,4.63107,4.95875,5.28643,5.61411,5.94179,6.26948,6.59716,6.92484,7.25252,7.5802,7.90788,8.23556,8.56325,8.89093,9.21861,9.54629,9.87397,10.2017,10.5293,10.857,11.1847,11.5124,11.8401,12.1677,12.4954,12.8231,13.1508,13.4785,13.8062,14.1338,14.4615,14.7892,15.1169,15.4446,15.7722,16.0999,16.4276,16.7553,17.083,17.4106,17.7383,18.0651,18.3824,18.6996,19.0168,19.3341,19.6513,19.9546,20.1917,20.4288,20.6658,20.9029,21.14,21.377,21.6141,21.9027,22.199,22.4858,22.7601,23.0345,23.3088,23.5832,23.8575,24.1319,24.4062,24.6806,24.9549,25.2293,25.5036,25.7779,26.0523,26.3266,26.601,26.8753,27.1497,27.424,27.6984,27.9727,28.247,28.6436,28.984,29.2433,29.5026,29.7619,30.0212,30.2805,30.5398,30.7991,31.0584,31.3176,31.5769,31.8362,32.0955,32.3548,32.6259,32.906,33.1861,33.4662,33.7463,34.0265,34.3066,34.5867,34.8668,35.147,35.4271,35.7072,35.9873,36.2674,36.5476,36.8277,37.1078,37.3879,37.668,37.9482,38.2283,38.5084,38.7885,39.0686,39.3488,39.6289,39.909,40.1891,40.4693,40.7494,41.0295,41.3096,41.5897,41.8699,42.15,42.4301,42.7102,42.9903,43.2705,43.5506,43.8307,43.9168,44.2021,44.4873,44.7725,45.0577,45.3429,45.6197,45.8909,46.1622,46.4334,46.7047,46.9759,47.2472,47.5185,47.7897,48.061,48.3322,48.6035,48.8747,49.146,49.4172,49.6885,49.9598,50.231,50.5023,50.7735,51.0448,51.316,51.5873,51.8586,52.1298,52.4011,52.6723,53.0123,53.4271,53.8113,54.1847,54.5581,54.8808,55.0599,55.2391,55.4182,55.5974,55.7765,55.9557,56.1348,56.314,56.4931,56.6722,56.8514,57.0305,57.2097,57.3888,57.568,57.6551,57.768,57.9884,58.2088,58.4292,58.6496,58.8699,59.0903,59.3107,59.5311,59.7515,59.9719,60.2002,60.5114,60.8225,61.1111,61.3482,61.5852,61.8223,62.0594,62.2964,62.5335,62.7706,63.0076,63.2447,63.4818,63.7188,63.9559,64.1929,64.43,64.6671,64.9041,65.1412,65.3783,65.6153,65.8524,66.0895,66.2987,66.4646,66.6306,66.7965,66.9625,67.1284,67.2944,67.4182,67.7639,68.1097,68.1097,68.4073,68.7184,69.0126,69.2264,69.4401,69.6538,69.8675,70.0812,70.2949,70.5087,70.7224,70.9361,71.1498,71.3635,71.5772,71.791,72.0047,72.2184,72.4321,72.6458,72.8595,73.0733,73.287,73.5007,73.7144,73.9268,74.026,74.1164,74.2259,74.3354,74.4448,74.5543,74.6638,74.7733,74.8828,74.9922,75.1017,75.2112,75.3207,75.4301,75.5396,75.6491,75.7586,75.8681,75.9775,76.087,76.1965,76.306,76.4154,76.5249,76.6344,76.7439,76.8702,77.0012,77.1322,77.2632,77.3942,77.5253,77.6696,77.877,78.0303,78.1686,78.2107,78.2426,78.2745,78.3064,78.3383,78.355,78.414,78.4894,78.5649,78.6403,78.7157,78.7912,78.8666,78.9525,79.108,79.2636,79.3651,79.5159,79.6668,79.8177,79.9423,79.9753,80.0445,80.1136,80.1828,80.2406,80.2725,80.3044,80.3363,80.3682,80.4293,80.4984,80.6527,80.8442,81.0356,81.2271,81.4186,81.6101,81.8015,81.993,82.1845,82.376,82.5674,82.7589,82.9504,83.1419,83.3333,83.5248,83.7163,83.9078,83.9827,83.9827,83.9827,84.0255,84.0774,84.1288,84.1703,84.2118,84.2532,84.3494,84.4877,84.626,84.7643,84.8663,84.9117,84.9572,85.0027,85.0481,85.0936,85.1391,85.1845,85.23,85.2755,85.3209,85.3664,85.4118,85.4573,85.5028,85.5482,85.5937,85.6392,85.6846,85.7301,85.7756,85.821,85.8665,85.912,85.9574,86.0029,86.0029,86.0029,86.0029,86.0029,86.0221,86.0622,86.1024,86.1425,86.1827,86.2228,86.263,86.3031,86.3433,86.3834,86.4236,86.6089,86.8579,87.1068,87.3116,87.3577,87.4038,87.4519,87.521,87.5902,87.6279,87.6656,87.7033,87.741,87.7788,87.8165,87.8542,87.8788,87.8788,87.8788,87.8788,87.8788,87.9185,88.0014,88.0806,88.1584,88.2362,88.3139,88.3917,88.48,88.6544,88.7446,88.7446,88.7446,88.7446,88.7851,88.8274,88.8697,88.9121,88.9544,88.9967,89.0391,89.0814,89.1237,89.1661,89.2084,89.2507,89.2931,89.3354,89.3777,89.4201,89.4624,89.4661,89.4661,89.5175,89.5797,89.642,89.7042,89.7664,89.8286,89.8909,89.956,90.0215,90.087,90.1525,90.218,90.2835,90.3402,90.3721,90.404,90.436,90.4679,90.4942,90.9091,90.9921,91.075,91.158,91.1977,91.1977,91.2338,91.342,91.342,91.342,91.342,91.342,91.4863,91.4863,91.5158,91.5535,91.5912,91.629,91.6306,91.6306,91.6306,91.6306,91.7749,91.7749,91.7749,91.7749,91.7749,91.7749,91.7749,91.7749,91.7749,91.7749,91.7749,91.7749,91.7749,91.7749,91.7749,91.7749,91.7749,91.7749,91.7749,91.7749,91.7965,91.8795,91.9192,91.9192,91.9192,91.9192,91.9192,91.9192,91.9192,91.9192,91.9192,91.9192,91.9192,91.9192,91.9192,91.9192,91.9192,92.0274,92.1459,92.2739,92.4423,92.6407,92.6407,92.6407,92.6407,92.6407,92.6948,92.785,92.785,92.785,92.785,92.8887,92.9293,92.9714,93.0175,93.0636,93.0736,93.0736,93.0736,93.0736,93.0736,93.0762,93.088,93.0999,93.1117,93.1236,93.1354,93.1473,93.1591,93.171,93.1828,93.1947,93.2066,93.2179,93.2938,93.403,93.5122,93.6214,93.7305,93.8397,93.9394,93.9394,93.9875,94.0837,94.0837,94.0837,94.0837,94.0837,94.0837,94.0837,94.1478,94.24,94.3322,94.4016,94.4535,94.5053,94.6709,94.694,94.717,94.7401,94.7631,94.7862,94.8092,94.8323,94.8553,94.8783,94.9014,94.9244,94.9475,94.9495,94.9495,94.9495,94.9495,94.9495,94.9495,94.9558,94.9766,94.9973,95.018,95.0388,95.0595,95.0803,95.101,95.1218,95.1425,95.1632,95.184,95.2047,95.2255,95.3193,95.4185,95.5222,95.5267,95.5267,95.5267,95.5267,95.5267,95.5267,95.5267,95.5267,95.5267,95.5267,95.5988,95.7315,95.8047,95.8779,95.9511,96.0243,96.0975,96.1039,96.1039,96.1039,96.1039,96.1039,96.1039,96.1039,96.1039,96.1039,96.1039,96.1039,96.1039,96.1039,96.1039,96.1039,96.1039,96.1039,96.1039,96.1039,96.1039,96.1039,96.1039,96.1039,96.1039,96.1039,96.1039,96.1039,96.1039,96.1039,96.1039,96.1039,96.1039,96.1039,96.1039,96.2482,96.2482,96.2482,96.2482,96.2482,96.2482,96.2482,96.2482,96.2482,96.2482,96.2482,96.2482,96.2482,96.2482,96.2482,96.2482,96.2482,96.2482,96.2482,96.2482,96.2482,96.2482,96.2482,96.2722,96.4105,96.5488,96.6871,96.8254,96.9637,97.102,97.114,97.114,97.1168,97.1228,97.1287,97.1346,97.1405,97.1465,97.1524,97.1583,97.1642,97.1702,97.1761,97.182,97.188,97.1939,97.1998,97.2057,97.2117,97.2176,97.2235,97.2294,97.2354,97.2413,97.2472,97.2531,97.2583,97.2583,97.2583,97.2583,97.2583,97.2583,97.2583,97.2583,97.2583,97.2583,97.2583,97.2583,97.2583,97.2583,97.2583,97.2583,97.2583,97.2583,97.2606,97.3124,97.3643,97.4026,97.4026,97.4026,97.4026,97.4026,97.4026,97.4026,97.4026,97.4026,97.4026,97.4026,97.4026,97.4026,97.4026,97.4026,97.4026,97.4026,97.4026,97.4026,97.4026,97.4026,97.4026,97.4026,97.4026,97.4026,97.4026,97.4026,97.4026,97.4026,97.4026,97.4266,97.4958,97.5649,97.6341,97.7032,97.7724,97.8355,97.8355,97.8355,97.8355,97.8456,97.8622,97.8788,97.8954,97.912,97.9286,97.9452,97.9618,97.9784,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9798,97.9898,98.0013,98.0129,98.0244,98.0359,98.0474,98.059,98.0705,98.082,98.0935,98.1051,98.1166,98.1241,98.1241,98.1241,98.1241,98.1241,98.1241,98.1241,98.1241,98.1241,98.1241,98.1241,98.1241,98.1263,98.1392,98.152,98.1648,98.1777,98.1905,98.2033,98.2161,98.229,98.2418,98.2546,98.2675,98.2803,98.2931,98.306,98.3188,98.3316,98.3445,98.3573,98.3701,98.3829,98.3958,98.4086,98.4214,98.4343,98.4471,98.4599,98.4728,98.4856,98.4984,98.5113,98.5241,98.5369,98.5497,98.577,98.6231,98.6692,98.7139,98.7554,98.7969,98.8384,98.8799,98.9214,98.9628,99.0043,99.0458,99.0873,99.1288,99.1703,99.2118,99.2532,99.2947,99.3362,99.3777,99.4192,99.4607,99.5022,99.5437,99.5851,99.6266,99.6681,99.7096,99.7511,99.7926,99.8341,99.8755,99.917,99.9585,100] + } + ] + }, + { + "targetValue": "N", + "curves": [ + { + "classifier": "Optimal", + "values": [0,0.193212,0.386425,0.579637,0.772849,0.966062,1.15927,1.35249,1.5457,1.73891,1.93212,2.12534,2.31855,2.51176,2.70497,2.89819,3.0914,3.28461,3.47782,3.67103,3.86425,4.05746,4.25067,4.44388,4.6371,4.83031,5.02352,5.21673,5.40995,5.60316,5.79637,5.98958,6.1828,6.37601,6.56922,6.76243,6.95565,7.14886,7.34207,7.53528,7.72849,7.92171,8.11492,8.30813,8.50134,8.69456,8.88777,9.08098,9.27419,9.46741,9.66062,9.85383,10.047,10.2403,10.4335,10.6267,10.8199,11.0131,11.2063,11.3995,11.5927,11.786,11.9792,12.1724,12.3656,12.5588,12.752,12.9452,13.1384,13.3317,13.5249,13.7181,13.9113,14.1045,14.2977,14.4909,14.6841,14.8774,15.0706,15.2638,15.457,15.6502,15.8434,16.0366,16.2298,16.4231,16.6163,16.8095,17.0027,17.1959,17.3891,17.5823,17.7755,17.9688,18.162,18.3552,18.5484,18.7416,18.9348,19.128,19.3212,19.5144,19.7077,19.9009,20.0941,20.2873,20.4805,20.6737,20.8669,21.0601,21.2534,21.4466,21.6398,21.833,22.0262,22.2194,22.4126,22.6058,22.7991,22.9923,23.1855,23.3787,23.5719,23.7651,23.9583,24.1515,24.3448,24.538,24.7312,24.9244,25.1176,25.3108,25.504,25.6972,25.8905,26.0837,26.2769,26.4701,26.6633,26.8565,27.0497,27.2429,27.4362,27.6294,27.8226,28.0158,28.209,28.4022,28.5954,28.7886,28.9819,29.1751,29.3683,29.5615,29.7547,29.9479,30.1411,30.3343,30.5276,30.7208,30.914,31.1072,31.3004,31.4936,31.6868,31.88,32.0733,32.2665,32.4597,32.6529,32.8461,33.0393,33.2325,33.4257,33.619,33.8122,34.0054,34.1986,34.3918,34.585,34.7782,34.9714,35.1647,35.3579,35.5511,35.7443,35.9375,36.1307,36.3239,36.5171,36.7103,36.9036,37.0968,37.29,37.4832,37.6764,37.8696,38.0628,38.256,38.4493,38.6425,38.8357,39.0289,39.2221,39.4153,39.6085,39.8017,39.995,40.1882,40.3814,40.5746,40.7678,40.961,41.1542,41.3474,41.5407,41.7339,41.9271,42.1203,42.3135,42.5067,42.6999,42.8931,43.0864,43.2796,43.4728,43.666,43.8592,44.0524,44.2456,44.4388,44.6321,44.8253,45.0185,45.2117,45.4049,45.5981,45.7913,45.9845,46.1778,46.371,46.5642,46.7574,46.9506,47.1438,47.337,47.5302,47.7235,47.9167,48.1099,48.3031,48.4963,48.6895,48.8827,49.0759,49.2692,49.4624,49.6556,49.8488,50.042,50.2352,50.4284,50.6216,50.8149,51.0081,51.2013,51.3945,51.5877,51.7809,51.9741,52.1673,52.3606,52.5538,52.747,52.9402,53.1334,53.3266,53.5198,53.713,53.9062,54.0995,54.2927,54.4859,54.6791,54.8723,55.0655,55.2587,55.4519,55.6452,55.8384,56.0316,56.2248,56.418,56.6112,56.8044,56.9976,57.1909,57.3841,57.5773,57.7705,57.9637,58.1569,58.3501,58.5433,58.7366,58.9298,59.123,59.3162,59.5094,59.7026,59.8958,60.089,60.2823,60.4755,60.6687,60.8619,61.0551,61.2483,61.4415,61.6347,61.828,62.0212,62.2144,62.4076,62.6008,62.794,62.9872,63.1804,63.3737,63.5669,63.7601,63.9533,64.1465,64.3397,64.5329,64.7261,64.9194,65.1126,65.3058,65.499,65.6922,65.8854,66.0786,66.2718,66.4651,66.6583,66.8515,67.0447,67.2379,67.4311,67.6243,67.8175,68.0108,68.204,68.3972,68.5904,68.7836,68.9768,69.17,69.3632,69.5565,69.7497,69.9429,70.1361,70.3293,70.5225,70.7157,70.9089,71.1022,71.2954,71.4886,71.6818,71.875,72.0682,72.2614,72.4546,72.6478,72.8411,73.0343,73.2275,73.4207,73.6139,73.8071,74.0003,74.1935,74.3868,74.58,74.7732,74.9664,75.1596,75.3528,75.546,75.7392,75.9325,76.1257,76.3189,76.5121,76.7053,76.8985,77.0917,77.2849,77.4782,77.6714,77.8646,78.0578,78.251,78.4442,78.6374,78.8306,79.0239,79.2171,79.4103,79.6035,79.7967,79.9899,80.1831,80.3763,80.5696,80.7628,80.956,81.1492,81.3424,81.5356,81.7288,81.922,82.1153,82.3085,82.5017,82.6949,82.8881,83.0813,83.2745,83.4677,83.661,83.8542,84.0474,84.2406,84.4338,84.627,84.8202,85.0134,85.2067,85.3999,85.5931,85.7863,85.9795,86.1727,86.3659,86.5591,86.7524,86.9456,87.1388,87.332,87.5252,87.7184,87.9116,88.1048,88.2981,88.4913,88.6845,88.8777,89.0709,89.2641,89.4573,89.6505,89.8438,90.037,90.2302,90.4234,90.6166,90.8098,91.003,91.1962,91.3894,91.5827,91.7759,91.9691,92.1623,92.3555,92.5487,92.7419,92.9351,93.1284,93.3216,93.5148,93.708,93.9012,94.0944,94.2876,94.4808,94.6741,94.8673,95.0605,95.2537,95.4469,95.6401,95.8333,96.0265,96.2198,96.413,96.6062,96.7994,96.9926,97.1858,97.379,97.5722,97.7655,97.9587,98.1519,98.3451,98.5383,98.7315,98.9247,99.1179,99.3112,99.5044,99.6976,99.8908,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100,100] + }, + { + "classifier": "Selective Naive Bayes", + "values": [0,0.15296,0.30592,0.458879,0.611839,0.764799,0.917759,1.07072,1.22368,1.37139,1.5121,1.63501,1.7367,1.83839,1.94008,2.04177,2.14346,2.24515,2.40255,2.56048,2.72177,2.91499,3.03259,3.1454,3.28341,3.42142,3.57934,3.73817,3.897,4.05582,4.21465,4.37348,4.5323,4.69113,4.84996,5.00878,5.16761,5.32644,5.48527,5.64409,5.80292,5.96175,6.12057,6.2794,6.43823,6.59706,6.75588,6.91471,7.07354,7.23236,7.39119,7.55002,7.70885,7.86767,8.0265,8.18533,8.34415,8.50298,8.66181,8.82064,8.97946,9.13829,9.29712,9.45594,9.61477,9.7736,9.93243,10.0638,10.1927,10.3215,10.4503,10.6004,10.7582,10.9159,11.0736,11.2313,11.3891,11.5468,11.7045,11.8622,12.02,12.1777,12.3354,12.4931,12.6509,12.8086,12.9663,13.124,13.259,13.384,13.5091,13.6341,13.7591,13.8841,14.002,14.1213,14.3145,14.3826,14.4871,14.5915,14.6959,14.8004,14.9048,15.0093,15.1137,15.2181,15.3226,15.427,15.5315,15.6359,15.7735,15.9204,16.0672,16.214,16.3609,16.5077,16.6546,16.8014,16.9483,17.0951,17.2419,17.3888,17.5356,17.6825,17.8293,17.9761,18.123,18.2728,18.4274,18.5652,18.6891,18.8099,18.9306,19.0619,19.1521,19.2423,19.3324,19.4226,19.5172,19.6246,19.7319,19.8447,19.9933,20.1419,20.2905,20.4392,20.6149,20.7535,20.8669,21.0259,21.1778,21.3296,21.4814,21.6332,21.785,21.9368,22.0886,22.2404,22.3922,22.544,22.6959,22.8477,22.9995,23.1513,23.3031,23.4549,23.6067,23.7585,23.9103,24.0621,24.2139,24.3658,24.5176,24.6178,24.7788,24.9398,25.1008,25.2618,25.4536,25.6468,25.8401,25.9535,26.0856,26.264,26.4423,26.6207,26.799,26.9321,27.0609,27.1897,27.2345,27.4126,27.5672,27.7143,27.8574,28.0005,28.1437,28.2868,28.4299,28.573,28.7161,28.8593,29.0024,29.1455,29.2886,29.4318,29.5749,29.718,29.8611,30.0042,30.1474,30.2905,30.4631,30.6403,30.8174,30.9945,31.1716,31.3487,31.5258,31.7029,31.88,32.0572,32.2343,32.4114,32.5605,32.6571,32.7537,32.8503,32.9469,33.0435,33.1401,33.2367,33.3333,33.4299,33.5265,33.6232,33.7198,33.8164,33.913,34.0096,34.1001,34.1791,34.2582,34.3372,34.4162,34.4953,34.5743,34.6534,34.8118,34.9657,35.0985,35.2313,35.3642,35.497,35.6298,35.7627,35.8955,36.0283,36.1612,36.294,36.4331,36.578,36.723,36.8308,36.8952,36.9582,37.0548,37.1514,37.248,37.3446,37.4412,37.5378,37.6344,37.731,37.8276,37.9242,38.0208,38.1174,38.214,38.3107,38.4073,38.5039,38.6005,38.7517,38.8575,39.0121,39.1213,39.2305,39.3721,39.5377,39.7033,39.8353,39.9642,40.093,40.2306,40.3707,40.5108,40.6508,40.7909,40.931,41.0711,41.2111,41.3512,41.4913,41.6314,41.7715,41.9115,42.0516,42.1733,42.2922,42.4111,42.53,42.6696,42.8479,43.0263,43.2046,43.383,43.4778,43.5551,43.6324,43.7404,43.906,44.0716,44.2372,44.4028,44.5685,44.7341,44.8997,45.0653,45.2309,45.3965,45.5621,45.7157,45.7969,45.9173,46.0504,46.1623,46.2741,46.386,46.4979,46.6097,46.7238,46.8526,46.9814,47.1102,47.3034,47.4462,47.5785,47.675,47.8312,47.9873,48.1435,48.2996,48.4558,48.612,48.7681,48.9243,49.0804,49.2366,49.3927,49.5489,49.7051,49.8612,50.0174,50.1735,50.3297,50.4859,50.642,50.7982,50.9543,51.1105,51.2666,51.4228,51.579,51.7641,51.9195,52.0161,52.0161,52.0833,52.0833,52.1841,52.3023,52.4354,52.5685,52.7016,52.8347,52.9678,53.1009,53.234,53.3671,53.5002,53.6333,53.7664,53.8995,54.0326,54.1657,54.2988,54.4319,54.565,54.6981,54.8312,54.9643,55.0974,55.2305,55.3636,55.4968,55.6299,55.763,55.8961,56.0292,56.1623,56.2954,56.4285,56.5272,56.6,56.761,56.922,56.9703,57.1069,57.3001,57.3253,57.3394,57.4481,57.5568,57.6655,57.7742,57.8829,57.9694,58.039,58.1085,58.1781,58.2476,58.3172,58.3868,58.4563,58.5259,58.679,58.8446,59.0102,59.1758,59.3414,59.4979,59.5703,59.6428,59.7782,59.8757,59.9614,60.1352,60.3091,60.483,60.5938,60.6641,60.7344,60.8046,60.8749,60.9451,61.0154,61.0857,61.1559,61.2262,61.2964,61.3726,61.4886,61.6179,61.7682,61.9185,62.0614,62.1834,62.3055,62.4275,62.5495,62.6715,62.7936,62.8801,63.025,63.1916,63.2818,63.372,63.4621,63.5523,63.6425,63.7332,63.824,63.9147,64.0055,64.0962,64.187,64.2777,64.3685,64.4592,64.55,64.6407,64.7315,64.8222,64.913,65.0037,65.0945,65.1852,65.276,65.3667,65.4575,65.5482,65.639,65.7342,65.8602,65.989,66.0618,66.1321,66.2023,66.2886,66.4819,66.6166,66.7308,66.845,66.9592,67.0733,67.1875,67.3017,67.408,67.4322,67.4563,67.5319,67.6019,67.7167,67.9015,68.0465,68.1914,68.3363,68.4812,68.6744,68.7361,68.8238,68.9114,68.9991,69.0867,69.1744,69.262,69.3496,69.4373,69.5249,69.6126,69.7002,69.7879,69.8755,69.9631,70.0508,70.1384,70.2261,70.3137,70.4014,70.489,70.5766,70.6643,70.7519,70.8396,70.9272,71.0149,71.1025,71.1901,71.2778,71.3654,71.4531,71.5407,71.6284,71.6398,71.7164,71.7889,71.8613,71.9646,72.1256,72.2572,72.3055,72.3538,72.4021,72.454,72.5432,72.6323,72.7215,72.8107,72.9587,73.1351,73.2471,73.3367,73.3753,73.4388,73.5131,73.5874,73.6617,73.7399,73.8365,73.9331,74.0297,74.1263,74.223,74.3196,74.4162,74.5632,74.6479,74.7549,74.8618,74.9688,75.0758,75.1827,75.2897,75.3966,75.5036,75.6105,75.7175,75.8245,75.9314,76.0384,76.1453,76.2523,76.3592,76.4662,76.5732,76.6801,76.8089,76.8515,76.9027,76.9993,77.0937,77.1876,77.2816,77.3755,77.4694,77.5633,77.6573,77.7512,77.8451,77.939,78.0329,78.1269,78.2208,78.3147,78.4086,78.5026,78.5965,78.6904,78.7843,78.8782,78.9722,79.0661,79.16,79.2539,79.3479,79.4187,79.5228,79.6408,79.7589,79.877,79.9951,80.1131,80.2312,80.3493,80.4673,80.5854,80.7035,80.8216,80.928,80.9924,81.0484,81.0938,81.1517,81.2097,81.2696,81.334,81.3984,81.4628,81.5272,81.5916,81.6583,81.7742,81.894,82.0397,82.1714,82.256,82.3405,82.425,82.5096,82.5941,82.6786,82.7631,82.8477,82.9322,83.0167,83.1256,83.2661,83.4066,83.5472,83.6226,83.6562,83.6898,83.7234,83.757,83.7906,83.8242,83.8578,83.9102,83.9746,84.039,84.1034,84.1678,84.2826,84.3871,84.4528,84.5185,84.5842,84.6499,84.7156,84.7813,84.8469,84.9126,84.9783,85.044,85.1097,85.1754,85.2411,85.3068,85.3725,85.4382,85.4839,85.5201,85.5795,85.639,85.6984,85.7579,85.8173,85.8768,85.9362,85.9957,86.0551,86.1146,86.174,86.2335,86.2929,86.3524,86.4118,86.4713,86.5307,86.5902,86.6496,86.7091,86.7685,86.828,86.8874,86.9469,87.0063,87.0658,87.1153,87.1539,87.1925,87.2312,87.2698,87.3085,87.3471,87.3908,87.4391,87.4874,87.5357,87.584,87.6323,87.7856,87.8507,87.9183,87.986,88.0536,88.1212,88.1888,88.2565,88.3128,88.3369,88.3611,88.4661,88.561,88.6286,88.6962,88.7639,88.8315,88.8991,88.9667,89.1161,89.1402,89.1644,89.2168,89.3222,89.4275,89.5329,89.6337,89.7303,89.8269,89.9236,90.0202,90.1168,90.2134,90.2918,90.3562,90.399,90.4183,90.4377,90.457,90.4999,90.5429,90.5858,90.6287,90.6717,90.7146,90.7575,90.8005,90.8434,90.8864,90.9293,90.9722,91.0152,91.0581,91.101,91.144,91.1869,91.2298,91.2728,91.3157,91.3586,91.3978,91.4006,91.4629,91.5252,91.5875,91.6499,91.7122,91.7745,91.8368,91.8992,91.9615,92.0238,92.0894,92.164,92.2387,92.3133,92.388,92.4626,92.5373,92.6119,92.6866,92.7612,92.8359,92.9105,92.9852,93.0598,93.1345,93.2091,93.272,93.3343,93.3966,93.459,93.5213,93.5836,93.6459,93.7083,93.7706,93.8329,93.8952,93.9576,94.0199,94.0822,94.1446,94.2069,94.2692,94.3315,94.3939,94.4562,94.5185,94.5691,94.6013,94.6405,94.6957,94.7509,94.7917,94.8312,94.8767,94.9221,94.9676,95.013,95.0585,95.1088,95.1764,95.244,95.3117,95.3793,95.4469,95.5145,95.5822,95.6498,95.7174,95.785,95.8527,95.9203,95.9879,96.0349,96.0349,96.0711,96.1108,96.1504,96.19,96.2297,96.2693,96.3089,96.3486,96.3882,96.4278,96.4675,96.5071,96.5467,96.5864,96.626,96.6656,96.7053,96.7449,96.7845,96.8242,96.8638,96.9034,96.9431,96.9827,97.0223,97.062,97.1016,97.1412,97.1809,97.2205,97.2601,97.2998,97.3394,97.379,97.4187,97.4583,97.4979,97.5376,97.5772,97.6168,97.6639,97.7375,97.8111,97.8847,97.9583,98.0319,98.1055,98.1791,98.1959,98.2072,98.2186,98.23,98.2413,98.2527,98.2891,98.3255,98.3619,98.3983,98.4347,98.4711,98.5075,98.5439,98.5803,98.6167,98.6531,98.6895,98.7259,98.7623,98.7987,98.8351,98.8715,98.9079,98.9443,98.9807,99.0171,99.0535,99.0899,99.1263,99.1627,99.1991,99.2356,99.272,99.3084,99.3448,99.3812,99.4176,99.454,99.4904,99.5268,99.5632,99.5996,99.636,99.6724,99.7088,99.7452,99.7816,99.818,99.8544,99.8908,99.9272,99.9636,100] + } + ] + } + ] + }, + "preparationReport": { + "reportType": "Preparation", + "summary": { + "dictionary": "SpliceJunction", + "variables": { + "types": [ + "Categorical", + "Numerical", + "Table" + ], + "numbers": [ + 12, + 90, + 1 + ] + }, + "database": "..\/..\/..\/MTdatasets\/SpliceJunction\/SpliceJunction.txt", + "samplePercentage": 10, + "samplingMode": "Include sample", + "selectionVariable": "", + "selectionValue": "", + "instances": 303, + "learningTask": "Classification analysis", + "targetVariable": "Class", + "targetDescriptiveStats": { + "values": 3, + "mode": "N", + "modeFrequency": 167, + "missingNumber": 0, + "sparseMissingNumber": 0 + }, + "targetValues": { + "values": ["EI","IE","N"], + "frequencies": [67,69,167] + }, + "evaluatedVariables": 101, + "nativeVariables": 1, + "constructedVariables": 100, + "informativeVariables": 7, + "selectedVariables": 7, + "featureEngineering": { + "maxNumberOfConstructedVariables": 100, + "maxNumberOfTextFeatures": 0, + "maxNumberOfTrees": 0, + "maxNumberOfVariablePairs": 0 + }, + "discretization": "MODL", + "valueGrouping": "MODL", + "nullModel": { + "constructionCost": 0.693147, + "preparationCost": 10.7442, + "dataCost": 296.926 + } + }, + "variablesStatistics": [ + { + "rank": "R1", + "name": "Mode(DNA.Char) where Pos <= 30.5", + "type": "Categorical", + "level": 0.0744365, + "parts": 2, + "values": 4, + "mode": "C", + "modeFrequency": 122, + "missingNumber": 0, + "sparseMissingNumber": 0, + "constructionCost": 7.76817, + "preparationCost": 23.5167, + "dataCost": 254.125, + "derivationRule": "TableMode(`DNA where Pos <= 30.5`, Char)" + }, + { + "rank": "R2", + "name": "Mode(DNA.Char) where Pos in ]15.5, 30.5]", + "type": "Categorical", + "level": 0.0508826, + "parts": 3, + "values": 4, + "mode": "C", + "modeFrequency": 114, + "missingNumber": 0, + "sparseMissingNumber": 0, + "constructionCost": 9.15447, + "preparationCost": 31.5551, + "dataCost": 251.964, + "derivationRule": "TablePartitionMode(`TablePartition(DNA) per Pos(4)`, Char)" + }, + { + "rank": "R3", + "name": "Mode(DNA.Char)", + "type": "Categorical", + "level": 0.0461059, + "parts": 3, + "values": 4, + "mode": "C", + "modeFrequency": 119, + "missingNumber": 0, + "sparseMissingNumber": 0, + "constructionCost": 4.27667, + "preparationCost": 30.8759, + "dataCost": 258.994, + "derivationRule": "TableMode(DNA, Char)" + }, + { + "rank": "R4", + "name": "Mean(DNA.Pos) where Char = G", + "type": "Numerical", + "level": 0.0274408, + "parts": 2, + "values": 281, + "min": 16, + "max": 46.55555556, + "mean": 31.84768883, + "stdDev": 4.848828493, + "missingNumber": 1, + "sparseMissingNumber": 0, + "constructionCost": 9.15447, + "preparationCost": 19.4458, + "dataCost": 271.302, + "derivationRule": "TableMean(`DNA where Char = G`, Pos)" + }, + { + "rank": "R5", + "name": "Median(DNA.Pos) where Char = G", + "type": "Numerical", + "level": 0.0225808, + "parts": 3, + "values": 59, + "min": 8, + "max": 48, + "mean": 32.44701987, + "stdDev": 6.725618787, + "missingNumber": 1, + "sparseMissingNumber": 0, + "constructionCost": 9.15447, + "preparationCost": 29.3732, + "dataCost": 262.873, + "derivationRule": "TableMedian(`DNA where Char = G`, Pos)" + }, + { + "rank": "R6", + "name": "Count(DNA) where Char = G", + "type": "Numerical", + "level": 0.0160697, + "parts": 2, + "values": 26, + "min": 0, + "max": 32, + "mean": 15.81188119, + "stdDev": 5.174919597, + "missingNumber": 0, + "sparseMissingNumber": 0, + "constructionCost": 9.15447, + "preparationCost": 21.0465, + "dataCost": 273.207, + "derivationRule": "TableCount(`DNA where Char = G`)" + }, + { + "rank": "R7", + "name": "Median(DNA.Pos) where Char = C", + "type": "Numerical", + "level": 0.0101423, + "parts": 2, + "values": 63, + "min": 9, + "max": 56.5, + "mean": 29.24587459, + "stdDev": 7.66919868, + "missingNumber": 0, + "sparseMissingNumber": 0, + "constructionCost": 9.15447, + "preparationCost": 19.4456, + "dataCost": 276.636, + "derivationRule": "TableMedian(`DNA where Char = C`, Pos)" + }, + { + "rank": "R8", + "name": "SampleId", + "type": "Categorical", + "level": 0, + "parts": 1, + "values": 303, + "mode": "AGMRSKPNI-NEG-1141", + "modeFrequency": 1, + "missingNumber": 0, + "sparseMissingNumber": 0, + "constructionCost": 0.693147, + "preparationCost": 10.7442, + "dataCost": 296.926 + } + ], + "variablesDetailedStatistics": { + "R1": { + "dataGrid": { + "isSupervised": true, + "dimensions": [ + { + "variable": "Mode(DNA.Char) where Pos <= 30.5", + "type": "Categorical", + "partitionType": "Value groups", + "partition": [ + ["C","T"], + ["G","A"] + ], + "defaultGroupIndex": 1 + }, + { + "variable": "Class", + "type": "Categorical", + "partitionType": "Values", + "partition": ["EI","IE","N"] + } + ], + "partTargetFrequencies": [ + [29,69,88], + [38,0,79] + ], + "partInterests": [0.247927,0.752073] + }, + "inputValues": { + "values": ["C","T","G","A"], + "frequencies": [122,64,61,56] + } + }, + "R2": { + "dataGrid": { + "isSupervised": true, + "dimensions": [ + { + "variable": "Mode(DNA.Char) where Pos in ]15.5, 30.5]", + "type": "Categorical", + "partitionType": "Value groups", + "partition": [ + ["A","G"], + ["C"], + ["T"] + ], + "defaultGroupIndex": 0 + }, + { + "variable": "Class", + "type": "Categorical", + "partitionType": "Values", + "partition": ["EI","IE","N"] + } + ], + "partTargetFrequencies": [ + [38,3,87], + [26,41,47], + [3,25,33] + ], + "partInterests": [0.593857,0.15488,0.251263] + }, + "inputValues": { + "values": ["C","A","T","G"], + "frequencies": [114,69,61,59] + } + }, + "R3": { + "dataGrid": { + "isSupervised": true, + "dimensions": [ + { + "variable": "Mode(DNA.Char)", + "type": "Categorical", + "partitionType": "Value groups", + "partition": [ + ["C"], + ["T","A"], + ["G"] + ], + "defaultGroupIndex": 1 + }, + { + "variable": "Class", + "type": "Categorical", + "partitionType": "Values", + "partition": ["EI","IE","N"] + } + ], + "partTargetFrequencies": [ + [17,49,53], + [16,17,74], + [34,3,40] + ], + "partInterests": [0.33703,0.144491,0.518478] + }, + "inputValues": { + "values": ["C","G","T","A"], + "frequencies": [119,77,57,50] + } + }, + "R4": { + "dataGrid": { + "isSupervised": true, + "dimensions": [ + { + "variable": "Mean(DNA.Pos) where Char = G", + "type": "Numerical", + "partitionType": "Intervals", + "partition": [ + [16,31.698], + [31.698,46.55555556] + ] + }, + { + "variable": "Class", + "type": "Categorical", + "partitionType": "Values", + "partition": ["EI","IE","N"] + } + ], + "partTargetFrequencies": [ + [34,12,105], + [33,57,62] + ], + "partInterests": [0.57478,0.42522] + } + }, + "R5": { + "dataGrid": { + "isSupervised": true, + "dimensions": [ + { + "variable": "Median(DNA.Pos) where Char = G", + "type": "Numerical", + "partitionType": "Intervals", + "partition": [ + [8,27.75], + [27.75,37.25], + [37.25,48] + ] + }, + { + "variable": "Class", + "type": "Categorical", + "partitionType": "Values", + "partition": ["EI","IE","N"] + } + ], + "partTargetFrequencies": [ + [11,3,58], + [44,31,86], + [12,35,23] + ], + "partInterests": [0.478566,0.0530889,0.468345] + } + }, + "R6": { + "dataGrid": { + "isSupervised": true, + "dimensions": [ + { + "variable": "Count(DNA) where Char = G", + "type": "Numerical", + "partitionType": "Intervals", + "partition": [ + [0,19.5], + [19.5,32] + ] + }, + { + "variable": "Class", + "type": "Categorical", + "partitionType": "Values", + "partition": ["EI","IE","N"] + } + ], + "partTargetFrequencies": [ + [38,68,130], + [29,1,37] + ], + "partInterests": [0.187162,0.812838] + } + }, + "R7": { + "dataGrid": { + "isSupervised": true, + "dimensions": [ + { + "variable": "Median(DNA.Pos) where Char = C", + "type": "Numerical", + "partitionType": "Intervals", + "partition": [ + [9,27.75], + [27.75,56.5] + ] + }, + { + "variable": "Class", + "type": "Categorical", + "partitionType": "Values", + "partition": ["EI","IE","N"] + } + ], + "partTargetFrequencies": [ + [35,53,62], + [32,16,105] + ], + "partInterests": [0.454456,0.545544] + } + } + } + }, + "khiops_encoding": "ascii" +} diff --git a/test/LearningTest/TestKhiops/Standard/JsonSpliceJunction/results.ref/C100_AnalysisResults.model.kdic b/test/LearningTest/TestKhiops/Standard/JsonSpliceJunction/results.ref/C100_AnalysisResults.model.kdic new file mode 100644 index 00000000..09ebaa26 --- /dev/null +++ b/test/LearningTest/TestKhiops/Standard/JsonSpliceJunction/results.ref/C100_AnalysisResults.model.kdic @@ -0,0 +1,48 @@ +#Khiops 10.5.5-b.0 + +Root Dictionary SNB_SpliceJunction (SampleId) + +{ + Categorical SampleId ; +Unused Categorical Class ; +Unused Table(SNB_SpliceJunctionDNA) DNA ; +Unused Categorical `Mode(DNA.Char)` = TableMode(DNA, Char) ; +Unused Categorical `Mode(DNA.Char) where Pos <= 30.5` = TableMode(`DNA where Pos <= 30.5`, Char) ; +Unused Numerical `Count(DNA) where Char = G` = TableCount(`DNA where Char = G`) ; +Unused Numerical `Mean(DNA.Pos) where Char = G` = TableMean(`DNA where Char = G`, Pos) ; +Unused Numerical `Median(DNA.Pos) where Char = C` = TableMedian(`DNA where Char = C`, Pos) ; +Unused Numerical `Median(DNA.Pos) where Char = G` = TableMedian(`DNA where Char = G`, Pos) ; +Unused Table(SNB_SpliceJunctionDNA) `DNA where Pos <= 30.5` = TableSelection(DNA, LE(Pos, 30.5)) ; +Unused Table(SNB_SpliceJunctionDNA) `DNA where Char = C` = TableSelection(DNA, EQc(Char, "C")) ; +Unused Table(SNB_SpliceJunctionDNA) `DNA where Char = G` = TableSelection(DNA, EQc(Char, "G")) ; + { +Unused Categorical `Mode(DNA.Char) where Pos in ]15.5, 30.5]` ; + } `Mode(DNA.Char) per Pos(4)` = TablePartitionMode(`TablePartition(DNA) per Pos(4)`, Char) ; +Unused Structure(Partition) `Partition(SpliceJunctionDNA) per Pos(4)` = Partition(IntervalBounds(15.5, 30.5, 45.5)) ; + { +Unused Table(SNB_SpliceJunctionDNA) `DNA where Pos in ]15.5, 30.5]` ; + } `TablePartition(DNA) per Pos(4)` = TablePartition(DNA, `Partition(SpliceJunctionDNA) per Pos(4)`, Pos) ; +Unused Structure(DataGrid) VClass = DataGrid(ValueSetC("EI", "IE", "N"), Frequencies(67, 69, 167)) ; +Unused Structure(DataGrid) `PMode(DNA.Char)` = DataGrid(ValueGroups(ValueGroup("C"), ValueGroup("T", "A", " * "), ValueGroup("G")), ValueSetC("EI", "IE", "N"), Frequencies(17, 16, 34, 49, 17, 3, 53, 74, 40)) ; // DataGrid(`Mode(DNA.Char)`, Class) +Unused Structure(DataGrid) `PMode(DNA.Char) where Pos <= 30.5` = DataGrid(ValueGroups(ValueGroup("C", "T"), ValueGroup("G", "A", " * ")), ValueSetC("EI", "IE", "N"), Frequencies(29, 38, 69, 0, 88, 79)) ; // DataGrid(`Mode(DNA.Char) where Pos <= 30.5`, Class) +Unused Structure(DataGrid) `PCount(DNA) where Char = G` = DataGrid(IntervalBounds(19.5), ValueSetC("EI", "IE", "N"), Frequencies(38, 29, 68, 1, 130, 37)) ; // DataGrid(`Count(DNA) where Char = G`, Class) +Unused Structure(DataGrid) `PMean(DNA.Pos) where Char = G` = DataGrid(IntervalBounds(31.698), ValueSetC("EI", "IE", "N"), Frequencies(34, 33, 12, 57, 105, 62)) ; // DataGrid(`Mean(DNA.Pos) where Char = G`, Class) +Unused Structure(DataGrid) `PMedian(DNA.Pos) where Char = C` = DataGrid(IntervalBounds(27.75), ValueSetC("EI", "IE", "N"), Frequencies(35, 32, 53, 16, 62, 105)) ; // DataGrid(`Median(DNA.Pos) where Char = C`, Class) +Unused Structure(DataGrid) `PMedian(DNA.Pos) where Char = G` = DataGrid(IntervalBounds(27.75, 37.25), ValueSetC("EI", "IE", "N"), Frequencies(11, 44, 12, 3, 31, 35, 58, 86, 23)) ; // DataGrid(`Median(DNA.Pos) where Char = G`, Class) +Unused Structure(DataGrid) `PMode(DNA.Char) where Pos in ]15.5, 30.5]` = DataGrid(ValueGroups(ValueGroup("A", "G", " * "), ValueGroup("C"), ValueGroup("T")), ValueSetC("EI", "IE", "N"), Frequencies(38, 26, 3, 3, 41, 25, 87, 47, 33)) ; // DataGrid(`Mode(DNA.Char) where Pos in ]15.5, 30.5]`, Class) +Unused Structure(DataGridBlock) `PBMode(DNA.Char) per Pos(4)` = DataGridBlock(ValueSet(2), `PMode(DNA.Char) where Pos in ]15.5, 30.5]`) ; +Unused Structure(Classifier) SNBClass = SNBClassifier(Vector(0.47265625, 0.36328125, 0.33203125, 0.28515625, 0.30078125, 0.43359375, 0.46484375), DataGridStats(`PMode(DNA.Char)`, `Mode(DNA.Char)`), DataGridStats(`PMode(DNA.Char) where Pos <= 30.5`, `Mode(DNA.Char) where Pos <= 30.5`), DataGridStats(`PCount(DNA) where Char = G`, `Count(DNA) where Char = G`), DataGridStats(`PMean(DNA.Pos) where Char = G`, `Mean(DNA.Pos) where Char = G`), DataGridStats(`PMedian(DNA.Pos) where Char = C`, `Median(DNA.Pos) where Char = C`), DataGridStats(`PMedian(DNA.Pos) where Char = G`, `Median(DNA.Pos) where Char = G`), DataGridStatsBlock(`PBMode(DNA.Char) per Pos(4)`, `Mode(DNA.Char) per Pos(4)`), VClass) ; + Categorical PredictedClass = TargetValue(SNBClass) ; +Unused Numerical ScoreClass = TargetProb(SNBClass) ; + Numerical ProbClassEI = TargetProbAt(SNBClass, "EI") ; + Numerical ProbClassIE = TargetProbAt(SNBClass, "IE") ; + Numerical ProbClassN = TargetProbAt(SNBClass, "N") ; +}; + +Dictionary SNB_SpliceJunctionDNA (SampleId) + +{ + Categorical SampleId ; + Numerical Pos ; + Categorical Char ; +}; diff --git a/test/LearningTest/TestKhiops/Standard/JsonSpliceJunction/results.ref/err.txt b/test/LearningTest/TestKhiops/Standard/JsonSpliceJunction/results.ref/err.txt new file mode 100644 index 00000000..e01004b0 --- /dev/null +++ b/test/LearningTest/TestKhiops/Standard/JsonSpliceJunction/results.ref/err.txt @@ -0,0 +1,11 @@ +Variable construction time: 0:00:00.06 (100 constructed variables) +Train supervised model for classification of target variable Class +Database ../../../MTdatasets/SpliceJunction/SpliceJunction.txt: Read records: 3178 Selected records: 303 + Table ../../../MTdatasets/SpliceJunction/SpliceJunctionDNA.txt Records: 18,180 +Data preparation time: 0:00:00.10 +SNB train time: 0:00:00 +Build Selective Naive Bayes SNB_SpliceJunction +Train evaluation of Selective Naive Bayes on database ../../../MTdatasets/SpliceJunction/SpliceJunction.txt +Test evaluation of Selective Naive Bayes on database ../../../MTdatasets/SpliceJunction/SpliceJunction.txt +Write report ./results/C100_AnalysisResults.khj + diff --git a/test/LearningTest/TestKhiops/Standard/JsonSpliceJunction/results.ref/time.log b/test/LearningTest/TestKhiops/Standard/JsonSpliceJunction/results.ref/time.log new file mode 100644 index 00000000..c986520b --- /dev/null +++ b/test/LearningTest/TestKhiops/Standard/JsonSpliceJunction/results.ref/time.log @@ -0,0 +1 @@ +0.4199361801147461 diff --git a/test/LearningTest/TestKhiops/Standard/JsonSpliceJunction/test.json b/test/LearningTest/TestKhiops/Standard/JsonSpliceJunction/test.json new file mode 100644 index 00000000..2c32dcc2 --- /dev/null +++ b/test/LearningTest/TestKhiops/Standard/JsonSpliceJunction/test.json @@ -0,0 +1,18 @@ +{ + "dictionaryFile": "../../../MTdatasets/SpliceJunction/SpliceJunction.kdic", + "dictionaryName": "SpliceJunction", + "dataTables": [ + { + "dataPath": "", + "dataFile": "../../../MTdatasets/SpliceJunction/SpliceJunction.txt" + }, + { + "dataPath": "DNA", + "dataFile": "../../../MTdatasets/SpliceJunction/SpliceJunctionDNA.txt" + } + ], + "detectFormat": false, + "trainPercentage": 10, + "maxConstructedAttributeNumber": 100, + "targetName": "Class" +} \ No newline at end of file diff --git a/test/LearningTest/TestKhiops/Standard/JsonSpliceJunction/test.prm b/test/LearningTest/TestKhiops/Standard/JsonSpliceJunction/test.prm new file mode 100644 index 00000000..16b54f0a --- /dev/null +++ b/test/LearningTest/TestKhiops/Standard/JsonSpliceJunction/test.prm @@ -0,0 +1,48 @@ +// 2024-10-22 13:59:12 +// khiops +// Output command file + +// This file contains recorded commands, that can be replayed. +// Commands are based on user interactions: +// field update +// list item selection +// menu action +// Every command can be commented, using . +// For example, commenting the last Exit command will allow other +// user interactions, after the commands have been replayed. + + +// -> Khiops +ClassManagement.OpenFile // Open... + +// -> Open +ClassFileName __dictionaryFile__ // Dictionary file +OK // Open +// <- Open + +TrainDatabase.ClassName __dictionaryName__ // Analysis dictionary + +LOOP __dataTables__ +TrainDatabase.DatabaseSpec.Data.DatabaseFiles.List.Key __dataPath__ // List item selection +TrainDatabase.DatabaseSpec.Data.DatabaseFiles.DataTableName __dataFile__ // Data table file +END LOOP + +TrainDatabase.DatabaseSpec.Sampling.SampleNumberPercentage __trainPercentage__ // Sample percentage + +IF __detectFormat__ +TrainDatabase.DatabaseSpec.Data.DatabaseFormatDetector.DetectFileFormat // Detect file format +END IF + +AnalysisSpec.TargetAttributeName __targetName__ // Target variable +AnalysisSpec.PredictorsSpec.ConstructionSpec.MaxConstructedAttributeNumber __maxConstructedAttributeNumber__ // Max number of constructed variables +AnalysisSpec.PredictorsSpec.ConstructionSpec.MaxTextFeatureNumber 0 // Max number of text features +AnalysisSpec.PredictorsSpec.ConstructionSpec.MaxTreeNumber 0 // Max Tree number +AnalysisResults.ReportFileName ./results/C__maxConstructedAttributeNumber___AnalysisResults.khj // Analysis report +ComputeStats // Train model +Exit // Close +// <- Khiops + + +// -> Khiops +OK // Yes +// <- Khiops diff --git a/test/LearningTestTool/README.md b/test/LearningTestTool/README.md index d74144d0..4c715b3d 100644 --- a/test/LearningTestTool/README.md +++ b/test/LearningTestTool/README.md @@ -130,7 +130,7 @@ Available LearningTest commands are - processes - forced-platform: for the context of reference results - min-test-time, max-test-time, test-timeout-limit - - user-interface, task-file, output-scenario + - user-interface, task-file, output-scenario, nop-output-scenario - **_kht_apply_** (LearningTest source dir) (instruction) [options] - apply an instruction on a subset of test dirs - options: diff --git a/test/LearningTestTool/py/_kht_results_management.py b/test/LearningTestTool/py/_kht_results_management.py index ca7ef853..3159fb86 100644 --- a/test/LearningTestTool/py/_kht_results_management.py +++ b/test/LearningTestTool/py/_kht_results_management.py @@ -248,7 +248,7 @@ def get_candidate_results_ref_dirs(test_dir): assert os.path.isdir(file_path), ( "file name starting " + kht.RESULTS_REF - + " shoud be a directory (" + + " should be a directory (" + file_path + ")" ) diff --git a/test/LearningTestTool/py/kht_test.py b/test/LearningTestTool/py/kht_test.py index 03729b41..d0278ba5 100644 --- a/test/LearningTestTool/py/kht_test.py +++ b/test/LearningTestTool/py/kht_test.py @@ -199,6 +199,7 @@ def evaluate_tool_on_test_dir( test_timeout_limit=None, task_file=False, output_scenario=False, + nop_output_scenario=False, user_interface=False, ): """Evaluation d'un outil sur un repertoire de test terminal et comparaison des resultats @@ -389,6 +390,9 @@ def evaluate_tool_on_test_dir( if output_scenario: khiops_params.append("-o") khiops_params.append(os.path.join(results_dir, "output_test.prm")) + if nop_output_scenario: + khiops_params.append("-O") + khiops_params.append(os.path.join(results_dir, "nop_output_test.prm")) if task_file: khiops_params.append("-p") khiops_params.append(os.path.join(results_dir, "task_progression.log")) @@ -908,6 +912,13 @@ def build_usage_help( action="store_true", ) + # Mode avec scenario en sortie + parser.add_argument( + "--nop-output-scenario", + help="create an output scenario nop_output_test.prm in results dir, without replaying commands", + action="store_true", + ) + # Mode interface utilisateur parser.add_argument( "--user-interface", @@ -986,6 +997,7 @@ def build_usage_help( test_timeout_limit=args.test_timeout_limit, task_file=args.task_file, output_scenario=args.output_scenario, + nop_output_scenario=args.nop_output_scenario, user_interface=args.user_interface, ) diff --git a/test/UnitTests/Norm/results.ref/base_TextService.txt b/test/UnitTests/Norm/results.ref/base_TextService.txt index 78119af7..7a1c324d 100644 --- a/test/UnitTests/Norm/results.ref/base_TextService.txt +++ b/test/UnitTests/Norm/results.ref/base_TextService.txt @@ -1,4 +1,4 @@ -Json string aa\u221Ebb\u00e9cc\/\\dd\tee aa?bbécc/\dd ee Index Char Unicode Utf8 Ansi code Utf8 code Ansi code from utf8 Valid +Json string aa\u221Ebb\u00e9cc\/\\dd\tee aa∞bbécc/\dd ee Index Char Unicode Utf8 Ansi code Utf8 code Ansi code from utf8 Valid 0 0000 00 0 0 0 1 1 0001 01 1 1 1 1 2 0002 02 2 2 2 1 @@ -280,4 +280,4 @@ Json string aa\u221Ebb\u00e9cc\/\\dd\tee aa?bb 22 ??? (â„¢) E284A2 (base64: 4oSi) 23 ?? (Ëœ) CB9C (base64: y5w=) 24 ???? (ð’€€) F0928080 (base64: 8JKAgA==) -SYS TIME Word encoding/decoding time for 1000000 values: 0.338258 +SYS TIME Word encoding/decoding time for 1000000 values: 0.358896