bumped simplecpp to 1.1.5 / improved testing (and output) / added TOD…

…Os (#6767)

.github/workflows/CI-unixish.yml

@@ -462,6 +462,7 @@ jobs:
       # TODO: move to scriptcheck.yml so these are tested with all Python versions?
       - name: Test addons
         run: |
+          set -x
           ./cppcheck --error-exitcode=1 --inline-suppr --addon=threadsafety addons/test/threadsafety
           ./cppcheck --error-exitcode=1 --inline-suppr --addon=threadsafety --std=c++03 addons/test/threadsafety
           ./cppcheck --error-exitcode=1 --inline-suppr --addon=misra addons/test/misra/crash*.c

.github/workflows/CI-windows.yml

@@ -194,6 +194,7 @@ jobs:
       - name: Test addons
         if: matrix.config == 'release'
         run: |
+          echo on
           .\cppcheck --addon=threadsafety addons\test\threadsafety || exit /b !errorlevel!
           .\cppcheck --addon=threadsafety --std=c++03 addons\test\threadsafety || exit /b !errorlevel!
           .\cppcheck --addon=misra --enable=style --inline-suppr --enable=information --error-exitcode=1 addons\test\misra\misra-ctu-*-test.c || exit /b !errorlevel!

externals/simplecpp/simplecpp.cpp

@@ -22,6 +22,7 @@
 #include <exception>
 #include <fstream>
 #include <iostream>
+#include <istream>
 #include <limits>
 #include <list>
 #include <map>
@@ -377,6 +378,42 @@ class StdIStream : public simplecpp::TokenList::Stream {
     std::istream &istr;
 };
 
+class StdCharBufStream : public simplecpp::TokenList::Stream {
+public:
+    // cppcheck-suppress uninitDerivedMemberVar - we call Stream::init() to initialize the private members
+    StdCharBufStream(const unsigned char* str, std::size_t size)
+        : str(str)
+        , size(size)
+        , pos(0)
+        , lastStatus(0)
+    {
+        init();
+    }
+
+    virtual int get() OVERRIDE {
+        if (pos >= size)
+            return lastStatus = EOF;
+        return str[pos++];
+    }
+    virtual int peek() OVERRIDE {
+        if (pos >= size)
+            return lastStatus = EOF;
+        return str[pos];
+    }
+    virtual void unget() OVERRIDE {
+        --pos;
+    }
+    virtual bool good() OVERRIDE {
+        return lastStatus != EOF;
+    }
+
+private:
+    const unsigned char *str;
+    const std::size_t size;
+    std::size_t pos;
+    int lastStatus;
+};
+
 class FileStream : public simplecpp::TokenList::Stream {
 public:
     // cppcheck-suppress uninitDerivedMemberVar - we call Stream::init() to initialize the private members
@@ -442,6 +479,20 @@ simplecpp::TokenList::TokenList(std::istream &istr, std::vector<std::string> &fi
     readfile(stream,filename,outputList);
 }
 
+simplecpp::TokenList::TokenList(const unsigned char* data, std::size_t size, std::vector<std::string> &filenames, const std::string &filename, OutputList *outputList)
+    : frontToken(nullptr), backToken(nullptr), files(filenames)
+{
+    StdCharBufStream stream(data, size);
+    readfile(stream,filename,outputList);
+}
+
+simplecpp::TokenList::TokenList(const char* data, std::size_t size, std::vector<std::string> &filenames, const std::string &filename, OutputList *outputList)
+        : frontToken(nullptr), backToken(nullptr), files(filenames)
+{
+    StdCharBufStream stream(reinterpret_cast<const unsigned char*>(data), size);
+    readfile(stream,filename,outputList);
+}
+
 simplecpp::TokenList::TokenList(const std::string &filename, std::vector<std::string> &filenames, OutputList *outputList)
         : frontToken(nullptr), backToken(nullptr), files(filenames)
 {
@@ -1447,8 +1498,7 @@ namespace simplecpp {
 
         Macro(const std::string &name, const std::string &value, std::vector<std::string> &f) : nameTokDef(nullptr), files(f), tokenListDefine(f), valueDefinedInCode_(false) {
             const std::string def(name + ' ' + value);
-            std::istringstream istr(def);
-            StdIStream stream(istr);
+            StdCharBufStream stream(reinterpret_cast<const unsigned char*>(def.data()), def.size());
             tokenListDefine.readfile(stream);
             if (!parseDefine(tokenListDefine.cfront()))
                 throw std::runtime_error("bad macro syntax. macroname=" + name + " value=" + value);
@@ -3310,11 +3360,24 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL
     macros.insert(std::make_pair("__TIME__", Macro("__TIME__", getTimeDefine(&ltime), dummy)));
 
     if (!dui.std.empty()) {
-        std::string std_def = simplecpp::getCStdString(dui.std);
-        if (!std_def.empty()) {
-            macros.insert(std::make_pair("__STDC_VERSION__", Macro("__STDC_VERSION__", std_def, dummy)));
+        const cstd_t c_std = simplecpp::getCStd(dui.std);
+        if (c_std != CUnknown) {
+            const std::string std_def = simplecpp::getCStdString(c_std);
+            if (!std_def.empty())
+                macros.insert(std::make_pair("__STDC_VERSION__", Macro("__STDC_VERSION__", std_def, dummy)));
         } else {
-            std_def = simplecpp::getCppStdString(dui.std);
+            const cppstd_t cpp_std = simplecpp::getCppStd(dui.std);
+            if (cpp_std == CPPUnknown) {
+                if (outputList) {
+                    simplecpp::Output err(files);
+                    err.type = Output::DUI_ERROR;
+                    err.msg = "unknown standard specified: '" + dui.std + "'";
+                    outputList->push_back(err);
+                }
+                output.clear();
+                return;
+            }
+            const std::string std_def = simplecpp::getCppStdString(cpp_std);
             if (!std_def.empty())
                 macros.insert(std::make_pair("__cplusplus", Macro("__cplusplus", std_def, dummy)));
         }
@@ -3463,7 +3526,8 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL
                     std::ifstream f;
                     header2 = openHeader(f, dui, rawtok->location.file(), header, systemheader);
                     if (f.is_open()) {
-                        TokenList * const tokens = new TokenList(f, files, header2, outputList);
+                        f.close();
+                        TokenList * const tokens = new TokenList(header2, files, outputList);
                         if (dui.removeComments)
                             tokens->removeComments();
                         filedata[header2] = tokens;
@@ -3726,56 +3790,105 @@ void simplecpp::cleanup(std::map<std::string, TokenList*> &filedata)
     filedata.clear();
 }
 
-std::string simplecpp::getCStdString(const std::string &std)
+simplecpp::cstd_t simplecpp::getCStd(const std::string &std)
 {
-    if (std == "c90" || std == "c89" || std == "iso9899:1990" || std == "iso9899:199409" || std == "gnu90" || std == "gnu89") {
-        // __STDC_VERSION__ is not set for C90 although the macro was added in the 1994 amendments
-        return "";
-    }
+    if (std == "c90" || std == "c89" || std == "iso9899:1990" || std == "iso9899:199409" || std == "gnu90" || std == "gnu89")
+        return C89;
     if (std == "c99" || std == "c9x" || std == "iso9899:1999" || std == "iso9899:199x" || std == "gnu99"|| std == "gnu9x")
-        return "199901L";
+        return C99;
     if (std == "c11" || std == "c1x" || std == "iso9899:2011" || std == "gnu11" || std == "gnu1x")
-        return "201112L";
+        return C11;
     if (std == "c17" || std == "c18" || std == "iso9899:2017" || std == "iso9899:2018" || std == "gnu17"|| std == "gnu18")
-        return "201710L";
-    if (std == "c23" || std == "gnu23" || std == "c2x" || std == "gnu2x") {
-        // supported by GCC 9+ and Clang 9+
-        // Clang 9, 10, 11, 12, 13 return "201710L"
-        // Clang 14, 15, 16, 17 return "202000L"
-        // Clang 9, 10, 11, 12, 13, 14, 15, 16, 17 do not support "c23" and "gnu23"
-        return "202311L";
+        return C17;
+    if (std == "c23" || std == "gnu23" || std == "c2x" || std == "gnu2x")
+        return C23;
+    return CUnknown;
+}
+
+std::string simplecpp::getCStdString(cstd_t std)
+{
+    switch (std)
+    {
+        case C89:
+            // __STDC_VERSION__ is not set for C90 although the macro was added in the 1994 amendments
+            return "";
+        case C99:
+            return "199901L";
+        case C11:
+            return "201112L";
+        case C17:
+            return "201710L";
+        case C23:
+            // supported by GCC 9+ and Clang 9+
+            // Clang 9, 10, 11, 12, 13 return "201710L"
+            // Clang 14, 15, 16, 17 return "202000L"
+            // Clang 9, 10, 11, 12, 13, 14, 15, 16, 17 do not support "c23" and "gnu23"
+            return "202311L";
+        case CUnknown:
+            return "";
     }
     return "";
 }
 
-std::string simplecpp::getCppStdString(const std::string &std)
+std::string simplecpp::getCStdString(const std::string &std)
+{
+    return getCStdString(getCStd(std));
+}
+
+simplecpp::cppstd_t simplecpp::getCppStd(const std::string &std)
 {
     if (std == "c++98" || std == "c++03" || std == "gnu++98" || std == "gnu++03")
-        return "199711L";
+        return CPP03;
     if (std == "c++11" || std == "gnu++11" || std == "c++0x" || std == "gnu++0x")
-        return "201103L";
+        return CPP11;
     if (std == "c++14" || std == "c++1y" || std == "gnu++14" || std == "gnu++1y")
-        return "201402L";
+        return CPP14;
     if (std == "c++17" || std == "c++1z" || std == "gnu++17" || std == "gnu++1z")
-        return "201703L";
-    if (std == "c++20" || std == "c++2a" || std == "gnu++20" || std == "gnu++2a") {
-        // GCC 10 returns "201703L" - correct in 11+
-        return "202002L";
-    }
-    if (std == "c++23" || std == "c++2b" || std == "gnu++23" || std == "gnu++2b") {
-        // supported by GCC 11+ and Clang 12+
-        // GCC 11, 12, 13 return "202100L"
-        // Clang 12, 13, 14, 15, 16 do not support "c++23" and "gnu++23" and return "202101L"
-        // Clang 17, 18 return "202302L"
-        return "202302L";
-    }
-    if (std == "c++26" || std == "c++2c" || std == "gnu++26" || std == "gnu++2c") {
-        // supported by Clang 17+
-        return "202400L";
+        return CPP17;
+    if (std == "c++20" || std == "c++2a" || std == "gnu++20" || std == "gnu++2a")
+        return CPP20;
+    if (std == "c++23" || std == "c++2b" || std == "gnu++23" || std == "gnu++2b")
+        return CPP23;
+    if (std == "c++26" || std == "c++2c" || std == "gnu++26" || std == "gnu++2c")
+        return CPP26;
+    return CPPUnknown;
+}
+
+std::string simplecpp::getCppStdString(cppstd_t std)
+{
+    switch (std)
+    {
+        case CPP03:
+            return "199711L";
+        case CPP11:
+            return "201103L";
+        case CPP14:
+            return "201402L";
+        case CPP17:
+            return "201703L";
+        case CPP20:
+            // GCC 10 returns "201703L" - correct in 11+
+            return "202002L";
+        case CPP23:
+            // supported by GCC 11+ and Clang 12+
+            // GCC 11, 12, 13 return "202100L"
+            // Clang 12, 13, 14, 15, 16 do not support "c++23" and "gnu++23" and return "202101L"
+            // Clang 17, 18 return "202302L"
+            return "202302L";
+        case CPP26:
+            // supported by Clang 17+
+            return "202400L";
+        case CPPUnknown:
+            return "";
     }
     return "";
 }
 
+std::string simplecpp::getCppStdString(const std::string &std)
+{
+    return getCppStdString(getCppStd(std));
+}
+
 #if (__cplusplus < 201103L) && !defined(__APPLE__)
 #undef nullptr
 #endif

externals/simplecpp/simplecpp.h

@@ -1,4 +1,4 @@
-/*
+/* -*- C++ -*-
  * simplecpp - A simple and high-fidelity C/C++ preprocessor library
  * Copyright (C) 2016-2023 simplecpp team
  */
@@ -8,7 +8,7 @@
 
 #include <cctype>
 #include <cstring>
-#include <istream>
+#include <iosfwd>
 #include <list>
 #include <map>
 #include <set>
@@ -39,6 +39,11 @@
 #endif
 
 namespace simplecpp {
+    /** C code standard */
+    enum cstd_t { CUnknown=-1, C89, C99, C11, C17, C23 };
+
+    /** C++ code standard */
+    enum cppstd_t { CPPUnknown=-1, CPP03, CPP11, CPP14, CPP17, CPP20, CPP23, CPP26 };
 
     typedef std::string TokenString;
     class Macro;
@@ -181,7 +186,8 @@ namespace simplecpp {
             PORTABILITY_BACKSLASH,
             UNHANDLED_CHAR_ERROR,
             EXPLICIT_INCLUDE_NOT_FOUND,
-            FILE_NOT_FOUND
+            FILE_NOT_FOUND,
+            DUI_ERROR
         } type;
         explicit Output(const std::vector<std::string>& files, Type type, const std::string& msg) : type(type), location(files), msg(msg) {}
         Location location;
@@ -198,6 +204,10 @@ namespace simplecpp {
         explicit TokenList(std::vector<std::string> &filenames);
         /** generates a token list from the given std::istream parameter */
         TokenList(std::istream &istr, std::vector<std::string> &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr);
+        /** generates a token list from the given buffer */
+        TokenList(const unsigned char* data, std::size_t size, std::vector<std::string> &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr);
+        /** generates a token list from the given buffer */
+        TokenList(const char* data, std::size_t size, std::vector<std::string> &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr);
         /** generates a token list from the given filename parameter */
         TokenList(const std::string &filename, std::vector<std::string> &filenames, OutputList *outputList = nullptr);
         TokenList(const TokenList &other);
@@ -363,11 +373,19 @@ namespace simplecpp {
     /** Convert Cygwin path to Windows path */
     SIMPLECPP_LIB std::string convertCygwinToWindowsPath(const std::string &cygwinPath);
 
+    /** Returns the C version a given standard */
+    SIMPLECPP_LIB cstd_t getCStd(const std::string &std);
+
+    /** Returns the C++ version a given standard */
+    SIMPLECPP_LIB cppstd_t getCppStd(const std::string &std);
+
     /** Returns the __STDC_VERSION__ value for a given standard */
     SIMPLECPP_LIB std::string getCStdString(const std::string &std);
+    SIMPLECPP_LIB std::string getCStdString(cstd_t std);
 
     /** Returns the __cplusplus value for a given standard */
     SIMPLECPP_LIB std::string getCppStdString(const std::string &std);
+    SIMPLECPP_LIB std::string getCppStdString(cppstd_t std);
 }
 
 #if defined(_MSC_VER)

lib/preprocessor.cpp

@@ -735,6 +735,7 @@ bool Preprocessor::hasErrors(const simplecpp::Output &output)
     case simplecpp::Output::UNHANDLED_CHAR_ERROR:
     case simplecpp::Output::EXPLICIT_INCLUDE_NOT_FOUND:
     case simplecpp::Output::FILE_NOT_FOUND:
+    case simplecpp::Output::DUI_ERROR:
         return true;
     case simplecpp::Output::WARNING:
     case simplecpp::Output::MISSING_HEADER:
@@ -875,6 +876,7 @@ void Preprocessor::reportOutput(const simplecpp::OutputList &outputList, bool sh
             break;
         case simplecpp::Output::EXPLICIT_INCLUDE_NOT_FOUND:
         case simplecpp::Output::FILE_NOT_FOUND:
+        case simplecpp::Output::DUI_ERROR:
             error(emptyString, 0, out.msg);
             break;
         }

lib/tokenlist.cpp

@@ -375,6 +375,8 @@ bool TokenList::createTokensInternal(std::istream &code, const std::string& file
 // NOLINTNEXTLINE(cppcoreguidelines-rvalue-reference-param-not-moved)
 void TokenList::createTokens(simplecpp::TokenList&& tokenList)
 {
+    // TODO: what to do if the list has been filled already? clear mTokensFrontBack?
+
     // tokenList.cfront() might be NULL if the file contained nothing to tokenize so we need to check the files instead
     if (!tokenList.getFiles().empty()) {
         // this is a copy

test/helpers.cpp

@@ -170,12 +170,15 @@ void PreprocessorHelper::preprocess(const char code[], std::vector<std::string>
 
 void PreprocessorHelper::preprocess(const char code[], std::vector<std::string> &files, Tokenizer& tokenizer, ErrorLogger& errorlogger, const simplecpp::DUI& dui)
 {
+    // TODO: make sure the given Tokenizer has not been used yet
+
     std::istringstream istr(code);
     const simplecpp::TokenList tokens1(istr, files, files[0]);
 
     // Preprocess..
     simplecpp::TokenList tokens2(files);
     std::map<std::string, simplecpp::TokenList*> filedata;
+    // TODO: provide and handle outputList
     simplecpp::preprocess(tokens2, tokens1, files, filedata, dui);
 
     // Tokenizer..