Merge pull request #1108 from AntelopeIO/GH-641-escape-opt

Optimization: New escape_str function for sanitizing strings

libraries/chain/apply_context.cpp

@@ -19,15 +19,26 @@ namespace eosio { namespace chain {
 
 static inline void print_debug(account_name receiver, const action_trace& ar) {
    if (!ar.console.empty()) {
-      auto prefix = fc::format_string(
-                                      "\n[(${a},${n})->${r}]",
-                                      fc::mutable_variant_object()
-                                      ("a", ar.act.account)
-                                      ("n", ar.act.name)
-                                      ("r", receiver));
-      dlog(prefix + ": CONSOLE OUTPUT BEGIN =====================\n"
-           + ar.console
-           + prefix + ": CONSOLE OUTPUT END   =====================" );
+      if (fc::logger::get(DEFAULT_LOGGER).is_enabled( fc::log_level::debug )) {
+         std::string prefix;
+         prefix.reserve(3 + 13 + 1 + 13 + 3 + 13 + 1);
+         prefix += "\n[(";
+         prefix += ar.act.account.to_string();
+         prefix += ",";
+         prefix += ar.act.name.to_string();
+         prefix += ")->";
+         prefix += receiver.to_string();
+         prefix += "]";
+
+         std::string output;
+         output.reserve(512);
+         output += prefix;
+         output += ": CONSOLE OUTPUT BEGIN =====================\n";
+         output += ar.console;
+         output += prefix;
+         output += ": CONSOLE OUTPUT END   =====================";
+         dlog( std::move(output) );
+      }
    }
 }
 

libraries/libfc/include/fc/string.hpp

@@ -1,5 +1,6 @@
 #pragma once
 #include <cstdint>
+#include <limits>
 #include <string>
 
 namespace fc
@@ -20,4 +21,23 @@ namespace fc
   class variant_object;
   std::string format_string( const std::string&, const variant_object&, bool minimize = false );
   std::string trim( const std::string& );
+
+  /**
+   * Convert '\t', '\r', '\n', '\\' and '"'  to "\t\r\n\\\"" if escape_ctrl == on
+   * Convert all other < 32 & 127 ascii to escaped unicode "\u00xx"
+   * Removes invalid utf8 characters
+   * Escapes Control sequence Introducer 0x9b to \u009b
+   * All other characters unmolested.
+   *
+   * @param str input/output string to escape/truncate
+   * @param escape_ctrl if on escapes control chars in str
+   * @param max_len truncate string to max_len
+   * @param add_truncate_str if truncated by max_len, add add_truncate_str to end of any truncated string,
+   *                         new length with be max_len + add_truncate_str.size()
+   * @return pair<reference to possibly modified passed in str, true if modified>
+   */
+  enum class escape_control_chars { off, on };
+  std::pair<std::string&, bool> escape_str( std::string& str, escape_control_chars escape_ctrl = escape_control_chars::on,
+                                            std::size_t max_len = std::numeric_limits<std::size_t>::max(),
+                                            std::string_view add_truncate_str = "..." );
 }

libraries/libfc/src/string.cpp

@@ -1,4 +1,6 @@
 #include <fc/string.hpp>
+#include <fc/utf8.hpp>
+#include <fc/io/json.hpp>
 #include <fc/exception/exception.hpp>
 #include <boost/lexical_cast.hpp>
 #include <boost/algorithm/string.hpp>
@@ -83,13 +85,42 @@ namespace fc  {
   std::string trim( const std::string& s )
   {
       return boost::algorithm::trim_copy(s);
-      /*
-      std::string cpy(s);
-      boost::algorithm::trim(cpy);
-      return cpy;
-      */
   }
 
+  std::pair<std::string&, bool> escape_str( std::string& str, escape_control_chars escape_ctrl,
+                                            std::size_t max_len, std::string_view add_truncate_str )
+  {
+     bool modified = false, truncated = false;
+     // truncate early to speed up escape
+     if (str.size() > max_len) {
+        str.resize(max_len);
+        modified = truncated = true;
+     }
+     auto itr = escape_ctrl == escape_control_chars::on
+           ? std::find_if(str.begin(), str.end(),
+                          [](const auto& c) {
+              return c == '\x7f' || c == '\\' || c == '\"' ||  (c >= '\x00' && c <= '\x1f'); } )
+           : std::find_if(str.begin(), str.end(),
+                          [](const auto& c) {               // x09 = \t, x0a = \n,                   x0d = \r
+              return c == '\x7f' || (c >= '\x00' && c <= '\x08') || c == '\x0b' || c == '\x0c' || (c >= '\x0e' && c <= '\x1f'); } );
+
+     if (itr != str.end() || !fc::is_valid_utf8( str )) {
+        str = escape_string(str, nullptr, escape_ctrl == escape_control_chars::on);
+        modified = true;
+        if (str.size() > max_len) {
+           str.resize(max_len);
+           truncated = true;
+        }
+     }
+
+     if (truncated) {
+        str += add_truncate_str;
+     }
+
+     return std::make_pair(std::ref(str), modified);
+  }
+
+
 } // namespace fc
 
 

libraries/libfc/test/CMakeLists.txt

@@ -14,6 +14,7 @@ add_executable( test_fc
         variant/test_variant.cpp
         variant_estimated_size/test_variant_estimated_size.cpp
         test_base64.cpp
+        test_escape_str.cpp
         main.cpp
         )
 target_link_libraries( test_fc fc )

libraries/libfc/test/test_escape_str.cpp

@@ -0,0 +1,98 @@
+#include <boost/test/unit_test.hpp>
+
+#include <fc/string.hpp>
+#include <fc/exception/exception.hpp>
+
+using namespace fc;
+using namespace std::literals;
+
+BOOST_AUTO_TEST_SUITE(escape_str_test)
+
+BOOST_AUTO_TEST_CASE(escape_control_chars) try {
+   const std::string escape_input_str = "\b\f\n\r\t\\\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+                                        "\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f";
+   std::string escaped_str = "\\u0008\\u000c\\n\\r\\t\\\\\\u0001\\u0002\\u0003\\u0004\\u0005\\u0006\\u0007\\u0008\\t\\n\\u000b\\u000c\\r\\u000e\\u000f"
+                             "\\u0010\\u0011\\u0012\\u0013\\u0014\\u0015\\u0016\\u0017\\u0018\\u0019\\u001a\\u001b\\u001c\\u001d\\u001e\\u001f";
+
+   std::string input = escape_input_str;
+   BOOST_CHECK_EQUAL(escape_str(input).first, escaped_str);
+
+   input = escape_input_str;
+   escaped_str = "\\u0008\\u000c\n"
+                 "\r\t\\\\u0001\\u0002\\u0003\\u0004\\u0005\\u0006\\u0007\\u0008\t\n"
+                 "\\u000b\\u000c\r\\u000e\\u000f\\u0010\\u0011\\u0012\\u0013\\u0014\\u0015\\u0016\\u0017\\u0018\\u0019\\u001a\\u001b\\u001c\\u001d\\u001e\\u001f";
+   BOOST_CHECK_EQUAL(escape_str(input, fc::escape_control_chars::off).first, escaped_str);
+
+} FC_LOG_AND_RETHROW();
+
+BOOST_AUTO_TEST_CASE(empty) try {
+   std::string input;
+   BOOST_CHECK_EQUAL(escape_str(input, fc::escape_control_chars::on, 256, "").first, "");
+
+   input = "";
+   BOOST_CHECK_EQUAL(escape_str(input, fc::escape_control_chars::off, 512, {}).first, "");
+} FC_LOG_AND_RETHROW();
+
+BOOST_AUTO_TEST_CASE(truncate) try {
+   const std::string repeat_512_chars(512, 'a');
+   const std::string repeat_256_chars(256, 'a');
+
+   std::string input = repeat_512_chars;
+   BOOST_CHECK_EQUAL(escape_str(input, fc::escape_control_chars::on, 256, "").first, repeat_256_chars);
+
+   input = repeat_512_chars;
+   BOOST_CHECK_EQUAL(escape_str(input, fc::escape_control_chars::on, 256, {}).first, repeat_256_chars);
+
+   input = repeat_512_chars;
+   BOOST_CHECK_EQUAL(escape_str(input, fc::escape_control_chars::on, 256).first, repeat_256_chars + "...");
+
+   input = repeat_512_chars;
+   BOOST_CHECK_EQUAL(escape_str(input, fc::escape_control_chars::on, 256, "<-the end->").first, repeat_256_chars + "<-the end->");
+} FC_LOG_AND_RETHROW();
+
+BOOST_AUTO_TEST_CASE(modify) try {
+   const std::string repeat_512_chars(512, 'a');
+   const std::string repeat_256_chars(256, 'a');
+
+   std::string input = repeat_512_chars;
+   BOOST_CHECK(escape_str(input, fc::escape_control_chars::on, 256, "").second);
+
+   input = repeat_512_chars;
+   BOOST_CHECK(escape_str(input, fc::escape_control_chars::on, 256, {}).second);
+
+   input = repeat_512_chars;
+   BOOST_CHECK(escape_str(input, fc::escape_control_chars::on, 256).second);
+
+   input = repeat_512_chars;
+   BOOST_CHECK(!escape_str(input, fc::escape_control_chars::on, 512).second);
+
+   input = repeat_512_chars;
+   BOOST_CHECK(!escape_str(input, fc::escape_control_chars::on).second);
+
+   input = repeat_512_chars;
+   BOOST_CHECK(!escape_str(input, fc::escape_control_chars::on, 1024).second);
+
+   input = "";
+   BOOST_CHECK(!escape_str(input, fc::escape_control_chars::on, 1024).second);
+
+   input = "hello";
+   BOOST_CHECK(!escape_str(input, fc::escape_control_chars::on, 1024).second);
+
+   input = "\n";
+   BOOST_CHECK(escape_str(input, fc::escape_control_chars::on, 1024).second);
+
+   input ="\xb4";
+   BOOST_CHECK(escape_str(input, fc::escape_control_chars::on, 1024).second);
+   BOOST_CHECK_EQUAL(input, "");
+
+} FC_LOG_AND_RETHROW();
+
+BOOST_AUTO_TEST_CASE(remove_invalid_utf8) try {
+   auto input = "abc123$&()'?\xb4\xf5\x01\xfa~a"s; // remove invalid utf8 values, \x01 => \u0001
+   auto expected_output = "abc123$&()'?\\u0001~a"s;
+
+   BOOST_CHECK_EQUAL(escape_str(input).first, expected_output);
+} FC_LOG_AND_RETHROW();
+
+
+BOOST_AUTO_TEST_SUITE_END()

plugins/producer_plugin/producer_plugin.cpp

@@ -1984,23 +1984,21 @@ inline std::string get_detailed_contract_except_info(const packed_transaction_pt
 {
    std::string contract_name;
    std::string act_name;
-   std::string details;
-
    if( trace && !trace->action_traces.empty() ) {
       auto last_action_ordinal = trace->action_traces.size() - 1;
       contract_name = trace->action_traces[last_action_ordinal].receiver.to_string();
       act_name = trace->action_traces[last_action_ordinal].act.name.to_string();
    } else if ( trx ) {
       const auto& actions = trx->get_transaction().actions;
-      if( actions.empty() ) return details; // should not be possible
+      if( actions.empty() ) return {}; // should not be possible
       contract_name = actions[0].account.to_string();
       act_name = actions[0].name.to_string();
    }
 
-   details = except_ptr ? except_ptr->top_message() : (trace && trace->except) ? trace->except->top_message() : std::string();
-   if (!details.empty()) {
-      details = fc::format_string("${d}", fc::mutable_variant_object() ("d", details), true);  // true for limiting the formatted string size
-   }
+   std::string details = except_ptr ? except_ptr->top_message()
+                                    : ((trace && trace->except) ? trace->except->top_message()
+                                                                : std::string());
+   fc::escape_str(details, fc::escape_control_chars::on, 1024);
 
    // this format is parsed by external tools
    return "action: " + contract_name + ":" + act_name + ", " + details;