Merge pull request #1893 from AntelopeIO/GH-1461-base64-pad-main

[5.0 -> main] Fix base64 encoding - take 2

libraries/chain/include/eosio/chain/database_utils.hpp

@@ -182,8 +182,8 @@ namespace fc {
 
    inline
    void from_variant( const variant& v, eosio::chain::shared_blob& b ) {
-      std::string _s = base64_decode(v.as_string());
-      b = eosio::chain::shared_blob(_s.begin(), _s.end(), b.get_allocator());
+      std::vector<char> b64 = base64_decode(v.as_string());
+      b = eosio::chain::shared_blob(b64.begin(), b64.end(), b.get_allocator());
    }
 
    template<typename T>

libraries/libfc/include/fc/crypto/base64.hpp

@@ -1,14 +1,16 @@
 #pragma once
 #include <string>
+#include <string_view>
+#include <vector>
 
 namespace fc {
 std::string base64_encode(unsigned char const* bytes_to_encode, unsigned int in_len);
 inline std::string base64_encode(char const* bytes_to_encode, unsigned int in_len) { return base64_encode( (unsigned char const*)bytes_to_encode, in_len); }
 std::string base64_encode( const std::string& enc );
-std::string base64_decode( const std::string& encoded_string);
+std::vector<char> base64_decode( std::string_view encoded_string);
 
 std::string base64url_encode(unsigned char const* bytes_to_encode, unsigned int in_len);
 inline std::string base64url_encode(char const* bytes_to_encode, unsigned int in_len) { return base64url_encode( (unsigned char const*)bytes_to_encode, in_len); }
 std::string base64url_encode( const std::string& enc );
-std::string base64url_decode( const std::string& encoded_string);
+std::vector<char> base64url_decode( std::string_view encoded_string);
 }  // namespace fc

libraries/libfc/src/crypto/base64.cpp

@@ -107,13 +107,14 @@ std::string base64url_encode( const std::string& enc ) {
   return base64url_encode( (unsigned char const*)s, enc.size() );
 }
 
-std::string base64_decode_impl(std::string const& encoded_string, const char* const b64_chars) {
+std::vector<char> base64_decode_impl(std::string_view encoded_string, const char* const b64_chars) {
   int in_len = encoded_string.size();
   int i = 0;
   int j = 0;
   int in_ = 0;
   unsigned char char_array_4[4], char_array_3[3];
-  std::string ret;
+  std::vector<char> ret;
+  ret.reserve(in_len / 4 * 3);
 
   while (in_len-- && encoded_string[in_] != '=') {
     throw_on_nonbase64(encoded_string[in_], b64_chars);
@@ -127,7 +128,7 @@ std::string base64_decode_impl(std::string const& encoded_string, const char* co
       char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3];
 
       for (i = 0; (i < 3); i++)
-        ret += char_array_3[i];
+        ret.push_back(char_array_3[i]);
       i = 0;
     }
   }
@@ -143,17 +144,17 @@ std::string base64_decode_impl(std::string const& encoded_string, const char* co
     char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2);
     char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3];
 
-    for (j = 0; (j < i - 1); j++) ret += char_array_3[j];
+    for (j = 0; (j < i - 1); j++) ret.push_back(char_array_3[j]);
   }
 
   return ret;
 }
 
-std::string base64_decode(std::string const& encoded_string) {
+std::vector<char> base64_decode(std::string_view encoded_string) {
    return base64_decode_impl(encoded_string, base64_chars);
 }
 
-std::string base64url_decode(std::string const& encoded_string) {
+std::vector<char> base64url_decode(std::string_view encoded_string) {
    return base64_decode_impl(encoded_string, base64url_chars);
 }
 

libraries/libfc/src/crypto/bigint.cpp

@@ -221,8 +221,8 @@ namespace fc {
     else
     {
         std::string b64 = v.as_string();
-        std::string bin = base64_decode(b64);
-        bi = bigint(bin.c_str(), bin.size() );
+        std::vector<char> bin = base64_decode(b64);
+        bi = bigint(bin.data(), bin.size() );
     }
   }
 

libraries/libfc/src/crypto/elliptic_webauthn.cpp

@@ -225,7 +225,7 @@ public_key::public_key(const signature& c, const fc::sha256& digest, bool) {
 
    FC_ASSERT(handler.found_type == "webauthn.get", "webauthn signature type not an assertion");
 
-   std::string challenge_bytes = fc::base64url_decode(handler.found_challenge);
+   std::vector<char> challenge_bytes = fc::base64url_decode(handler.found_challenge);
    FC_ASSERT(fc::sha256(challenge_bytes.data(), challenge_bytes.size()) == digest, "Wrong webauthn challenge");
 
    char required_origin_scheme[] = "https://";

libraries/libfc/src/variant.cpp

@@ -490,7 +490,7 @@ std::string variant::as_string()const
           return *reinterpret_cast<const bool*>(this) ? "true" : "false";
       case blob_type:
           if( get_blob().data.size() )
-             return base64_encode( get_blob().data.data(), get_blob().data.size() ) + "=";
+             return base64_encode( get_blob().data.data(), get_blob().data.size() );
           return std::string();
       case null_type:
           return std::string();
@@ -533,10 +533,14 @@ blob variant::as_blob()const
       {
          const std::string& str = get_string();
          if( str.size() == 0 ) return blob();
-         if( str.back() == '=' )
-         {
-            std::string b64 = base64_decode( get_string() );
-            return blob( { std::vector<char>( b64.begin(), b64.end() ) } );
+         try {
+            // pre-5.0 versions of variant added `=` to end of base64 encoded string in as_string() above.
+            // fc version of base64_decode allows for extra `=` at the end of the string.
+            // Other base64 decoders will not accept the extra `=`.
+            std::vector<char> b64 = base64_decode( str );
+            return { std::move(b64) };
+         } catch(const std::exception&) {
+            // unable to decode, return the raw chars
          }
          return blob( { std::vector<char>( str.begin(), str.end() ) } );
       }
@@ -758,8 +762,7 @@ void to_variant( const blob& b, variant& v ) {
 }
 
 void from_variant( const variant& v, blob& b ) {
-   std::string _s = base64_decode(v.as_string());
-   b.data = std::vector<char>(_s.begin(), _s.end());
+   b.data = base64_decode(v.as_string());
 }
 
 void to_variant( const UInt<8>& n, variant& v ) { v = uint64_t(n); }

libraries/libfc/test/test_base64.cpp

@@ -26,21 +26,24 @@ BOOST_AUTO_TEST_CASE(base64dec) try {
    auto input = "YWJjMTIzJCYoKSc/tPUB+n5h"s;
    auto expected_output = "abc123$&()'?\xb4\xf5\x01\xfa~a"s;
 
-   BOOST_CHECK_EQUAL(expected_output, base64_decode(input));
+   std::vector<char> b64 = base64_decode(input);
+   BOOST_CHECK_EQUAL(expected_output, std::string_view(b64.begin(), b64.end()));
 } FC_LOG_AND_RETHROW();
 
 BOOST_AUTO_TEST_CASE(base64urldec) try {
    auto input = "YWJjMTIzJCYoKSc_tPUB-n5h"s;
    auto expected_output = "abc123$&()'?\xb4\xf5\x01\xfa~a"s;
 
-   BOOST_CHECK_EQUAL(expected_output, base64url_decode(input));
+   std::vector<char> b64 = base64url_decode(input);
+   BOOST_CHECK_EQUAL(expected_output, std::string_view(b64.begin(), b64.end()));
 } FC_LOG_AND_RETHROW();
 
 BOOST_AUTO_TEST_CASE(base64dec_extraequals) try {
    auto input = "YWJjMTIzJCYoKSc/tPUB+n5h========="s;
    auto expected_output = "abc123$&()'?\xb4\xf5\x01\xfa~a"s;
 
-   BOOST_CHECK_EQUAL(expected_output, base64_decode(input));
+   std::vector<char> b64 = base64_decode(input);
+   BOOST_CHECK_EQUAL(expected_output, std::string_view(b64.begin(), b64.end()));
 } FC_LOG_AND_RETHROW();
 
 BOOST_AUTO_TEST_CASE(base64dec_bad_stuff) try {

libraries/libfc/test/variant/test_variant.cpp

@@ -88,11 +88,87 @@ BOOST_AUTO_TEST_CASE(variant_format_string_limited)
       const string target_result = format_prefix + a_short_list + " " +
                                    "{" + "\"b\":\"" + b_short_list + "\",\"c\":\"" + c_short_list + "\"}" + " " +
                                    "[\"" + d_short_list + "\",\"" + e_short_list + "\"]" + " " +
-                                   base64_encode( a_blob.data.data(), a_blob.data.size() ) + "=" + " " +
+                                   base64_encode( a_blob.data.data(), a_blob.data.size() ) + " " +
                                    g_short_list;
 
       BOOST_CHECK_EQUAL( result, target_result);
       BOOST_CHECK_LT(result.size(), 1024 + 3 * mu.size());
    }
 }
+
+BOOST_AUTO_TEST_CASE(variant_blob)
+{
+   // Some test cases from https://github.com/ReneNyffenegger/cpp-base64
+   {
+      std::string a17_orig = "aaaaaaaaaaaaaaaaa";
+      std::string a17_encoded = "YWFhYWFhYWFhYWFhYWFhYWE=";
+      fc::mutable_variant_object mu;
+      mu("blob", blob{{a17_orig.begin(), a17_orig.end()}});
+      mu("str", a17_encoded);
+
+      BOOST_CHECK_EQUAL(mu["blob"].as_string(), a17_encoded);
+      std::vector<char> b64 = mu["str"].as_blob().data;
+      std::string_view b64_str(b64.data(), b64.size());
+      BOOST_CHECK_EQUAL(b64_str, a17_orig);
+   }
+   {
+      std::string s_6364 = "\x03" "\xef" "\xff" "\xf9";
+      std::string s_6364_encoded = "A+//+Q==";
+      fc::mutable_variant_object mu;
+      mu("blob", blob{{s_6364.begin(), s_6364.end()}});
+      mu("str", s_6364_encoded);
+
+      BOOST_CHECK_EQUAL(mu["blob"].as_string(), s_6364_encoded);
+      std::vector<char> b64 = mu["str"].as_blob().data;
+      std::string_view b64_str(b64.data(), b64.size());
+      BOOST_CHECK_EQUAL(b64_str, s_6364);
+   }
+   {
+      std::string org = "abc";
+      std::string encoded = "YWJj";
+
+      fc::mutable_variant_object mu;
+      mu("blob", blob{{org.begin(), org.end()}});
+      mu("str", encoded);
+
+      BOOST_CHECK_EQUAL(mu["blob"].as_string(), encoded);
+      std::vector<char> b64 = mu["str"].as_blob().data;
+      std::string_view b64_str(b64.data(), b64.size());
+      BOOST_CHECK_EQUAL(b64_str, org);
+   }
+}
+
+BOOST_AUTO_TEST_CASE(variant_blob_backwards_compatibility)
+{
+   // pre-5.0 variant would add an additional `=` as a flag that the blob data was base64 encoded
+   // verify variant can process encoded data with the extra `=`
+   {
+      std::string a17_orig = "aaaaaaaaaaaaaaaaa";
+      std::string a17_encoded = "YWFhYWFhYWFhYWFhYWFhYWE=";
+      std::string a17_encoded_old = a17_encoded + '=';
+      fc::mutable_variant_object mu;
+      mu("blob", blob{{a17_orig.begin(), a17_orig.end()}});
+      mu("str", a17_encoded_old);
+
+      BOOST_CHECK_EQUAL(mu["blob"].as_string(), a17_encoded);
+      std::vector<char> b64 = mu["str"].as_blob().data;
+      std::string_view b64_str(b64.data(), b64.size());
+      BOOST_CHECK_EQUAL(b64_str, a17_orig);
+   }
+   {
+      std::string org = "abc";
+      std::string encoded = "YWJj";
+      std::string encoded_old = encoded + '=';
+
+      fc::mutable_variant_object mu;
+      mu("blob", blob{{org.begin(), org.end()}});
+      mu("str", encoded_old);
+
+      BOOST_CHECK_EQUAL(mu["blob"].as_string(), encoded);
+      std::vector<char> b64 = mu["str"].as_blob().data;
+      std::string_view b64_str(b64.data(), b64.size());
+      BOOST_CHECK_EQUAL(b64_str, org);
+   }
+}
+
 BOOST_AUTO_TEST_SUITE_END()

tests/nodeos_run_test.py

@@ -41,7 +41,7 @@
 errFileName=f"{cluster.nodeosLogPath}/node_00/stderr.txt"
 if args.error_log_path:
     errFileName=args.error_log_path
-walletMgr=WalletMgr(True, port=walletPort)
+walletMgr=WalletMgr(True, port=walletPort, keepRunning=args.leave_running, keepLogs=args.keep_logs)
 testSuccessful=False
 dontBootstrap=sanityTest # intent is to limit the scope of the sanity test to just verifying that nodes can be started