Merge pull request #32 from copperbits/codebase_02_08_2019

codebase updated

ton-test-liteclient-full/lite-client/CMake/FindJeMalloc.cmake

@@ -0,0 +1,21 @@
+# - Find JeMalloc library
+# Find the native JeMalloc includes and library
+#
+# JEMALLOC_INCLUDE_DIR - where to find jemalloc.h, etc.
+# JEMALLOC_LIBRARIES - List of libraries when using jemalloc.
+# JEMALLOC_FOUND - True if jemalloc found.
+
+find_path(JEMALLOC_INCLUDE_DIR
+  NAMES jemalloc/jemalloc.h
+  HINTS ${JEMALLOC_ROOT_DIR}/include)
+
+find_library(JEMALLOC_LIBRARIES
+  NAMES jemalloc
+  HINTS ${JEMALLOC_ROOT_DIR}/lib)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(jemalloc DEFAULT_MSG JEMALLOC_LIBRARIES JEMALLOC_INCLUDE_DIR)
+
+mark_as_advanced(
+  JEMALLOC_LIBRARIES
+  JEMALLOC_INCLUDE_DIR)

ton-test-liteclient-full/lite-client/CMakeLists.txt

@@ -1,30 +1,88 @@
 cmake_minimum_required(VERSION 3.0.2)
 
-project(LITE_CLIENT)
+project(TON VERSION 0.5 LANGUAGES C CXX)
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+#set(OPENSSL_USE_STATIC_LIBS TRUE)
+
+# Prevent in-source build
+get_filename_component(TON_REAL_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}" REALPATH)
+get_filename_component(TON_REAL_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}" REALPATH)
+
+if (TON_REAL_BINARY_DIR STREQUAL TON_REAL_SOURCE_DIR)
+  message("  Out-of-source build should be used to build TDLib.")
+  message("  You need to remove the files already created by CMake and")
+  message("  rerun CMake from a new directory:")
+  message("  rm -rf CMakeFiles CMakeCache.txt")
+  message("  mkdir build")
+  message("  cd build")
+  message("  cmake ..")
+  message(FATAL_ERROR "In-source build failed.")
+endif()
+
+
+option(TONLIB_ENABLE_JNI "Use \"ON\" to enable JNI-compatible TonLib API.")
+option(TON_USE_ASAN "Use \"ON\" to enable AddressSanitizer." OFF)
+option(TON_USE_TSAN "Use \"ON\" to enable ThreadSanitizer." OFF)
+option(TON_USE_UBSAN "Use \"ON\" to enable UndefinedBehaviorSanitizer." OFF)
+set(TON_ARCH "native" CACHE STRING "Architecture, will be passed to -march=")
+
+if (TON_USE_ABSEIL)
+  message("Add abseil-cpp")
+  add_subdirectory(third-party/abseil-cpp EXCLUDE_FROM_ALL)
+  set(ABSL_FOUND 1)
+endif()
 
-add_subdirectory(third-party/abseil-cpp EXCLUDE_FROM_ALL)
+#add_subdirectory(third-party/libcuckoo EXCLUDE_FROM_ALL)
+#add_subdirectory(third-party/junction EXCLUDE_FROM_ALL)
+
+if (WIN32)
+  message("Add wingetopt")
+  add_subdirectory(third-party/wingetopt EXCLUDE_FROM_ALL)
+  set(WINGETOPT_FOUND 1)
+  message(STATUS "Use wingetopt")
+endif()
 
 set(CRC32C_BUILD_TESTS OFF CACHE BOOL "Build CRC32C's unit tests")
 set(CRC32C_BUILD_BENCHMARKS OFF CACHE BOOL "Build CRC32C's benchmarks")
 set(CRC32C_USE_GLOG OFF CACHE BOOL "Build CRC32C's tests with Google Logging")
 set(CRC32C_INSTALL OFF CACHE BOOL "Install CRC32C's header and library")
+message("Add crc32c")
 add_subdirectory(third-party/crc32c EXCLUDE_FROM_ALL)
 set(CRC32C_FOUND 1)
-set(HAVE_SSE42)
+
+if (TON_USE_ROCKSDB)
+  if (ANDROID) 
+    set(PORTABLE ON CACHE BOOL "portable")
+  endif()
+  set(WITH_TESTS OFF CACHE BOOL "build with tests")
+  set(WITH_TOOLS OFF CACHE BOOL "build with tools")
+  set(FAIL_ON_WARNINGS OFF CACHE BOOL "fail on warnings")
+  message("Add rocksdb")
+  add_subdirectory(third-party/rocksdb EXCLUDE_FROM_ALL)
+endif()
+
+option(USE_COROUTINES "experimental support of coroutines" OFF)
+if (USE_COROUTINES)
+  if (${CMAKE_CXX_COMPILER_ID} MATCHES Clang)
+    set(TD_HAVE_COROUTINES 1)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fcoroutines-ts")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++")
+  endif()
+endif()
 
 option(USE_LIBRAPTORQ "use libraptorq for tests" OFF)
 if (USE_LIBRAPTORQ)
   set(USE_LZ4 OFF CACHE BOOL "use lz4")
   if (${CMAKE_CXX_COMPILER_ID} MATCHES Clang)
     set(CLANG_STDLIB "ON") # for libraptorq
   endif()
+  message("Add libraptorq")
   add_subdirectory(third-party/libraptorq EXCLUDE_FROM_ALL)
 endif()
 
+message("Add ton")
 set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMake" ${CMAKE_MODULE_PATH})
 
-set(CMAKE_POSITION_INDEPENDENT_CODE ON)
-
 # Configure CCache if available
 find_program(CCACHE_FOUND ccache)
 #set(CCACHE_FOUND 0)
@@ -72,27 +130,53 @@ set(CMAKE_THREAD_PREFER_PTHREAD ON)
 set(THREADS_PREFER_PTHREAD_FLAG ON)
 find_package(Threads REQUIRED)
 
-if (NOT MSVC)
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=core2")
+if (TON_ARCH AND NOT MSVC)
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=${TON_ARCH}")
 endif()
 if (THREADS_HAVE_PTHREAD_ARG)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread")
 endif()
 
+if (TON_USE_JEMALLOC)
+  find_package(JeMalloc REQUIRED)
+endif()
+
+set(MEMPROF "" CACHE STRING "Use one of \"ON\", \"FAST\" or \"SAFE\" to enable memory profiling. \
+Works under macOS and Linux when compiled using glibc. \
+In FAST mode stack is unwinded only using frame pointers, which may fail. \
+In SAFE mode stack is unwinded using backtrace function from execinfo.h, which may be very slow. \
+By default both methods are used to achieve maximum speed and accuracy")
+
+if (CLANG OR GCC)
+  if (MEMPROF)
+    check_cxx_compiler_flag(-no-pie CXX_NO_PIE_FLAG)
+    if (CXX_NO_PIE_FLAG)
+      set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -no-pie")
+    elseif (APPLE)
+      set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-no_pie")
+    endif()
+  endif()
+endif()
+
 if (MSVC)
   if (CMAKE_CXX_FLAGS_DEBUG MATCHES "/RTC1")
     string(REPLACE "/RTC1" " " CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG}")
   endif()
   add_definitions(-D_SCL_SECURE_NO_WARNINGS -D_CRT_SECURE_NO_WARNINGS)
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /W4 /wd4100 /wd4127 /wd4324 /wd4505 /wd4702")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /MP /W4 /wd4100 /wd4127 /wd4324 /wd4505 /wd4702")
 elseif (CLANG OR GCC)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${STD14_FLAG} -fno-omit-frame-pointer")
   if (APPLE)
     #use "-Wl,-exported_symbols_list,${CMAKE_CURRENT_SOURCE_DIR}/export_list" for exported symbols
     set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fvisibility=hidden -Wl,-dead_strip,-x,-S")
+    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fvisibility=hidden -Wl,-dead_strip,-x,-S")
   else()
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ffunction-sections -fdata-sections")
     set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--gc-sections -Wl,--exclude-libs,ALL")
+    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections")
+    if (NOT TON_USE_ASAN AND NOT TON_USE_TSAN AND NOT MEMPROF)
+      set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--exclude-libs,ALL")
+    endif()
   endif()
 elseif (INTEL)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${STD14_FLAG}")
@@ -110,6 +194,9 @@ if (NOT ANDROID) # _FILE_OFFSET_BITS is broken in ndk r15 and r15b and doesn't w
 endif()
 
 include(AddCXXCompilerFlag)
+if (MSVC)
+  add_cxx_compiler_flag("/experimental:external /external:anglebrackets /external:W0")
+endif()
 if (NOT MSVC)
   add_cxx_compiler_flag("-Wall")
 endif()
@@ -131,10 +218,42 @@ add_cxx_compiler_flag("-Wconversion")
 add_cxx_compiler_flag("-Wno-sign-conversion")
 add_cxx_compiler_flag("-Qunused-arguments")
 add_cxx_compiler_flag("-Wno-unused-private-field")
+add_cxx_compiler_flag("-Wno-redundant-move")
+#add_cxx_compiler_flag("-Werror")
+
+#set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -isystem /usr/include/c++/v1")
+if (CLANG)
+  #set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++")
+endif()
+if (TON_USE_ASAN)
+  if (CLANG)
+    add_cxx_compiler_flag("-stdlib=libc++")
+  endif()
+  add_cxx_compiler_flag("-fsanitize=address")
+  add_definitions(-DTD_USE_ASAN=1)
+endif()
+if (TON_USE_TSAN)
+  if (CLANG)
+    add_cxx_compiler_flag("-stdlib=libc++")
+  endif()
+  add_cxx_compiler_flag("-fsanitize=thread")
+endif()
+if (TON_USE_UBSAN)
+  if (CLANG)
+    add_cxx_compiler_flag("-stdlib=libc++")
+  endif()
+  add_cxx_compiler_flag("-fsanitize=undefined")
+endif()
+#set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=thread")
+#set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address")
+#set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=undefined")
+#set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=leak")
+#set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -finstrument-functions")
 
 #Compilation database
 set(CMAKE_EXPORT_COMPILE_COMMANDS 1)
 
+
 function(target_link_libraries_system target)
   set(libs ${ARGN})
   foreach(lib ${libs})
@@ -145,23 +264,65 @@ function(target_link_libraries_system target)
 endfunction(target_link_libraries_system)
 
 add_subdirectory(tdutils)
+add_subdirectory(memprof)
 add_subdirectory(tdactor)
 add_subdirectory(tdnet)
+if (TON_USE_ROCKSDB)
+  option(TDDB_USE_ROCKSDB "Use rockdb" ON)
+endif()
 add_subdirectory(tddb)
 add_subdirectory(tdtl)
 add_subdirectory(tl)
-if (USE_LIBRAPTORQ) 
-  target_link_libraries(test-fec PRIVATE third_party_fec)
-  target_compile_definitions(test-fec PRIVATE "USE_LIBRAPTORQ=1")
-endif()
-
 add_subdirectory(terminal)
 add_subdirectory(keys)
 add_subdirectory(tl-utils)
 add_subdirectory(adnl)
-
 add_subdirectory(crypto)
+add_subdirectory(lite-client)
+
+
+
+if (NOT CMAKE_CROSSCOMPILING)
+  add_custom_target(prepare_cross_compiling DEPENDS tl_generate_common tdmime_auto tlb_generate_block)
+endif()
+
+#TESTS
+add_executable(test-ed25519 test/test-td-main.cpp ${ED25519_TEST_SOURCE})
+target_link_libraries(test-ed25519 PRIVATE ton_crypto)
+
+add_executable(test-vm test/test-td-main.cpp ${TONVM_TEST_SOURCE})
+target_link_libraries(test-vm PRIVATE ton_crypto fift-lib)
+
+add_executable(test-cells test/test-td-main.cpp ${CELLS_TEST_SOURCE})
+target_link_libraries(test-cells PRIVATE ton_crypto)
+
+add_executable(test-fift test/test-td-main.cpp ${FIFT_TEST_SOURCE})
+target_link_libraries(test-fift PRIVATE fift-lib)
+
+add_executable(test-tdutils test/test-td-main.cpp ${TDUTILS_TEST_SOURCE})
+target_link_libraries(test-tdutils PRIVATE tdutils ${CMAKE_THREAD_LIBS_INIT} memprof ${JEMALLOC_LIBRARIES})
+#target_link_libraries_system(test-tdutils absl::base absl::container absl::hash )
+#target_link_libraries_system(test-tdutils libcuckoo)
+#target_include_directories(test-tdutils PRIVATE SYSTEM ${JUNCTION_ALL_INCLUDE_DIRS})
+#target_link_libraries(test-tdutils PRIVATE ${JUNCTION_ALL_LIBRARIES})
+
+add_executable(test-tdactor test/test-td-main.cpp ${TDACTOR_TEST_SOURCE})
+target_link_libraries(test-tdactor PRIVATE tdactor ${CMAKE_THREAD_LIBS_INIT})
+
+add_executable(test-net test/test-td-main.cpp ${NET_TEST_SOURCE})
+target_link_libraries(test-net PRIVATE tdnet tdutils ${CMAKE_THREAD_LIBS_INIT})
+
+
+
+enable_testing()
+set(TEST_OPTIONS "--regression ${CMAKE_CURRENT_SOURCE_DIR}/test/regression-tests.ans --filter -Bench")
+separate_arguments(TEST_OPTIONS)
+add_test(test-ed25519 test-ed25519)
+add_test(test-vm test-vm ${TEST_OPTIONS})
+add_test(test-fift test-fift ${TEST_OPTIONS})
+add_test(test-cells test-cells ${TEST_OPTIONS})
+add_test(test-net test-net)
+add_test(test-actors test-tdactor)
+
+
 
-add_executable (test-lite-client test/test-lite-client.cpp test/test-lite-client.h)
-target_link_libraries(test-lite-client tdutils tdactor adnllite tl_api 
-  ton_crypto ton_block terminal )

ton-test-liteclient-full/lite-client/HOWTO

@@ -110,7 +110,7 @@ Create the file `new-wallet.fif` containing the source of our new smart contract
 ------------------------------------
 "Asm.fif" include
 
--1 constant wc  // create a wallet in workchain -1 (masterchain)
+0 constant wc  // create a wallet in workchain 0 (basechain)
 
 // Create new simple wallet
 <{  SETCP0 DUP IFNOTRET INC 32 THROWIF  // return if recv_internal, fail unless recv_external
@@ -154,13 +154,13 @@ dup ."External message for initialization is " <s csr. cr
 ."(Saved to file " type .")" cr
 --------------------------------------------
 
-Incidentally, you can find this sample file in crypto/block/new-wallet.fif.
+Incidentally, you can find a more sophisticated version of this sample file in crypto/smartcont/new-wallet.fif.
 
 Now, provided that you have compiled Fift binary (usually located as "crypto/fift" with respect to the build directory), you can run
 
-crypto/fift -I<source-directory>/crypto/fift new-wallet.fif
+crypto/fift -I<source-directory>/crypto/fift/lib new-wallet.fif
 
-assuming that you have copied new-wallet.fif into the current directory. You may wish to set the FIFTPATH environment variable to <source-directory>/crypto/fift, the directory containing Fift.fif and Asm.fif library files; then you can omit the -I argument to the Fift interpreter.
+assuming that you have copied new-wallet.fif into the current directory. You may wish to set the FIFTPATH environment variable to <source-directory>/crypto/fift/lib, the directory containing Fift.fif and Asm.fif library files; then you can omit the -I argument to the Fift interpreter.
 
 If everything worked, you'll see something like the following
 
@@ -210,7 +210,7 @@ You need to know the address of the test giver smart contract. We'll assume that
 
 as explained above in Section 2. The only number you need from the output is the 32-bit sequence number stored in the smart contract data (it is zero in the example above, but generally it will be non-zero).
 
-Next, you create an external message to the test giver asking it to send another message to your (uninitialized) smart contract carrying a specified amount of test Grams. There is a special Fift source file for generating this external message, located at crypto/block/testgiver.fif:
+Next, you create an external message to the test giver asking it to send another message to your (uninitialized) smart contract carrying a specified amount of test Grams. There is a special Fift source file for generating this external message, located at crypto/smartcont/testgiver.fif:
 
 --------------------------------------------
 // "testgiver.addr" file>B 256 B>u@ 
@@ -402,7 +402,7 @@ You will see that the smart contract has been initialized using code and data fr
 
 Actually, the simple wallet smart contract used in this example can be used to transfer test Grams to any other accounts. It is in this respect similar to the test giver smart contract discussed above, with the difference that it processes only external messages signed by the correct private key (of its owner). In our case, it is the private key saved into the file "new-wallet.pk" during the compilation of the smart contract (see Section 3).
 
-An example of how you might use this smart contract is provided in sample file crypto/block/wallet.fif :
+An example of how you might use this smart contract is provided in sample file crypto/smartcont/wallet.fif :
 
 --------------------------------------------------------
 #!/usr/bin/fift -s
@@ -449,4 +449,4 @@ When you run this code (by invoking the Fift interpreter), you create an externa
 
 Of course, a true TON Blockchain wallet application would hide all the intermediate steps explained above. It would first communicate the address of the new smart contract to the user, asking them to transfer some funds to the indicated address (displayed in its non-bounceable user-friendly form) from another wallet or a cryptocurrency exchange, and then would provide a simple interface to display the current balance and to transfer funds to whatever other addresses the user wants. (The aim of this document is to explain how to create new non-trivial smart contracts and experiment with the TON Blockchain Test Network, rather than to explain how one could use the Lite Client instead of a more user-friendly wallet application.)
 
-One final remark: The above examples used smart contracts in the masterchain (workchain -1). They would work in exactly the same way in the basic workchain (workchain 0), if one changes the "-1 constant wc" to "0 constant wc" in relevant places. The only difference is that the processing and storage fees in the basic workchain are 10-100 times lower than in the masterchain. However, during the early testing phases of the TON Blockchain Test Network the basic workchain may occasionally be unavailable, so the examples above are given for the masterchain. 
+One final remark: The above examples used smart contracts in the basic workchain (workchain 0). They would work in exactly the same way in the masterchain (workchain -1), if one changes the "0 constant wc" to "-1 constant wc" in relevant places. The only difference is that the processing and storage fees in the basic workchain are 10-100 times lower than in the masterchain.

ton-test-liteclient-full/lite-client/README

@@ -20,7 +20,7 @@ The TON Blockchain Test Network is updated quite often, so we cannot guarantee t
 
 cd ~/liteclient-build
 cmake ~/lite-client
-cmake --build . --target test-lite-client
+cmake --build . --target lite-client
 
 You might also want to build some extra utilities useful for smart-contract development:
 
@@ -32,12 +32,12 @@ wget https://test.ton.org/ton-lite-client-test1.config.json
 
 5) Run the Lite Client:
 
-./test-lite-client -C ton-lite-client-test1.config.json
+./lite-client/lite-client -C ton-lite-client-test1.config.json
 
 If everything was installed successfully, the Lite Client will connect to a special server (a full node for the TON Blockchain Test Network #1) and will send some queries to the server.
 If you indicate a writeable "database" directory as an extra argument to the client, it will download and save the block and the state corresponding to the newest masterchain block:
 
-./test-lite-client -C ton-lite-client-test1.config.json -D ~/ton-db-dir
+./lite-client/lite-client -C ton-lite-client-test1.config.json -D ~/ton-db-dir
 
 Basic help info can be obtained by typing "help" into the Lite Client. Type "quit" or press Ctrl-C to exit.
 

ton-test-liteclient-full/lite-client/adnl/CMakeLists.txt

@@ -1,13 +1,16 @@
 cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR)
 
+
 set (ADNL_LITE_SOURCE
   adnl-ext-client.cpp
   adnl-ext-connection.cpp
   adnl-query.cpp
+
+  ${ADNL_LITE_HEADERS}
 )
 
+
 add_library(adnllite STATIC ${ADNL_LITE_SOURCE})
 
 target_include_directories(adnllite PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/..>)
 target_link_libraries(adnllite PUBLIC tdactor ton_crypto tl_api tdnet keys )
-