Merge pull request #12 from biomedbigdata/improved_realtime_server

Improved realtime server

.dockerignore

@@ -23,6 +23,7 @@ docs/
 
 # Binary directories
 **/bin
+!/ext/uWebSockets/uSockets/lsquic/bin
 
 # INI files for unit tests
 /test/unit/init/*
@@ -108,6 +109,8 @@ test/model/res/*
 /cmake-build-*
 /.idea/
 
+**/CMakeCache.txt
+
 #load
 # /data/LOAD/
 # /data/CAD_control/*
@@ -123,6 +126,8 @@ test/model/res/*
 /ext/boost_1_71_0/tools/build/src/engine/b2
 /ext/boost_1_71_0/tools/build/src/engine/bjam
 
+/ext/uWebSockets/uSockets/*.o
+
 .Rhistory
 
 

.github/workflows/docker-publish.yml

@@ -25,6 +25,8 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
+        with:
+          submodules: 'recursive'
 
       # Workaround: https://github.com/docker/build-push-action/issues/461
       - name: Setup Docker buildx

.gitignore

@@ -119,8 +119,9 @@ test/model/res/*
 .Rhistory
 
 
-.INSTALLED
-.RES_EXTRACTED
+**/.INSTALLED
+**/.uWebSockets_INSTALLED
+**/.RES_EXTRACTED
 .env
 
 docker/data

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "ext/uWebSockets"]
+	path = ext/uWebSockets
+	url = https://github.com/uNetworking/uWebSockets.git

CMakeLists.txt

@@ -19,10 +19,12 @@
 #                                                                             #
 ###############################################################################
 
-cmake_minimum_required(VERSION 3.2)
+cmake_minimum_required(VERSION 3.8)
 set(CMAKE_VERBOSE_MAKEFILE ON)
 project(NeEDL)
 
+include(CMakePrintHelpers)
+
 # Determine build type.
 if(NOT CMAKE_BUILD_TYPE)
   message(STATUS "Building NeEDL with build type 'Release', as none was specified.")
@@ -31,6 +33,22 @@ else()
   message(STATUS "Building NeEDL with build type '${CMAKE_BUILD_TYPE}'.")
 endif()
 
+set(USE_INCLUDED_BOOST "1" CACHE BOOL "Use the boost version shipped with this repository")
+cmake_print_variables(USE_INCLUDED_BOOST)
+if (USE_INCLUDED_BOOST)
+  message("-- boost version: vendored")
+  set(BOOST_INCLUDE ${CMAKE_SOURCE_DIR}/ext/boost_1_71_0)
+  set(BOOST_LINK_DIR ${CMAKE_SOURCE_DIR}/ext/boost_1_71_0/stage/lib)
+else()
+  message("-- boost version: system")
+  set(Boost_USE_STATIC_LIBS OFF) 
+  set(Boost_USE_MULTITHREADED ON)  
+  set(Boost_USE_STATIC_RUNTIME OFF) 
+  find_package(Boost 1.71.0 REQUIRED COMPONENTS filesystem graph)
+
+  set(BOOST_INCLUDE ${Boost_INCLUDE_DIRS})
+endif()
+
 # Find doxygen.
 #find_package(Doxygen)
 #if(DOXYGEN_FOUND)
@@ -43,7 +61,6 @@ find_package(Python COMPONENTS Interpreter Development)
 find_package(SQLite3 REQUIRED)
 
 # debug stuff
-include(CMakePrintHelpers)
 cmake_print_variables(CMAKE_C_COMPILER)
 cmake_print_variables(CMAKE_C_COMPILER_VERSION)
 cmake_print_variables(CMAKE_CXX_COMPILER)
@@ -52,7 +69,7 @@ cmake_print_variables(SQLite3_INCLUDE_DIRS)
 cmake_print_variables(SQLite3_LIBRARIES)
 
 # Set up the compiler.
-set(CMAKE_CXX_FLAGS "-std=c++14 -Wall -Wextra -pedantic -Wredundant-decls -Wdisabled-optimization -Wctor-dtor-privacy -Wnon-virtual-dtor -Werror=old-style-cast -Wsign-promo -Werror=return-type -Wno-unused-parameter")
+set(CMAKE_CXX_FLAGS "-std=c++20 -Wall -Wextra -pedantic -Wredundant-decls -Wdisabled-optimization -Wctor-dtor-privacy -Wnon-virtual-dtor -Werror=old-style-cast -Wsign-promo -Werror=return-type -Wno-unused-parameter")
 set(CMAKE_CXX_FLAGS_DEBUG "-O3 -g")
 set(CMAKE_CXX_FLAGS_RELEASE "-O3")
 
@@ -67,15 +84,14 @@ endif()
 set(CMAKE_MACOSX_RPATH ON)
 
 # Add include and link directories.
-set(BOOST_ROOT ${CMAKE_SOURCE_DIR}/ext/boost_1_71_0)
 set(EIGEN_ROOT ${CMAKE_SOURCE_DIR}/ext/eigen_3_3_7)
 set(CLI11_ROOT ${CMAKE_SOURCE_DIR}/ext/cli11_1_9_0)
 set(CATCH_ROOT ${CMAKE_SOURCE_DIR}/ext/catch_2_13_9)
 set(IGRAPH_ROOT ${CMAKE_SOURCE_DIR}/ext/igraph_0.9.8)
 set(PYBIND11_HOME ${CMAKE_SOURCE_DIR}/ext/pybind11)
 
-include_directories(SYSTEM ${SQLite3_INCLUDE_DIRS} ${BOOST_ROOT} ${EIGEN_ROOT} ${CLI11_ROOT} ${CATCH_ROOT} ${IGRAPH_ROOT}/include ${IGRAPH_ROOT}/build/include ${PYBIND11_HOME}/include ${SQLite3_INCLUDE_DIRS} ${Python_INCLUDE_DIRS} /usr/lib/llvm-10/include)
-link_directories(${BOOST_ROOT}/stage/lib /usr/lib/llvm-10/lib ${IGRAPH_ROOT}/build/src ${Python_LIBRARY_DIRS} ${SQLite3_LIBRARIES})
+include_directories(SYSTEM ${SQLite3_INCLUDE_DIRS} ${BOOST_INCLUDE} ${EIGEN_ROOT} ${CLI11_ROOT} ${CATCH_ROOT} ${IGRAPH_ROOT}/include ${IGRAPH_ROOT}/build/include ${PYBIND11_HOME}/include ${SQLite3_INCLUDE_DIRS} ${Python_INCLUDE_DIRS} /usr/lib/llvm-10/include)
+link_directories(${BOOST_LINK_DIR} /usr/lib/llvm-10/lib ${IGRAPH_ROOT}/build/src ${Python_LIBRARY_DIRS} ${SQLite3_LIBRARIES})
 if(APPLE)
   include_directories(SYSTEM ${OMP_ROOT}/include)
   link_directories(${OMP_ROOT}/lib)

docker/NeEDL/Dockerfile

@@ -32,7 +32,12 @@ WORKDIR /NeEDL
 ENV DEBIAN_FRONTEND=noninteractive
 
 # install build dependencies
-RUN apt-get update -y && apt-get install -y build-essential git cmake autoconf libtool pkg-config python3.10 python3.10-dev gcc g++ sqlite3 libsqlite3-dev python3.10-distutils unzip
+RUN apt-get update -y && \
+    apt-get install -y build-essential git cmake autoconf libtool pkg-config python3.10 python3.10-dev gcc g++ sqlite3 libsqlite3-dev python3.10-distutils unzip curl libevent-dev libboost-all-dev && \
+    curl https://dl.google.com/go/go1.21.1.linux-amd64.tar.gz | tar -xz -C /usr/local
+
+ENV PATH /usr/local/go/bin:$PATH
+
 
 # copy necessary NeEDL repo files
 COPY ext ./ext
@@ -47,11 +52,12 @@ COPY data/dbSNP/inc_pseudogenes /NeEDL/data/dbSNP/inc_pseudogenes
 COPY data/BIOGRID /NeEDL/data/BIOGRID
 
 # initialize everything needed for the build and build all targets
-RUN python3.10 install.py --clean && \ 
-    python3.10 install.py --target NeEDL && \
-    python3.10 install.py --target calculate_scores && \
-    python3.10 install.py --target convert_to_binary && \
-    python3.10 install.py --target epiJSON
+RUN python3.10 install.py --clean --no-submodule-extraction --system-boost && \ 
+    python3.10 install.py --target NeEDL --system-boost && \
+    python3.10 install.py --target calculate_scores --system-boost && \
+    python3.10 install.py --target convert_to_binary --system-boost && \
+    python3.10 install.py --target epiJSON --system-boost && \
+    python3.10 install.py --target realtime_scores --system-boost
 
 # ---- end of stage 2
 
@@ -65,10 +71,10 @@ LABEL org.opencontainers.image.source=https://github.com/biomedbigdata/NeEDL
 ENV DEBIAN_FRONTEND=noninteractive
 
 # install production dependencies
-RUN apt-get update -y && apt-get install -y python3.10 python3.10-dev sqlite3 g++ gcc
+RUN apt-get update -y && apt-get install -y python3.10 python3.10-dev sqlite3 g++ gcc libboost-all-dev
 
 # copy everything important over to the new container
-COPY --from=build /NeEDL/ext/boost_1_71_0/stage /NeEDL/ext/boost_1_71_0/stage
+# COPY --from=build /NeEDL/ext/boost_1_71_0/stage /NeEDL/ext/boost_1_71_0/stage
 COPY --from=build /NeEDL/test/model/bin/* /NeEDL/test/model/bin/
 COPY --from=build /NeEDL/data/dbSNP/inc_pseudogenes/snps_restruc_full_inc_pseudo.csv /NeEDL/data/dbSNP/inc_pseudogenes/snps_restruc_full_inc_pseudo.csv
 COPY --from=build /NeEDL/data/BIOGRID/BIOGRID-ORGANISM-Homo_sapiens-3.5.182.tab2.txt /NeEDL/data/BIOGRID/BIOGRID-ORGANISM-Homo_sapiens-current.tab2.txt
@@ -78,5 +84,7 @@ COPY --from=build /NeEDL/ext/plink/plink_linux_x86_64_20230116/ /NeEDL/ext/plink
 RUN ln -s /NeEDL/test/model/bin/NeEDL /usr/local/bin/NeEDL && \
     ln -s /NeEDL/test/model/bin/calculate_scores /usr/local/bin/calculate_scores && \
     ln -s /NeEDL/test/model/bin/convert_to_binary /usr/local/bin/convert_to_binary && \
-    ln -s /NeEDL/test/model/bin/convert_to_json /usr/local/bin/epiJSON
+    ln -s /NeEDL/test/model/bin/epiJSON /usr/local/bin/epiJSON && \
+    ln -s /NeEDL/test/model/bin/realtime_scores /usr/local/bin/realtime_scores
+
 # ---- end of stage 3

docker/realtime_scores/docker-compose.yml

@@ -1,13 +1,11 @@
 version: '3'
+name: needl_realtime_scores_api
 
 services:
   realtime_scores:
+    image: bigdatainbiomedicine/needl
     restart: always
-    command: ./test/model/bin/realtime_scores --input-dir /datasets --num-threads $NUM_THREADS --bind-addr 0.0.0.0 --bind-port 3000
-    build:
-      context: ../../
-      dockerfile: docker/realtime_scores/Dockerfile
-      # target: base
+    command: /NeEDL/test/model/bin/realtime_scores --input-dir /datasets --num-threads $NUM_THREADS --bind-addr 0.0.0.0 --bind-port 3000
     volumes:
       - ${DATASET_DIR}:/datasets:Z
     user: ${REALTIME_SCORES_USER}

install.py

@@ -53,6 +53,10 @@ def build_targets(args, all_targets, dependencies):
     if (not os.path.isfile("build/Makefile")):
         print("-- Running CMake.")
         commands = "cd build; rm -rf *; cmake .. -DCMAKE_BUILD_TYPE=" + ("Debug" if args.debug else "Release")
+
+        if args.system_boost:
+            commands += f' -DUSE_INCLUDED_BOOST=0'
+
         commands += f' -DPython_EXECUTABLE="{dependencies["python3"]}"'
         commands += f' -DCMAKE_C_COMPILER="{dependencies["gcc"]}" -DCMAKE_CXX_COMPILER="{dependencies["g++"]}"'
         if platform.system() == "Darwin":
@@ -75,7 +79,9 @@ def build_targets(args, all_targets, dependencies):
 
 def build_external_libraries(args, dependencies):
     # boost
-    if os.path.isfile("ext/boost_1_71_0/.INSTALLED") and not args.clean:
+    if args.system_boost:
+        print("-- Boost libraries: system installation is used --> build of boost is skipped")
+    elif os.path.isfile("ext/boost_1_71_0/.INSTALLED") and not args.clean:
         print("-- Boost libraries already built.")
     else:
         print("-- Building Boost libraries.")
@@ -100,6 +106,18 @@ def build_external_libraries(args, dependencies):
         f = open("ext/igraph_0.9.8/.INSTALLED", "w")
         f.close()
 
+    # uWebSockets
+    if os.path.isfile("ext/.uWebSockets_INSTALLED") and not args.clean:
+        print("-- uWebSocket library already built.")
+    else:
+        print("-- Building uWebSockets library")
+        if not args.no_submodule_extraction:
+            subprocess.call('git submodule update --init --recursive')
+
+        subprocess.call('cd ext/uWebSockets/uSockets;make boringssl;cd boringssl;BORINGSSL=$PWD;cd ../lsquic;cmake -DBORINGSSL_DIR=$BORINGSSL .;make;cd ..;WITH_LTO=1 WITH_QUIC=1 WITH_BORINGSSL=1 make', shell=True)
+        f = open("ext/.uWebSockets_INSTALLED", "w")
+        f.close()
+
 def extract_all_zips_in_folder(folder):
     print("extracting files in " + folder)
 
@@ -125,7 +143,7 @@ def extract_all_zips_in_folder(folder):
         for entry in files:
             if entry.name.endswith(".zip"):
                 print("Extracting " + entry.name)
-                subprocess.call('unzip -n -d "' + escaped_folder + '" "' + escaped_folder + entry.name.replace('"', '\\"') + '"', shell=True)
+                subprocess.call('unzip -qo -d "' + escaped_folder + '" "' + escaped_folder + entry.name.replace('"', '\\"') + '"', shell=True)
 
 def extract_resources(args):
     if (os.path.isfile(".RES_EXTRACTED") and not args.clean) or args.no_data_unpacking:
@@ -200,6 +218,8 @@ def find_dependencies(args):
 parser.add_argument("--gcc", help="one can select the path to gcc manually if the automatically selected compiler is not correct.", default=None)
 parser.add_argument("--gxx", help="one can select the path to g++ manually if the automatically selected compiler is not correct.", default=None)
 parser.add_argument("--extract-datasets", help="Also extracts simulated datasets. These might be necessary for the unit tests.", action="store_true")
+parser.add_argument("--no-submodule-extraction", help="When set the script does not recursively download submodules. This is used for building the docker container.", action="store_true")
+parser.add_argument("--system-boost", help="Use the system boost installation instead of the one in this repository", action="store_true")
 args = parser.parse_args()
 
 

quepistasis/header/cpu_sa.h

@@ -64,4 +64,8 @@ int general_simulated_annealing(
     void * const interrupt_function
 );
 
+#ifdef HEADER_ONLY
+#include "../src/cpu_sa.cpp"
+#endif
+
 #endif

quepistasis/header/cpu_sa_wrapper.h

@@ -39,4 +39,9 @@ void simulated_annealing_ising(
         uint64_t seed,
         sa_return* ret);
 
+
+#ifdef HEADER_ONLY
+#include "../src/cpu_sa_wrapper.cpp"
+#endif
+
 #endif

quepistasis/header/python_wrapper.h

@@ -209,4 +209,9 @@ class PythonWrapper {
         int clique_size, int shots, const char* save_path);
 };
 
+
+#ifdef HEADER_ONLY
+#include "../src/python_wrapper.cpp"
+#endif
+
 #endif

quepistasis/header/snps_optimization.h

@@ -244,4 +244,9 @@ class snps_qubo_matrix : public matrix {
 
 };
 
+
+#ifdef HEADER_ONLY
+#include "../src/snps_optimization.cpp"
+#endif
+
 #endif

src/jobs/ShinyAppLauncher.hpp

@@ -21,4 +21,8 @@ namespace epi {
 
 } // epi
 
+#ifdef HEADER_ONLY
+#include "ShinyAppLauncher.cpp"
+#endif
+
 #endif //GENEPISEEKER_SHINYAPPLAUNCHER_HPP

test/e2e/test_container.sh

@@ -25,6 +25,12 @@ python ./run/epiJSON.py --docker-image-name "$1" --docker-no-pulling  --help
 python ./run/calculate_scores.py --docker-image-name "$1" --docker-no-pulling  --help
 python ./run/convert_to_binary.py --docker-image-name "$1" --docker-no-pulling  --help
 
+# realtime_scores is a special case --> no python launcher script exists for it
+# run containers as current user
+user_id=$(id -u)
+group_id=$(id -g)
+docker run --user $user_id:$group_id "$1" /NeEDL/test/model/bin/realtime_scores --help
+
 
 
 # select small dummy dataset