diff --git a/.gitignore b/.gitignore index 951ba239..a6639b40 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,6 @@ czmq libsodium libzmq malamute + +# Generated files from .in templates +ldconf/halcs.conf diff --git a/Makefile b/Makefile index e6aa8bf0..bf14917b 100644 --- a/Makefile +++ b/Makefile @@ -51,6 +51,8 @@ SCRIPTS_ETC_PREFIX ?= # Installation prefix for the share scripts. # Usually this is set to /usr/local SCRIPTS_SHARE_PREFIX ?= /usr/local +# LDCONF script +LDCONF_ETC_PREFIX ?= /etc/ld.so.conf.d # Selects the install location of the config file PREFIX ?= /usr/local export PREFIX @@ -66,6 +68,8 @@ TOP := $(strip $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))) SRC_DIR := $(TOP)/src export TOP +LDCONF_DIR := ldconf + # Libraries LIBS_DIR := $(TOP)/libs export LIBS_DIR @@ -219,6 +223,9 @@ CFLAGS_DEBUG += -g CFLAGS_PLATFORM = -Wall -Wextra -Werror LDFLAGS_PLATFORM = -Wl,-T,$(LD_SCRIPT) +# Install halcsd LDCONFIG files +LDCONF = ldconf + # Libraries LIBS = -lm -lzmq -lczmq -lmlm @@ -262,6 +269,7 @@ include $(SRC_DIR)/revision/revision.mk include $(SRC_DIR)/boards/$(BOARD)/board.mk include $(SRC_DIR)/boards/common/common.mk include $(APPS_MKS) +include $(LDCONF_DIR)/ldconf.mk # Project boards boards_INCLUDE_DIRS = -Iinclude/boards/$(BOARD) @@ -316,6 +324,7 @@ apps_OBJS = $(foreach app_obj,$(APPS),$($(app_obj)_all_OBJS)) apps_ETC_SCRIPTS = $(foreach app,$(APPS),$($(app)_ETC_SCRIPTS)) apps_SHARE_SCRIPTS = $(foreach app,$(APPS),$($(app)_SHARE_SCRIPTS)) +ldconfig_ETC_SCRIPTS = $(foreach ldconf,$(LDCONF),$($(ldconf)_ETC_SCRIPTS)) .SECONDEXPANSION: @@ -663,8 +672,10 @@ scripts_install: $(foreach app_script,$(apps_ETC_SCRIPTS),cp --preserve=mode $(subst :,,$(app_script)) ${SCRIPTS_ETC_PREFIX}/$(shell echo $(app_script) | cut -f2 -d:) $(CMDSEP)) $(foreach app_script,$(apps_SHARE_SCRIPTS),mkdir -p $(dir ${SCRIPTS_SHARE_PREFIX}/$(shell echo $(app_script) | cut -f2 -d:)) $(CMDSEP)) $(foreach app_script,$(apps_SHARE_SCRIPTS),cp --preserve=mode $(subst :,,$(app_script)) ${SCRIPTS_SHARE_PREFIX}/$(shell echo $(app_script) | cut -f2 -d:) $(CMDSEP)) + $(foreach ldconf_script,$(ldconfig_ETC_SCRIPTS),cp --preserve=mode $(subst :,,$(ldconf_script)) ${LDCONF_ETC_PREFIX}/$(shell echo $(ldconf_script) | cut -f2 -d:) $(CMDSEP)) scripts_uninstall: + $(foreach ldconf_script,$(ldconfig_ETC_SCRIPTS),rm -f ${LDCONF_ETC_PREFIX}/$(shell echo $(ldconf_script) | cut -f2 -d:) $(CMDSEP)) $(foreach app_script,$(apps_SHARE_SCRIPTS),rm -f ${SCRIPTS_SHARE_PREFIX}/$(shell echo $(app_script) | cut -f2 -d:) $(CMDSEP)) $(foreach app_script,$(apps_ETC_SCRIPTS),rm -f ${SCRIPTS_ETC_PREFIX}/$(shell echo $(app_script) | cut -f2 -d:) $(CMDSEP)) diff --git a/ci_build.sh b/ci_build.sh index b4dba89f..58d4676f 100755 --- a/ci_build.sh +++ b/ci_build.sh @@ -364,7 +364,7 @@ elif [ "$CPACK" = yes ]; then PACKPACK_OPTS+=("${CPACK_GENERATORS}") PACKPACK_OPTS+=("-Dhalcs_DISTRO_VERSION=${CPACK_DISTRO_VERSION}") PACKPACK_OPTS+=("-DCMAKE_PREFIX_PATH=/source/${BUILD_PREFIX_BASENAME}") - PACKPACK_OPTS+=("-DBUILD_PCIE_DRIVER=OFF") + PACKPACK_OPTS+=("-DBUILD_PCIE_DRIVER=ON") PACKPACK_OPTS+=("-Dhalcs_BOARD_OPT=${CPACK_BOARDS}") # only expand and add ":" to LD_LIBRARY_PATH if non-empty LD_LIBRARY_PATH=${LOCAL_LD_LIBRARY_PATH}${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH} \ diff --git a/cmake/Modules/AddExternalProjectPciedriver.cmake b/cmake/Modules/AddExternalProjectPciedriver.cmake index c4c0a6c7..377784d3 100644 --- a/cmake/Modules/AddExternalProjectPciedriver.cmake +++ b/cmake/Modules/AddExternalProjectPciedriver.cmake @@ -118,99 +118,97 @@ install(FILES option(BUILD_PCIE_DRIVER "Install PCIe driver" ON) if(BUILD_PCIE_DRIVER) - # get kernel utils - include(GetKernelUtils) - - ExternalProject_Add(pciedriver_driver_ext - URL ${CMAKE_CURRENT_SOURCE_DIR}/foreign/pcie-driver - PREFIX ${CMAKE_CURRENT_BINARY_DIR}/pciedriver - CONFIGURE_COMMAND "" - # Build only driver - BUILD_COMMAND make core_driver - INSTALL_COMMAND "" - # PCIEDRIVER assumes in-source build - BUILD_IN_SOURCE ON - ) - - ExternalProject_Get_property(pciedriver_driver_ext BINARY_DIR) - # this is not a CMake project, so no *Config.cmake or *Targets.cmake - # files will be generated. Hardcode library and include paths so when - # libraries try to search for pciedriver they can find it locally! - set(pciedriver_driver_KO_DIRECTORIES - ${BINARY_DIR}/drivers/pcie/src + set(pciedriver_driver_ext_DIR + ${CMAKE_CURRENT_SOURCE_DIR}/foreign/pcie-driver ) - set(pciedriver_driver_INCLUDE_DIRECTORIES - ${BINARY_DIR}/include/pcie/driver - ${BINARY_DIR}/include/pcie/driver/../.. + # Driver directory + set(pciedriver_driver_KO_DIRECTORY + ${pciedriver_driver_ext_DIR}/drivers/pcie/src ) - # Hack to make it work, otherwise INTERFACE_INCLUDE_DIRECTORIES will not be propagated - file(MAKE_DIRECTORY ${pciedriver_driver_INCLUDE_DIRECTORIES}) - - set(pciedriver_driver_public_headers - ${BINARY_DIR}/include/pcie/driver/pciDriver.h - ) + # DKMS install directory + set (pciedriver_DKMS_INSTALL_DIR "/usr/src/pciDriver-${VERSION}") - # Get filenames only so we install only those listed above - list(APPEND pciedriver_driver_public_files "") + # DKMS files + set(DKMS_FILE_NAME "dkms.conf") + set(DKMS_POSTINST "postinst") + set(DKMS_PRERM "prerm") - foreach(public_dir_header ${pciedriver_driver_public_headers}) - get_filename_component(public_dir - ${public_dir_header} - NAME - ) - - list(APPEND pciedriver_driver_public_files ${public_dir}) - endforeach() + configure_file ( + "${pciedriver_driver_KO_DIRECTORY}/${DKMS_FILE_NAME}.in" + "${CMAKE_CURRENT_BINARY_DIR}/${DKMS_FILE_NAME}" + @ONLY + ) - # remove, well, duplicates ... - list(REMOVE_DUPLICATES pciedriver_driver_public_files) + set(PCIDRIVER_VERSION_STRING + ${VERSION} + ) - # Get directory names only. In this case, we must install - # pciDriver/lib directory structure, so list them here - # separately - list(APPEND pciedriver_driver_public_dirs - ${BINARY_DIR}/include/pcie + configure_file ( + "${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules/${DKMS_POSTINST}.in" + "${CMAKE_CURRENT_BINARY_DIR}/${DKMS_POSTINST}" + @ONLY ) - # add imported library so we can link to it later - add_library(pciedriver_driver SHARED IMPORTED) - - # set library locations - set_target_properties(pciedriver_driver - PROPERTIES - IMPORTED_LOCATION - ${pciedriver_driver_KO_DIRECTORIES}/pciDriver.ko - INTERFACE_INCLUDE_DIRECTORIES - "${pciedriver_driver_INCLUDE_DIRECTORIES}" - PUBLIC_HEADER - ${pciedriver_driver_public_headers} + configure_file ( + "${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules/${DKMS_PRERM}.in" + "${CMAKE_CURRENT_BINARY_DIR}/${DKMS_PRERM}" + @ONLY ) - add_dependencies(pciedriver_driver pciedriver_driver_ext) ####################################### # Driver installer ####################################### - set(pciedriver_driver_install_dir "/lib/modules/${UNAME_KERNEL_VERSION}/extra") + # FIXME. Get all filenames REALPATHs? + get_filename_component(pciedriver_driver_HEADER_ABS + ${pciedriver_driver_KO_DIRECTORY}/pciDriver.h + REALPATH + ) + +get_filename_component(pciedriver_driver_UDEV_ABS + ${pciedriver_driver_KO_DIRECTORY}/60-udev_fpga.rules + REALPATH + ) - # install headers that are inside another directory - install(DIRECTORY - ${pciedriver_driver_public_dirs} - DESTINATION - ${CMAKE_INSTALL_INCLUDEDIR} - FILES_MATCHING PATTERN - "${pciedriver_driver_public_files}" + set(pciedriver_driver_SRCS + ${pciedriver_driver_KO_DIRECTORY}/base.c + ${pciedriver_driver_KO_DIRECTORY}/base.h + ${pciedriver_driver_KO_DIRECTORY}/common.h + ${pciedriver_driver_KO_DIRECTORY}/compat.h + ${pciedriver_driver_KO_DIRECTORY}/config.h + ${pciedriver_driver_KO_DIRECTORY}/int.c + ${pciedriver_driver_KO_DIRECTORY}/int.h + ${pciedriver_driver_KO_DIRECTORY}/ioctl.c + ${pciedriver_driver_KO_DIRECTORY}/ioctl.h + ${pciedriver_driver_KO_DIRECTORY}/kmem.c + ${pciedriver_driver_KO_DIRECTORY}/kmem.h + ${pciedriver_driver_KO_DIRECTORY}/sysfs.c + ${pciedriver_driver_KO_DIRECTORY}/sysfs.h + ${pciedriver_driver_KO_DIRECTORY}/umem.c + ${pciedriver_driver_KO_DIRECTORY}/umem.h + ${pciedriver_driver_KO_DIRECTORY}/Makefile + ${pciedriver_driver_HEADER_ABS} + ${pciedriver_driver_UDEV_ABS} ) - # as we can't intall IMPORTED libraries as TARGETS, just use plain - # install(FILES) - install(FILES - ${pciedriver_driver_KO_DIRECTORIES}/pciDriver.ko - DESTINATION - ${pciedriver_driver_install_dir} + set(pciedriver_driver_SCRIPTS + ${pciedriver_driver_KO_DIRECTORY}/version.sh ) + + install(FILES ${pciedriver_driver_SRCS} DESTINATION ${pciedriver_DKMS_INSTALL_DIR}) + install(FILES ${pciedriver_driver_SCRIPT} DESTINATION ${pciedriver_DKMS_INSTALL_DIR} + PERMISSIONS + OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE + ) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/${DKMS_FILE_NAME} DESTINATION ${pciedriver_DKMS_INSTALL_DIR}) + + # For CPack + set(CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA "${CMAKE_CURRENT_BINARY_DIR}/postinst;${CMAKE_CURRENT_BINARY_DIR}/prerm") + set(CPACK_RPM_POST_INSTALL_SCRIPT_FILE "${CMAKE_CURRENT_BINARY_DIR}/postinst") + set(CPACK_RPM_PRE_UNINSTALL_SCRIPT_FILE "${CMAKE_CURRENT_BINARY_DIR}/prerm") + endif() diff --git a/cmake/Modules/TestPciedriverVersion.cmake b/cmake/Modules/TestPciedriverVersion.cmake index 8316ffcc..89ea4b2e 100644 --- a/cmake/Modules/TestPciedriverVersion.cmake +++ b/cmake/Modules/TestPciedriverVersion.cmake @@ -1,16 +1,24 @@ # Read header file and extract version numbers -# PCIe Driver doesn't have the notion of version numbers.... -# assign 1.0.0 +file(READ + "${PROJECT_SOURCE_DIR}/foreign/pcie-driver/include/pcie/driver/pciDriver.h" + _pciedriver_H_CONTENTS +) -set(pciedriver_VERSION_MAJOR - 1 +string(REGEX REPLACE + ".*#define PCIEDRIVER_VERSION_MAJOR ([0-9]+).*" + "\\1" pciedriver_VERSION_MAJOR + "${_pciedriver_H_CONTENTS}" ) -set(pciedriver_VERSION_MINOR - 0 +string(REGEX REPLACE + ".*#define PCIEDRIVER_VERSION_MINOR ([0-9]+).*" + "\\1" pciedriver_VERSION_MINOR + "${_pciedriver_H_CONTENTS}" ) -set(pciedriver_VERSION_PATCH - 0 +string(REGEX REPLACE + ".*#define PCIEDRIVER_VERSION_PATCH ([0-9]+).*" + "\\1" pciedriver_VERSION_PATCH + "${_pciedriver_H_CONTENTS}" ) set(pciedriver_VERSION diff --git a/cmake/Modules/postinst.in b/cmake/Modules/postinst.in new file mode 100755 index 00000000..a15a15f5 --- /dev/null +++ b/cmake/Modules/postinst.in @@ -0,0 +1,47 @@ +#!/bin/sh + +# Adapted from Xilinx XRT project: +# https://github.com/Xilinx/XRT/blob/master/src/CMake/config/postinst.in + +rmmodules() +{ + find /lib/modules -type f -name pciDriver.ko -delete + depmod -A +} + + +echo "Unloading old pciDriver Linux kernel modules" +rmmod pciDriver + +PCIDRIVER_VERSION_STRING_OLD=`dkms status -m pciDriver | awk -F, '{print $2}'` +for OLD in $PCIDRIVER_VERSION_STRING_OLD; do + echo "Unregistering old pciDriver Linux kernel module sources $OLD from dkms" + dkms remove -m pciDriver -v $OLD --all + break +done + +rmmodules + +echo "Invoking DKMS common.postinst for pciDriver" +/usr/lib/dkms/common.postinst pciDriver @PCIDRIVER_VERSION_STRING@ "" "" $2 + +if [ $? -eq 0 ]; then + echo "Finished DKMS common.postinst" + install -m 644 /usr/src/pciDriver-@PCIDRIVER_VERSION_STRING@/60-udev_fpga.rules /etc/udev/rules.d + + echo "Loading new pciDriver Linux kernel modules" + udevadm control --reload-rules + modprobe pciDriver + udevadm trigger +fi + +if [ -z "`dkms status -m pciDriver -v @PCIDRIVER_VERSION_STRING@ |grep installed`" ]; then + echo "****************************************************************" + echo "* DKMS failed to install pciDriver drivers." + echo "* Please check if kernel development headers are installed for OS variant used." + echo "* " + echo "* Check build logs in /var/lib/dkms/pciDriver/@PCIDRIVER_VERSION_STRING@" + echo "****************************************************************" +fi + +exit 0 diff --git a/cmake/Modules/prerm.in b/cmake/Modules/prerm.in new file mode 100755 index 00000000..9e2baa10 --- /dev/null +++ b/cmake/Modules/prerm.in @@ -0,0 +1,33 @@ +#!/bin/sh + +# Adapted from Xilinx XRT project: +# https://github.com/Xilinx/XRT/blob/master/src/CMake/config/postinst.in + +lsb_release -si | grep -Eq "^RedHat|^CentOS" +if [ $? -eq 0 ] && [ $1 -ge 1 ]; then + echo "Cleanup is skipped for package upgrade/downgrade/re-install on RHEL/CentOS" + exit 0 +fi + +lsb_release -si | grep -Eq "^Ubuntu|^Debian" +if [ $? -eq 0 ] && [ "$1" = "upgrade" ]; then + echo "Unregistering old pciDriver Linux kernel module sources @PCIDRIVER_VERSION_STRING@ from dkms on Ubuntu/Debian" + dkms remove -m pciDriver -v @PCIDRIVER_VERSION_STRING@ --all + echo "Cleanup is skipped for package upgrade/downgrade/re-install on Ubuntu/Debian" + exit 0 +fi + +# If we are here pciDriver is being uninstalled. It is NOT one of upgrade, downgrade or +# re-install. + +echo "Unloading old pciDriver Linux kernel modules" +rmmod pciDriver + +echo "Unregistering pciDriver Linux kernel module sources @PCIDRIVER_VERSION_STRING@ from dkms" +dkms remove -m pciDriver -v @PCIDRIVER_VERSION_STRING@ --all +find /lib/modules -type f -name pciDriver.ko -delete +depmod -A + +rm -f /etc/udev/rules.d/60-udev_fpga.rules + +exit 0 diff --git a/foreign/pcie-driver b/foreign/pcie-driver index 1b3f3cb7..6eb66abb 160000 --- a/foreign/pcie-driver +++ b/foreign/pcie-driver @@ -1 +1 @@ -Subproject commit 1b3f3cb7725ac7a748306687fe5dda0ce1efbaf3 +Subproject commit 6eb66abb7e70e21094507f97aa14f059eb4aa7d8 diff --git a/ldconf/ldconf.mk b/ldconf/ldconf.mk new file mode 100644 index 00000000..4e691d5d --- /dev/null +++ b/ldconf/ldconf.mk @@ -0,0 +1,11 @@ +ldconf_DIR = $(LDCONF_DIR) + +ldconf_SCRIPT_IN = halcs.conf.in +ldconf_SCRIPT = halcs.conf + +# configure file +DUMMY := $(shell sed 's/@ldconf_PREFIX@/$(ldconf_PREFIX)/g' $(ldconf_DIR)/$(ldconf_SCRIPT_IN) > $(ldconf_DIR)/$(ldconf_SCRIPT)) + +# Top Makefile uses $(ldconf)_ETC_SCRIPTS this to install +# config files +ldconf_ETC_SCRIPTS = $(addprefix $(ldconf_DIR)/:,$(ldconf_SCRIPT)) diff --git a/src/apps/halcs_generic_udev/CMakeLists.txt b/src/apps/halcs_generic_udev/CMakeLists.txt index 8edd314a..f1f47845 100644 --- a/src/apps/halcs_generic_udev/CMakeLists.txt +++ b/src/apps/halcs_generic_udev/CMakeLists.txt @@ -108,13 +108,13 @@ if(BUILD_halcs_generic_udev) # udev rules install(CODE "set(input_file - \"${CMAKE_CURRENT_SOURCE_DIR}/init/systemd/etc/udev/rules.d/50-udev_fpga.rules.in\")") + \"${CMAKE_CURRENT_SOURCE_DIR}/init/systemd/etc/udev/rules.d/40-udev_fpga.rules.in\")") install(CODE "set(output_file - \"${CMAKE_CURRENT_BINARY_DIR}/init/systemd/50-udev_fpga.rules\")") + \"${CMAKE_CURRENT_BINARY_DIR}/init/systemd/40-udev_fpga.rules\")") install(SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/CMakeConfigureInstall.cmake") install(FILES - ${CMAKE_CURRENT_BINARY_DIR}/init/systemd/50-udev_fpga.rules + ${CMAKE_CURRENT_BINARY_DIR}/init/systemd/40-udev_fpga.rules DESTINATION ${UDEV_SYSCONFDIR} PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ GROUP_WRITE WORLD_READ diff --git a/src/apps/halcs_generic_udev/init/systemd/etc/udev/rules.d/50-udev_fpga.rules b/src/apps/halcs_generic_udev/init/systemd/etc/udev/rules.d/40-udev_fpga.rules similarity index 100% rename from src/apps/halcs_generic_udev/init/systemd/etc/udev/rules.d/50-udev_fpga.rules rename to src/apps/halcs_generic_udev/init/systemd/etc/udev/rules.d/40-udev_fpga.rules diff --git a/src/apps/halcs_generic_udev/init/systemd/etc/udev/rules.d/50-udev_fpga.rules.in b/src/apps/halcs_generic_udev/init/systemd/etc/udev/rules.d/40-udev_fpga.rules.in similarity index 100% rename from src/apps/halcs_generic_udev/init/systemd/etc/udev/rules.d/50-udev_fpga.rules.in rename to src/apps/halcs_generic_udev/init/systemd/etc/udev/rules.d/40-udev_fpga.rules.in diff --git a/src/apps/halcsd/CMakeLists.txt b/src/apps/halcsd/CMakeLists.txt index 56340354..eb68edaf 100644 --- a/src/apps/halcsd/CMakeLists.txt +++ b/src/apps/halcsd/CMakeLists.txt @@ -431,13 +431,13 @@ foreach(bin_target ${apps_halcsd}) # udev rules install(CODE "set(input_file - \"${CMAKE_CURRENT_SOURCE_DIR}/init/systemd/etc/udev/rules.d/60-udev_fpga.rules.in\")") + \"${CMAKE_CURRENT_SOURCE_DIR}/init/systemd/etc/udev/rules.d/50-udev_fpga.rules.in\")") install(CODE "set(output_file - \"${CMAKE_CURRENT_BINARY_DIR}/init/systemd/60-udev_fpga.rules\")") + \"${CMAKE_CURRENT_BINARY_DIR}/init/systemd/50-udev_fpga.rules\")") install(SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/CMakeConfigureInstall.cmake") install(FILES - ${CMAKE_CURRENT_BINARY_DIR}/init/systemd/60-udev_fpga.rules + ${CMAKE_CURRENT_BINARY_DIR}/init/systemd/50-udev_fpga.rules DESTINATION ${UDEV_SYSCONFDIR} PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ GROUP_WRITE WORLD_READ diff --git a/src/apps/halcsd/init/systemd/etc/udev/rules.d/50-udev_fpga.rules b/src/apps/halcsd/init/systemd/etc/udev/rules.d/50-udev_fpga.rules new file mode 100644 index 00000000..9ec2741e --- /dev/null +++ b/src/apps/halcsd/init/systemd/etc/udev/rules.d/50-udev_fpga.rules @@ -0,0 +1,5 @@ +# Permissions for the fpga devices + +KERNEL=="fpga*", ACTION=="add", OWNER="root", GROUP="users", MODE="660", PROGRAM="/usr/local/share/halcs/scripts/generate-dev-idx.sh %k", SYMLINK+="fpga/%c{1}", RUN+="/usr/bin/systemctl --no-block start halcs-ioc@%c{2}.target halcs-ioc@%c{3}.target" + +KERNEL=="fpga*", ACTION=="remove", PROGRAM="/usr/local/share/halcs/scripts/generate-dev-idx.sh %k", RUN+="/usr/bin/systemctl --no-block stop halcs@%c{2}.target halcs@%c{3}.target" diff --git a/src/apps/halcsd/init/systemd/etc/udev/rules.d/60-udev_fpga.rules.in b/src/apps/halcsd/init/systemd/etc/udev/rules.d/50-udev_fpga.rules.in similarity index 100% rename from src/apps/halcsd/init/systemd/etc/udev/rules.d/60-udev_fpga.rules.in rename to src/apps/halcsd/init/systemd/etc/udev/rules.d/50-udev_fpga.rules.in