From be3c2f061b9f82c16624662d587869271d02815c Mon Sep 17 00:00:00 2001 From: Dan Isla Date: Wed, 20 Sep 2023 16:40:00 +0000 Subject: [PATCH] Import of Selkies JS Interposer for WebRTC to Linux Joystick support - LD_PRELOAD syscall interposer to emulate gamepads. - Intercepts open() requests to /dev/input/jsX devices. - Interposer connects to selkies unix domain socket. - Interposer writes log to /tmp/selkies_js.log - Selkies WebRTC app sends Javascript gamepad events to socket. - Up to 4 simultaneous gamepads supported from WebRTC. - Build as debian package and installed to /usr/local/lib/selkies-js-interposer/joystick_interposer.so - Dev container updated to auto-load interposer. - Update to github workflow to build and publish image and release asset. - Add joystick interposer to example Dockerfile --- .devcontainer/Dockerfile | 5 + .../desktop-selkies/src/start-selkies.sh | 10 + .../build_and_publish_all_images.yaml | 17 +- .github/workflows/build_changed_images.yaml | 14 +- .github/workflows/publish_release.yaml | 72 +++- .gitignore | 3 +- .vscode/c_cpp_properties.json | 24 ++ .vscode/settings.json | 6 + .vscode/tasks.json | 10 + Dockerfile.example | 10 + addons/gst-web/src/app.js | 2 + addons/gst-web/src/gamepad.js | 84 +--- addons/gst-web/src/input.js | 14 +- addons/js-interposer/.gitignore | 1 + addons/js-interposer/Dockerfile.ubuntu_debpkg | 39 ++ addons/js-interposer/Makefile | 11 + addons/js-interposer/README.md | 27 ++ addons/js-interposer/build_ubuntu_deb.sh | 27 ++ .../js-interposer/example-button-mappings.txt | 9 + addons/js-interposer/joystick_interposer.c | 386 +++++++++++++++++ addons/js-interposer/js-interposer-test.py | 267 ++++++++++++ src/selkies_gstreamer/__main__.py | 20 +- src/selkies_gstreamer/gamepad.py | 395 ++++++++++++++++++ src/selkies_gstreamer/input_event_codes.py | 117 ++++++ src/selkies_gstreamer/webrtc_input.py | 232 +++++----- static/index.html | 24 ++ web.py | 28 ++ 27 files changed, 1653 insertions(+), 201 deletions(-) create mode 100644 .vscode/c_cpp_properties.json create mode 100644 .vscode/settings.json create mode 100644 addons/js-interposer/.gitignore create mode 100644 addons/js-interposer/Dockerfile.ubuntu_debpkg create mode 100644 addons/js-interposer/Makefile create mode 100644 addons/js-interposer/README.md create mode 100755 addons/js-interposer/build_ubuntu_deb.sh create mode 100644 addons/js-interposer/example-button-mappings.txt create mode 100644 addons/js-interposer/joystick_interposer.c create mode 100644 addons/js-interposer/js-interposer-test.py create mode 100644 src/selkies_gstreamer/gamepad.py create mode 100644 src/selkies_gstreamer/input_event_codes.py create mode 100644 static/index.html create mode 100644 web.py diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 21a39d86..8221c422 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -71,3 +71,8 @@ RUN cd /tmp && sudo apt-get update && \ sudo DEBIAN_FRONTEND=noninteractive apt-get install -y xdg-utils ./google-chrome-stable_current_amd64.deb && \ sudo rm -f google-chrome-stable_current_amd64.deb && \ xdg-settings set default-web-browser google-chrome.desktop + +# Install other dev utils +RUN sudo apt-get update && sudo apt-get install -y \ + jstest-gtk \ + netcat diff --git a/.devcontainer/features/desktop-selkies/src/start-selkies.sh b/.devcontainer/features/desktop-selkies/src/start-selkies.sh index 8138e722..2a194da4 100755 --- a/.devcontainer/features/desktop-selkies/src/start-selkies.sh +++ b/.devcontainer/features/desktop-selkies/src/start-selkies.sh @@ -37,6 +37,16 @@ export PULSE_SERVER=tcp:127.0.0.1:4713 sudo /usr/bin/pulseaudio -k >/dev/null 2>&1 sudo /usr/bin/pulseaudio --daemonize --system --verbose --log-target=file:/tmp/pulseaudio.log --realtime=true --disallow-exit -L 'module-native-protocol-tcp auth-ip-acl=127.0.0.0/8 port=4713 auth-anonymous=1' +# Create /dev/input/jsX if they don't already exists +sudo mkdir -p /dev/input +sudo touch /dev/input/{js0,js1,js2,js3} + +# If installed, add the joystick interposer to the LD_PRELOAD environment +if [[ -e /usr/local/lib/selkies-js-interposer/joystick_interposer.so ]]; then + export LD_PRELOAD=${LD_PRELOAD}:/usr/local/lib/selkies-js-interposer/joystick_interposer.so + export SDL_JOYSTICK_DEVICE=/dev/input/js0 +fi + # Start desktop environment case ${DESKTOP:-XFCE} in FLUXBOX) diff --git a/.github/workflows/build_and_publish_all_images.yaml b/.github/workflows/build_and_publish_all_images.yaml index 496449be..af132f9c 100644 --- a/.github/workflows/build_and_publish_all_images.yaml +++ b/.github/workflows/build_and_publish_all_images.yaml @@ -38,6 +38,18 @@ jobs: version_suffix: -ubuntu22.04 build_args: UBUNTU_RELEASE=22.04 source_directory: addons/gstreamer + + - name: js-interposer + version_suffix: -ubuntu20.04 + build_args: DISTRIB_RELEASE=20.04;DEBFULLNAME="$GITHUB_ACTOR";DEBEMAIL="$GITHUB_ACTOR@users.noreply.github.com";PKG_NAME=selkies-js-interposer;PKG_VERSION=0.0.0 + source_directory: addons/js-interposer + dockerfile: Dockerfile.ubuntu_debpkg + + - name: js-interposer + version_suffix: -ubuntu22.04 + build_args: DISTRIB_RELEASE=22.04;DEBFULLNAME="$GITHUB_ACTOR";DEBEMAIL="$GITHUB_ACTOR@users.noreply.github.com";PKG_NAME=selkies-js-interposer;PKG_VERSION=0.0.0 + source_directory: addons/js-interposer + dockerfile: Dockerfile.ubuntu_debpkg - name: infra-gcp-installer source_directory: infra/gce/installer-image @@ -59,6 +71,7 @@ jobs: image_name: ${{ matrix.name }} image_source_directory: ${{ matrix.source_directory }} image_version_1: $GITHUB_REF_NAME${{ matrix.version_suffix }} + dockerfile: ${{ matrix.dockerfile || 'Dockerfile' }} # Note: When modifying this job, copy modifications to all other workflows' image jobs. all_example_images: @@ -69,13 +82,13 @@ jobs: include: - name: gst-py-example version_suffix: -ubuntu20.04 - build_args: PACKAGE_VERSION=0.0.0.dev0;UBUNTU_RELEASE=20.04;GSTREAMER_BASE_IMAGE_RELEASE=${{ github.ref_name }};PY_BUILD_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/py-build:${{ github.ref_name }};WEB_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/gst-web:${{ github.ref_name }} + build_args: PACKAGE_VERSION=0.0.0.dev0;UBUNTU_RELEASE=20.04;GSTREAMER_BASE_IMAGE_RELEASE=${{ github.ref_name }};PY_BUILD_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/py-build:${{ github.ref_name }};WEB_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/gst-web:${{ github.ref_name }};JS_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/js-interposer:${{ github.ref_name }} dockerfile: Dockerfile.example source_directory: . - name: gst-py-example version_suffix: -ubuntu22.04 - build_args: PACKAGE_VERSION=0.0.0.dev0;UBUNTU_RELEASE=22.04;GSTREAMER_BASE_IMAGE_RELEASE=${{ github.ref_name }};PY_BUILD_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/py-build:${{ github.ref_name }};WEB_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/gst-web:${{ github.ref_name }} + build_args: PACKAGE_VERSION=0.0.0.dev0;UBUNTU_RELEASE=22.04;GSTREAMER_BASE_IMAGE_RELEASE=${{ github.ref_name }};PY_BUILD_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/py-build:${{ github.ref_name }};WEB_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/gst-web:${{ github.ref_name }};JS_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/js-interposer:${{ github.ref_name }} dockerfile: Dockerfile.example source_directory: . diff --git a/.github/workflows/build_changed_images.yaml b/.github/workflows/build_changed_images.yaml index 15999f3d..f3c6c31f 100644 --- a/.github/workflows/build_changed_images.yaml +++ b/.github/workflows/build_changed_images.yaml @@ -46,6 +46,16 @@ jobs: build_args: UBUNTU_RELEASE=22.04 source_directory: addons/gstreamer source_files_for_diff: addons/gstreamer + + - name: js-interposer + version_suffix: -ubuntu20.04 + build_args: DISTRIB_RELEASE=20.04;DEBFULLNAME="$GITHUB_ACTOR";DEBEMAIL="$GITHUB_ACTOR@users.noreply.github.com";PKG_NAME=selkies-js-interposer;PKG_VERSION=0.0.0 + source_directory: addons/js-interposer + + - name: js-interposer + version_suffix: -ubuntu22.04 + build_args: DISTRIB_RELEASE=22.04;DEBFULLNAME="$GITHUB_ACTOR";DEBEMAIL="$GITHUB_ACTOR@users.noreply.github.com";PKG_NAME=selkies-js-interposer;PKG_VERSION=0.0.0 + source_directory: addons/js-interposer - name: infra-gcp-installer source_directory: infra/gce/installer-image @@ -95,7 +105,7 @@ jobs: - name: gst-py-example version_suffix: -ubuntu20.04 push_image: "false" - build_args: PACKAGE_VERSION=0.0.0.dev0;UBUNTU_RELEASE=20.04;GSTREAMER_BASE_IMAGE_RELEASE=pr;PY_BUILD_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/py-build:pr;WEB_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/gst-web:pr + build_args: PACKAGE_VERSION=0.0.0.dev0;UBUNTU_RELEASE=20.04;GSTREAMER_BASE_IMAGE_RELEASE=pr${{ matrix.version_suffix }};PY_BUILD_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/py-build:pr${{ matrix.version_suffix }};WEB_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/gst-web:pr${{ matrix.version_suffix }};JS_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/js-interposer:pr${{ matrix.version_suffix }} dockerfile: Dockerfile.example source_directory: . source_files_for_diff: | @@ -110,7 +120,7 @@ jobs: - name: gst-py-example version_suffix: -ubuntu22.04 push_image: "false" - build_args: PACKAGE_VERSION=0.0.0.dev0;UBUNTU_RELEASE=22.04;GSTREAMER_BASE_IMAGE_RELEASE=pr;PY_BUILD_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/py-build:pr;WEB_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/gst-web:pr + build_args: PACKAGE_VERSION=0.0.0.dev0;UBUNTU_RELEASE=22.04;GSTREAMER_BASE_IMAGE_RELEASE=pr${{ matrix.version_suffix }};PY_BUILD_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/py-build:pr${{ matrix.version_suffix }};WEB_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/gst-web:pr${{ matrix.version_suffix }};JS_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/js-interposer:pr${{ matrix.version_suffix }} dockerfile: Dockerfile.example source_directory: . source_files_for_diff: | diff --git a/.github/workflows/publish_release.yaml b/.github/workflows/publish_release.yaml index d18f27c0..145c129b 100644 --- a/.github/workflows/publish_release.yaml +++ b/.github/workflows/publish_release.yaml @@ -52,6 +52,16 @@ jobs: version_suffix: -ubuntu22.04 build_args: UBUNTU_RELEASE=22.04 source_directory: addons/gstreamer + + - name: js-interposer + version_suffix: -ubuntu20.04 + build_args: DISTRIB_RELEASE=20.04;DEBFULLNAME="$GITHUB_ACTOR";DEBEMAIL="$GITHUB_ACTOR@users.noreply.github.com";PKG_NAME=selkies-js-interposer;PKG_VERSION=${{ needs.get_semver.outputs.semver }} + source_directory: addons/js-interposer + + - name: js-interposer + version_suffix: -ubuntu22.04 + build_args: DISTRIB_RELEASE=22.04;DEBFULLNAME="$GITHUB_ACTOR";DEBEMAIL="$GITHUB_ACTOR@users.noreply.github.com";PKG_NAME=selkies-js-interposer;PKG_VERSION=${{ needs.get_semver.outputs.semver }} + source_directory: addons/js-interposer - name: infra-gcp-installer source_directory: infra/gce/installer-image @@ -86,13 +96,13 @@ jobs: include: - name: gst-py-example version_suffix: -ubuntu20.04 - build_args: PACKAGE_VERSION=${{ needs.get_semver.outputs.semver }};UBUNTU_RELEASE=20.04;GSTREAMER_BASE_IMAGE_RELEASE=$GITHUB_REF_NAME;PY_BUILD_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/py-build:$GITHUB_REF_NAME;WEB_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/gst-web:$GITHUB_REF_NAME + build_args: PACKAGE_VERSION=${{ needs.get_semver.outputs.semver }};UBUNTU_RELEASE=20.04;GSTREAMER_BASE_IMAGE_RELEASE=$GITHUB_REF_NAME;PY_BUILD_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/py-build:$GITHUB_REF_NAME;WEB_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/gst-web:$GITHUB_REF_NAME;JS_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/js-interposer:$GITHUB_REF_NAME dockerfile: Dockerfile.example source_directory: . - name: gst-py-example version_suffix: -ubuntu22.04 - build_args: PACKAGE_VERSION=${{ needs.get_semver.outputs.semver }};UBUNTU_RELEASE=22.04;GSTREAMER_BASE_IMAGE_RELEASE=$GITHUB_REF_NAME;PY_BUILD_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/py-build:$GITHUB_REF_NAME;WEB_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/gst-web:$GITHUB_REF_NAME + build_args: PACKAGE_VERSION=${{ needs.get_semver.outputs.semver }};UBUNTU_RELEASE=22.04;GSTREAMER_BASE_IMAGE_RELEASE=$GITHUB_REF_NAME;PY_BUILD_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/py-build:$GITHUB_REF_NAME;WEB_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/gst-web:$GITHUB_REF_NAME;JS_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/js-interposer:$GITHUB_REF_NAME dockerfile: Dockerfile.example source_directory: . @@ -126,6 +136,14 @@ jobs: gst22_mimetype: ${{ steps.extract.outputs.gst22_mimetype }} gst22_name: ${{ steps.extract.outputs.gst22_name }} gst22_path: ${{ steps.extract.outputs.gst22_path }} + js20_cache_key: ${{ steps.extract.outputs.js20_cache_key }} + js20_mimetype: ${{ steps.extract.outputs.js20_mimetype }} + js20_name: ${{ steps.extract.outputs.js20_name }} + js20_path: ${{ steps.extract.outputs.js20_path }} + js22_cache_key: ${{ steps.extract.outputs.js22_cache_key }} + js22_mimetype: ${{ steps.extract.outputs.js22_mimetype }} + js22_name: ${{ steps.extract.outputs.js22_name }} + js22_path: ${{ steps.extract.outputs.js22_path }} py_cache_key: ${{ steps.extract.outputs.py_cache_key }} py_mimetype: ${{ steps.extract.outputs.py_mimetype }} py_name: ${{ steps.extract.outputs.py_name }} @@ -156,6 +174,26 @@ jobs: target_directory: /tmp target_name: selkies-gstreamer-${{ github.ref_name }}-ubuntu22.04.tgz upload_bucket_path: gs://selkies-project-releases/selkies-gstreamer/${{ github.ref_name }}/ + + - id: js20 + cache_key: js-interposer-asset-ubuntu2004 + description: JS Interposr Ubuntu 20.04 + image_tag: ghcr.io/selkies-project/selkies-gstreamer/js-interposer:${{ github.ref_name }}-ubuntu20.04 + mimetype: application/octet-stream + source_path: /opt/selkies-js-interposer_${{ needs.get_semver.outputs.semver }}.deb + target_directory: /tmp + target_name: selkies-js-interposer-${{ github.ref_name }}-ubuntu20.04.deb + upload_bucket_path: gs://selkies-project-releases/selkies-gstreamer/${{ github.ref_name }}/ + + - id: js22 + cache_key: js-interposer-asset-ubuntu2204 + description: JS Interposr Ubuntu 22.04 + image_tag: ghcr.io/selkies-project/selkies-gstreamer/js-interposer:${{ github.ref_name }}-ubuntu22.04 + mimetype: application/octet-stream + source_path: /opt/selkies-js-interposer_${{ needs.get_semver.outputs.semver }}.deb + target_directory: /tmp + target_name: selkies-js-interposer-${{ github.ref_name }}-ubuntu22.04.deb + upload_bucket_path: gs://selkies-project-releases/selkies-gstreamer/${{ github.ref_name }}/ - id: py cache_key: gst-py-asset @@ -226,6 +264,18 @@ jobs: key: ${{ needs.all_assets.outputs.gst20_cache_key }} path: ${{ needs.all_assets.outputs.gst20_path }} + - name: Ubuntu 22.04 cache read + uses: actions/cache@v3 + with: + key: ${{ needs.all_assets.outputs.gst22_cache_key }} + path: ${{ needs.all_assets.outputs.gst22_path }} + + - name: JS Interposer Ubuntu 20.04 cache read + uses: actions/cache@v3 + with: + key: ${{ needs.all_assets.outputs.js20_cache_key }} + path: ${{ needs.all_assets.outputs.js20_path }} + - name: Ubuntu 22.04 cache read uses: actions/cache@v3 with: @@ -261,6 +311,24 @@ jobs: file: ${{ needs.all_assets.outputs.gst22_path }} asset_name: ${{ needs.all_assets.outputs.gst22_name }} overwrite: true + + - name: JS Interceptor Ubuntu 20.04 upload + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + tag: ${{ github.ref }} + file: ${{ needs.all_assets.outputs.js20_path }} + asset_name: ${{ needs.all_assets.outputs.js20_name }} + overwrite: true + + - name: JS Interceptor Ubuntu 22.04 upload + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + tag: ${{ github.ref }} + file: ${{ needs.all_assets.outputs.js22_path }} + asset_name: ${{ needs.all_assets.outputs.js22_name }} + overwrite: true - name: Python upload uses: svenstaro/upload-release-action@v2 diff --git a/.gitignore b/.gitignore index 9706c6f0..86381a07 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ addons/_archive dist coturn_env src/*.egg-info -NOTES.md \ No newline at end of file +NOTES.md +**/__pycache__/ \ No newline at end of file diff --git a/.vscode/c_cpp_properties.json b/.vscode/c_cpp_properties.json new file mode 100644 index 00000000..e85d3737 --- /dev/null +++ b/.vscode/c_cpp_properties.json @@ -0,0 +1,24 @@ +{ + "env": { + "myDefaultIncludePath": [ + "/usr/include/**", + "${workspaceFolder}/**", + ] + }, + "configurations": [ + { + "name": "Dev", + "includePath": [ + "${myDefaultIncludePath}" + ], + "browse": { + "path": [ + "${myDefaultIncludePath}" + ], + "limitSymbolsToIncludedHeaders": true, + "databaseFilename": "" + } + } + ], + "version": 4 +} \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000..ba77eac9 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,6 @@ +{ + "[python]": { + "editor.defaultFormatter": "ms-python.python" + }, + "python.formatting.provider": "none" +} \ No newline at end of file diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 61d4ffb8..fd790c34 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -3,6 +3,15 @@ // for the documentation about the tasks.json format "version": "2.0.0", "tasks": [ + { + "label": "[build] build joystick interposer library", + "type": "shell", + "command": "sudo make install", + "problemMatcher": [], + "options": { + "cwd": "${workspaceFolder}/addons/js-interposer" + } + }, { "label": "[build] build python package", "type": "shell", @@ -34,6 +43,7 @@ "problemMatcher": [], "dependsOrder": "sequence", "dependsOn": [ + "[build] build joystick interposer library", "[build] build python package", "[install] re-install python package", "[run] Start selkies-gstreamer" diff --git a/Dockerfile.example b/Dockerfile.example index 994d06eb..1efb95b8 100644 --- a/Dockerfile.example +++ b/Dockerfile.example @@ -8,9 +8,11 @@ ARG GSTREAMER_BASE_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/gstreamer ARG GSTREAMER_BASE_IMAGE_RELEASE=main ARG PY_BUILD_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/py-build:main ARG WEB_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/gst-web:main +ARG JS_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/js-interposer:main FROM ${GSTREAMER_BASE_IMAGE}:${GSTREAMER_BASE_IMAGE_RELEASE}-ubuntu${UBUNTU_RELEASE} as selkies-gstreamer FROM ${PY_BUILD_IMAGE} as selkies-build FROM ${WEB_IMAGE} as selkies-web +FROM ${JS_IMAGE}-ubuntu${UBUNTU_RELEASE} as selkies-js-interposer FROM ubuntu:${UBUNTU_RELEASE} ARG UBUNTU_RELEASE @@ -120,6 +122,10 @@ COPY --from=selkies-gstreamer /opt/gstreamer ./gstreamer # Install web application COPY --from=selkies-web /usr/share/nginx/html ./gst-web +# Install Joystick Interposer +COPY --from=selkies-js-interposer /opt/*.deb /opt/selkies-js-interposer.deb +RUN apt-get install -y /opt/selkies-js-interposer.deb + # Update PWA manifest.json with application information and route. ARG PWA_APP_NAME="Selkies WebRTC" ARG PWA_APP_SHORT_NAME="selkies" @@ -174,6 +180,10 @@ export DISPLAY=:0\n\ export GST_DEBUG=*:2\n\ export GSTREAMER_PATH=/opt/gstreamer\n\ source /opt/gstreamer/gst-env\n\ +export LD_PRELOAD=/usr/local/lib/selkies-js-interposer/joystick_interposer.so\n\ +export SDL_JOYSTICK_DEVICE=/dev/input/js0\n\ +sudo mkdir -p /dev/input\n\ +sudo touch /dev/input/{js0,js1,js2,js3}\n\ Xvfb -screen :0 8192x4096x24 +extension RANDR +extension GLX +extension MIT-SHM -nolisten tcp -noreset -shmem 2>&1 >/tmp/Xvfb.log &\n\ until [[ -S /tmp/.X11-unix/X0 ]]; do sleep 1; done && echo 'X Server is ready'\n\ export PULSE_SERVER=unix:/run/pulse/native\n\ diff --git a/addons/gst-web/src/app.js b/addons/gst-web/src/app.js index c0c122b1..f59db04a 100644 --- a/addons/gst-web/src/app.js +++ b/addons/gst-web/src/app.js @@ -439,12 +439,14 @@ webrtc.onconnectionstatechange = (state) => { webrtc.ondatachannelopen = () => { // Bind gamepad connected handler. webrtc.input.ongamepadconnected = (gamepad_id) => { + webrtc._setStatus('Gamepad connected: ' + gamepad_id); app.gamepadState = "connected"; app.gamepadName = gamepad_id; } // Bind gamepad disconnect handler. webrtc.input.ongamepaddisconnected = () => { + webrtc._setStatus('Gamepad disconnected: ' + gamepad_id); app.gamepadState = "disconnected"; app.gamepadName = "none"; } diff --git a/addons/gst-web/src/gamepad.js b/addons/gst-web/src/gamepad.js index 98178a86..5c9f8568 100644 --- a/addons/gst-web/src/gamepad.js +++ b/addons/gst-web/src/gamepad.js @@ -24,56 +24,17 @@ /*eslint no-unused-vars: ["error", { "vars": "local" }]*/ -const GP_TIMEOUT = 20; +const GP_TIMEOUT = 16; const MAX_GAMEPADS = 4; -// Map of gamepad buttons to uinput buttons -// Mapping from sc-controller: -// https://github.com/kozec/sc-controller/blob/master/default_profiles/XBox%20Controller.sccprofile -const UINPUT_BTN_MAP = { - 0: 304, // BTN_GAMEPAD - 1: 305, // BTN_EAST - 2: 307, // BTN_NORTH - 3: 308, // BTN_WEST - 4: 310, // BTN_TL - 5: 311, // BTN_TR - 6: 10004, // [axis 4] ABS_Z - 7: 10005, // [axis 5] ABS_RZ - 8: 314, // BTN_SELECT - 9: 315, // BTN_START - 10: 317, // BTN_THUMBL - 11: 318, // BTN_THUMBR - 12: -20007, // [axis 17 -] ABS_HAT0Y - 13: 20007, // [axis 17 +] ABS_HAT0Y - 14: -20006, // [axis 16 -] ABS_HAT0X - 15: 20006, // [axis 16 +] ABS_HAT0X - 16: 316, // BTN_MODE -} - -// Map of gamepad axis to uinput axis -const UINPUT_AXIS_MAP = { - 0: 0, // ABS_X - 1: 1, // ABS_Y - 2: 3, // ABS_RX - 3: 4, // ABS_RY - 4: 2, // ABS_Z - 5: 5, // ABS_RZ - 6: 16, // ABS_HAT0X - 7: 17, // ABS_HAT0Y -} - class GamepadManager { - constructor(gamepad, onButton, onAxis, onDisconnect) { + constructor(gamepad, onButton, onAxis) { this.gamepad = gamepad; + this.numButtons = gamepad.buttons.length + this.numAxes = gamepad.axes.length this.onButton = onButton; this.onAxis = onAxis; - this.onDisconnect = onDisconnect; this.state = {}; - this.buttonMap = UINPUT_BTN_MAP; - this.axisMap = UINPUT_AXIS_MAP; - this.numButtons = Object.keys(this.buttonMap).length; - this.numAxes = Object.keys(this.axisMap).length; - this.interval = setInterval(() => { this._poll(); }, GP_TIMEOUT); @@ -93,20 +54,7 @@ class GamepadManager { const value = gamepads[i].buttons[x].value; if (gp.buttons[x] !== undefined && gp.buttons[x] !== value) { //eslint-disable-line no-undefined - var axisNum; - if (Math.abs(this.buttonMap[x]) > 20000) { - // translate to HAT axis. - axisNum = Math.abs(this.buttonMap[x]) - 20000; - var axisVal = Math.sign(this.buttonMap[x]) * value; - this.onAxis(i, this.axisMap[axisNum], this.normalizeAxisValue(axisNum, axisVal)); - } else if (this.buttonMap[x] > 10000) { - // translate to Z, RZ (trigger) axis - axisNum = Math.abs(this.buttonMap[x]) - 10000; - this.onAxis(i, this.axisMap[axisNum], this.normalizeAxisValue(axisNum, value)); - } else { - // send button - this.onButton(i, this.buttonMap[x], Math.round(value)); - } + this.onButton(i, x, value); } gp.buttons[x] = value; @@ -117,37 +65,17 @@ class GamepadManager { if (Math.abs(val) < 0.05) val = 0; if (gp.axes[x] !== undefined && gp.axes[x] !== val) //eslint-disable-line no-undefined - this.onAxis(i, this.axisMap[x], this.normalizeAxisValue(x, val)); + this.onAxis(i, x, val); gp.axes[x] = val; } } else if (this.state[i]) { delete this.state[i]; - this.onDisconnect(i); } } } - normalizeAxisValue(axisNum, value) { - // gamepad values are between [-1, 1], normalize them to their respective ranges. - switch (axisNum) { - case 0: - case 1: - case 2: - case 3: - case 4: - case 5: - // range: [-32768, 32767] - return Math.round(-32768 + ((value + 1) * 65535) / 2); - case 6: - case 7: - // range: [-1, 1] - return Math.round(value); - } - - } - destroy() { clearInterval(this.interval); } diff --git a/addons/gst-web/src/input.js b/addons/gst-web/src/input.js index b8405e54..3768ac95 100644 --- a/addons/gst-web/src/input.js +++ b/addons/gst-web/src/input.js @@ -364,23 +364,23 @@ class Input { } // Initialize the gamepad manager. - this.gamepadManager = new GamepadManager(event.gamepad, this._gamepadButton.bind(this), this._gamepadAxis.bind(this), this._gamepadDisconnect.bind(this)); + this.gamepadManager = new GamepadManager(event.gamepad, this._gamepadButton.bind(this), this._gamepadAxis.bind(this)); // Send joystick connect message over data channel. - this.send("js,c," + this.gamepadManager.numAxes + "," + this.gamepadManager.numButtons); + this.send("js,c," + event.gamepad.index + "," + btoa(event.gamepad.id) + "," + this.gamepadManager.numAxes + "," + this.gamepadManager.numButtons); } /** * Sends joystick disconnect command to WebRTC app. */ - _gamepadDisconnect() { - console.log("Gamepad disconnected"); + _gamepadDisconnect(event) { + console.log(`Gamepad %d disconnected`, event.gamepad.index); if (this.ongamepaddisconneceted !== null) { this.ongamepaddisconneceted(); } - this.send("js,d") + this.send("js,d," + event.gamepad.index); } /** @@ -391,7 +391,7 @@ class Input { * @param {number} val - the button value, 1 or 0 for pressed or not-pressed. */ _gamepadButton(gp_num, btn_num, val) { - this.send("js,b," + btn_num + "," + val); + this.send("js,b," + gp_num + "," + btn_num + "," + val); } /** @@ -402,7 +402,7 @@ class Input { * @param {number} val - the normalize value between [0, 255] */ _gamepadAxis(gp_num, axis_num, val) { - this.send("js,a," + axis_num + "," + val) + this.send("js,a," + gp_num + "," + axis_num + "," + val) } /** diff --git a/addons/js-interposer/.gitignore b/addons/js-interposer/.gitignore new file mode 100644 index 00000000..f1fe8d1e --- /dev/null +++ b/addons/js-interposer/.gitignore @@ -0,0 +1 @@ +*.so \ No newline at end of file diff --git a/addons/js-interposer/Dockerfile.ubuntu_debpkg b/addons/js-interposer/Dockerfile.ubuntu_debpkg new file mode 100644 index 00000000..1900a944 --- /dev/null +++ b/addons/js-interposer/Dockerfile.ubuntu_debpkg @@ -0,0 +1,39 @@ +# Example docker build: +# docker build \ +# --build-arg=DEBFULLNAME="Dan Isla" \ +# --build-arg=DEBEMAIL=dan.isla@gmail.com \ +# --build-arg=PKG_NAME=selkies-js-interposer \ +# --build-arg=PKG_VERSION=0.0.1 \ +# --build-arg=DISTRIB_RELEASE=22.04 \ +# -t selkies-js-interposer-deb:latest -f Dockerfile.ubuntu_debpkg . + +ARG DISTRIB_RELEASE=22.04 +FROM ubuntu:${DISTRIB_RELEASE} as build + +RUN apt-get update && apt-get install -y \ + build-essential && \ + rm -rf /var/lib/apt/lists/* + +ARG PKG_NAME "selkies-js-interposer" +ARG PKG_VERSION "0.0.0" +ARG DEBFULLNAME "Dan Isla" +ARG DEBEMAIL "danisla@users.noreply.github.com" + +WORKDIR /opt/build +COPY . . +RUN ./build_ubuntu_deb.sh + +ARG DISTRIB_RELEASE +FROM ubuntu:${DISTRIB_RELEASE} as test +ARG PKG_NAME +ARG PKG_VERSION + +# COPY --from=build /opt/${PKG_NAME}_${PKG_VERSION}.deb /opt/${PKG_NAME}.deb +COPY --from=build /opt/*.deb /opt/${PKG_NAME}.deb + +WORKDIR /opt + +RUN apt-get install -y --no-install-recommends \ + /opt/${PKG_NAME}.deb + +RUN stat /usr/local/lib/selkies-js-interposer/joystick_interposer.so \ No newline at end of file diff --git a/addons/js-interposer/Makefile b/addons/js-interposer/Makefile new file mode 100644 index 00000000..ba5b8f9d --- /dev/null +++ b/addons/js-interposer/Makefile @@ -0,0 +1,11 @@ +PREFIX ?= /usr/local +PKG_NAME ?= selkies-js-interposer + +all: + gcc -shared -fPIC -o joystick_interposer.so joystick_interposer.c -ldl + +install: all + mkdir -p $(PREFIX)/lib/$(PKG_NAME) + cp *.so $(PREFIX)/lib/$(PKG_NAME)/ +clean: + rm -f *.so diff --git a/addons/js-interposer/README.md b/addons/js-interposer/README.md new file mode 100644 index 00000000..f00244e0 --- /dev/null +++ b/addons/js-interposer/README.md @@ -0,0 +1,27 @@ +# Selkies Joystick (Gamepad) Interposer + +LD_PRELOAD library for interposing application calls to open a Linux joystick device and pass data via a unix domain socket. + +This allows the selkies-gstreamer WebRTC interface to pass gamepad events over the Data Channel and translate them to joystick events without requiring access to /dev/input/js0 or depend kernel modules like uinput to emulate devices. + +## Compiling + +```bash +gcc -shared -fPIC -o joystick_interposer.so joystick_interposer.c -ldl +``` + +## Testing + +1. Start the python joystick emulator: + +```bash +python3 js-interposer-test.py +``` + +This creates a new unix domain socket at `/tmp/selkies_js0.sock` and simulates joystick button presses and axis motion when a connection from the interposer is detected. + +2. Run `jstest` with the interposer library: + +```bash +LD_PRELOAD=${PWD}/joystick_interposer.so jstest /dev/input/js0 +``` \ No newline at end of file diff --git a/addons/js-interposer/build_ubuntu_deb.sh b/addons/js-interposer/build_ubuntu_deb.sh new file mode 100755 index 00000000..b741a33b --- /dev/null +++ b/addons/js-interposer/build_ubuntu_deb.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -ex + +PKG_DIR=/opt/${PKG_NAME?missing env}_${PKG_VERSION?missing env} +mkdir -p ${PKG_DIR}/DEBIAN + +DEST_DIR=${PKG_DIR}/usr/local/lib/${PKG_NAME?missing env} +mkdir -p ${DEST_DIR} + +make -e PREFIX=${PKG_DIR}/usr/local install + +PKG_SIZE=$(du -s ${DEST_DIR} | awk '{print $1}' | xargs) + +cat - > ${PKG_DIR}/DEBIAN/control < +Description: Joystick device interposer for Selkies GStreamer project +EOF + +dpkg-deb --build ${PKG_DIR} diff --git a/addons/js-interposer/example-button-mappings.txt b/addons/js-interposer/example-button-mappings.txt new file mode 100644 index 00000000..00d46534 --- /dev/null +++ b/addons/js-interposer/example-button-mappings.txt @@ -0,0 +1,9 @@ +Xbox 360 Controller Mapping +Driver version is 2.1.0. +Joystick (Microsoft Xbox Series S|X Controller) has 8 axes (X, Y, Z, Rx, Ry, Rz, Hat0X, Hat0Y) +and 11 buttons (BtnA, BtnB, BtnX, BtnY, BtnTL, BtnTR, BtnSelect, BtnStart, BtnMode, BtnThumbL, BtnThumbR). + +Stadia Controller mapping +Driver version is 2.1.0. +Joystick (Google LLC Stadia Controller rev. A) has 8 axes (X, Y, Z, Rz, Gas, Brake, Hat0X, Hat0Y) +and 15 buttons (BtnA, BtnB, BtnX, BtnY, BtnTL, BtnTR, BtnSelect, BtnStart, BtnMode, BtnThumbL, BtnThumbR, (null), (null), (null), (null)). \ No newline at end of file diff --git a/addons/js-interposer/joystick_interposer.c b/addons/js-interposer/joystick_interposer.c new file mode 100644 index 00000000..8b44d781 --- /dev/null +++ b/addons/js-interposer/joystick_interposer.c @@ -0,0 +1,386 @@ +/* +This Source Code Form is subject to the terms of the Mozilla Public +License, v. 2.0. If a copy of the MPL was not distributed with this +file, You can obtain one at https://mozilla.org/MPL/2.0/. +*/ + +/* + This is a LD_PRELOAD interposer library to connect /dev/input/jsX devices to unix domain sockets. + The unix domain sockets are used to send/receive joystick cofiguration and events. + + The open() SYSCALL is interposed to initiate the socket connection + and recieve the joystick configuration like name, button and axes mappings. + + The ioctl() SYSCALL is interposed to fake the behavior of a input event character device. + These ioctl requests were mostly reverse engineered from the joystick.h source and using the jstest command to test. + + Note that some applications list the /dev/input/* directory to discover JS devices, to solve for this, create empty files at the following paths: + sudo mkdir -p /dev/input + sudo touch /dev/input/{js0,js1,js2,js3} + + For SDL2 support, only 1 interposed joystick device is supported at a time and the following env var must be set: + export SDL_JOYSTICK_DEVICE=/dev/input/js0 +*/ + +#define _GNU_SOURCE // Required for RTLD_NEXT + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define LOG_FILE "/tmp/selkies_js.log" + +// Timeout to wait for unix domain socket to exist and connect. +#define SOCKET_CONNECT_TIMEOUT_MS 250 + +#define JS0_DEVICE_PATH "/dev/input/js0" +#define JS0_SOCKET_PATH "/tmp/selkies_js0.sock" +#define JS1_DEVICE_PATH "/dev/input/js1" +#define JS1_SOCKET_PATH "/tmp/selkies_js1.sock" +#define JS2_DEVICE_PATH "/dev/input/js2" +#define JS2_SOCKET_PATH "/tmp/selkies_js2.sock" +#define JS3_DEVICE_PATH "/dev/input/js3" +#define JS3_SOCKET_PATH "/tmp/selkies_js3.sock" +#define NUM_JS_INTERPOSERS 4 + +// Define the function signature for the original open and ioctl syscalls +typedef int (*open_func_t)(const char *pathname, int flags, ...); +typedef int (*ioctl_func_t)(int fd, unsigned long request, ...); + +// Function pointers to the original open and ioctl syscalls +static open_func_t real_open = NULL; +static ioctl_func_t real_ioctl = NULL; + +// type definition for correction struct +typedef struct js_corr js_corr_t; + +typedef struct +{ + char name[255]; // Name of the controller + uint16_t num_btns; // Number of buttons + uint16_t num_axes; // Number of axes + uint16_t btn_map[512]; // Button map + uint8_t axes_map[64]; // axes map +} js_config_t; + +// Struct for storing information about each interposed joystick device. +typedef struct +{ + char open_dev_name[255]; + char socket_path[255]; + int sockfd; + js_corr_t corr; + js_config_t js_config; +} js_interposer_t; + +static js_interposer_t interposers[NUM_JS_INTERPOSERS] = { + { + open_dev_name : JS0_DEVICE_PATH, + socket_path : JS0_SOCKET_PATH, + sockfd : -1, + corr : {}, + js_config : {}, + }, + { + open_dev_name : JS1_DEVICE_PATH, + socket_path : JS1_SOCKET_PATH, + sockfd : -1, + corr : {}, + js_config : {}, + }, + { + open_dev_name : JS2_DEVICE_PATH, + socket_path : JS2_SOCKET_PATH, + sockfd : -1, + corr : {}, + js_config : {}, + }, + { + open_dev_name : JS3_DEVICE_PATH, + socket_path : JS3_SOCKET_PATH, + sockfd : -1, + corr : {}, + js_config : {}, + }, +}; + +static FILE *log_file_fd = NULL; +void init_log_file() +{ + if (log_file_fd != NULL) + return; + log_file_fd = fopen(LOG_FILE, "a"); +} + +// Log messages from the interposer go to stderr +#define LOG_INFO "[INFO]" +#define LOG_WARN "[WARN]" +#define LOG_ERROR "[ERROR]" +static void interposer_log(const char *level, const char *msg, ...) +{ + init_log_file(); + va_list argp; + va_start(argp, msg); + fprintf(log_file_fd, "[%lu][Selkies Joystick Interposer]%s ", (unsigned long)time(NULL), level); + vfprintf(log_file_fd, msg, argp); + fprintf(log_file_fd, "\n"); + fflush(log_file_fd); + va_end(argp); +} + +void init_real_ioctl() +{ + if (real_ioctl != NULL) + return; + real_ioctl = (ioctl_func_t)dlsym(RTLD_NEXT, "ioctl"); +} + +void init_real_open() +{ + if (real_open != NULL) + return; + real_open = (open_func_t)dlsym(RTLD_NEXT, "open"); +} + +int read_config(int fd, js_config_t *js_config) +{ + ssize_t bytesRead; + + // Read config from the file descriptor + bytesRead = read(fd, js_config, sizeof(js_config_t)); + + if (bytesRead == -1) + { + interposer_log(LOG_ERROR, "Failed to read config"); + return -1; + } + else if (bytesRead == 0) + { + interposer_log(LOG_ERROR, "Failed to read config, reached socket EOF and 0 bytes read"); + // End of file reached + return -1; + } + + interposer_log(LOG_INFO, "Read config from socket:"); + interposer_log(LOG_INFO, " name: %s", js_config->name); + interposer_log(LOG_INFO, " num buttons: %d", js_config->num_btns); + interposer_log(LOG_INFO, " num axes: %d", js_config->num_axes); + + return 0; +} + +// Interposer function for open syscall +int open(const char *pathname, int flags, ...) +{ + init_real_open(); + if (real_open == NULL) + { + interposer_log("Error getting original open function: %s", dlerror()); + return -1; + } + + // Find matching device in interposer list + js_interposer_t *interposer = NULL; + for (size_t i = 0; i < NUM_JS_INTERPOSERS; i++) + { + if (strcmp(pathname, interposers[i].open_dev_name) == 0) + { + interposer = &interposers[i]; + break; + } + } + + // Call real open function if interposer was not found. + if (interposer == NULL) + { + va_list args; + va_start(args, flags); + mode_t mode = va_arg(args, mode_t); + va_end(args); + return real_open(pathname, flags, mode); + } + + interposer_log(LOG_INFO, "Intercepted open call for %s", interposer->open_dev_name); + + // Open the existing Unix socket + interposer->sockfd = socket(AF_UNIX, SOCK_STREAM, 0); + if (interposer->sockfd == -1) + { + interposer_log(LOG_ERROR, "Failed to create socket file descriptor when opening devcie: %s", interposer->open_dev_name); + return -1; + } + + struct sockaddr_un addr; + memset(&addr, 0, sizeof(struct sockaddr_un)); + addr.sun_family = AF_UNIX; + strncpy(addr.sun_path, interposer->socket_path, sizeof(addr.sun_path) - 1); + + // Wait for socket to connect. + int attempt = 0; + while (attempt++ < SOCKET_CONNECT_TIMEOUT_MS) + { + if (connect(interposer->sockfd, (struct sockaddr *)&addr, sizeof(struct sockaddr_un)) == -1) + { + // sleep for 1ms + usleep(1000); + continue; + } + break; + } + if (attempt >= SOCKET_CONNECT_TIMEOUT_MS) + { + interposer_log(LOG_ERROR, "Failed to connect to socket at %s", interposer->socket_path); + close(interposer->sockfd); + return -1; + } + + // Read the joystick config from the socket. + if (read_config(interposer->sockfd, &(interposer->js_config)) != 0) + { + interposer_log(LOG_ERROR, "Failed to read config from socket: %s", interposer->socket_path); + close(interposer->sockfd); + return -1; + } + + // Return the file descriptor of the unix socket. + return interposer->sockfd; +} + +// Interposer function for ioctl syscall on joystick device +int ioctl(int fd, unsigned long request, ...) +{ + init_real_ioctl(); + if (real_ioctl == NULL) + { + interposer_log(LOG_ERROR, "Error getting original ioctl function: %s", dlerror()); + return -1; + } + + va_list args; + va_start(args, request); + + // Get interposer for fd + js_interposer_t *interposer = NULL; + for (size_t i = 0; i < NUM_JS_INTERPOSERS; i++) + { + if (fd == interposers[i].sockfd) + { + interposer = &interposers[i]; + break; + } + } + + if (interposer == NULL) + { + // Not an ioctl on an interposed device, return real ioctl() call. + void *arg = va_arg(args, void *); + va_end(args); + return real_ioctl(fd, request, arg); + } + + if (((request >> 8) & 0xFF) != (('j'))) + { + // Not a joystick type ioctl call, return real ioctl() call. + void *arg = va_arg(args, void *); + va_end(args); + return real_ioctl(fd, request, arg); + } + + // Handle the spoofed behavior for the character device + // Cases are the second argument to the _IOR and _IOW macro call found in linux/joystick.h + // The type of joystick ioctl is the first byte in the request. + switch (request & 0xFF) + { + case 0x01: /* JSIOCGVERSION get driver version */ + interposer_log(LOG_INFO, "Intercepted ioctl request %lu -> JSIOCGVERSION", request); + uint32_t *version = va_arg(args, uint32_t *); + *version = JS_VERSION; + + va_end(args); + return 0; // 0 indicates success + + case 0x11: /* JSIOCGAXES get number of axes */ + interposer_log(LOG_INFO, "Intercepted ioctl request %lu -> JSIOCGAXES", request); + uint8_t *num_axes = va_arg(args, uint8_t *); + *num_axes = interposer->js_config.num_axes; + + va_end(args); + return 0; // 0 indicates success + + case 0x12: /* JSIOCGBUTTONS get number of buttons */ + interposer_log(LOG_INFO, "Intercepted ioctl request %lu -> JSIOCGBUTTONS", request); + uint8_t *btn_count = va_arg(args, uint8_t *); + *btn_count = interposer->js_config.num_btns; + + va_end(args); + return 0; // 0 indicates success + + case 0x13: /* JSIOCGNAME(len) get identifier string */ + interposer_log(LOG_INFO, "Intercepted ioctl request %lu -> JSIOCGNAME", request); + char *name = va_arg(args, char *); + size_t *len = va_arg(args, size_t *); + strncpy(name, interposer->js_config.name, strlen(interposer->js_config.name)); + name[strlen(interposer->js_config.name)] = '\0'; + + va_end(args); + return 0; // 0 indicates success + + case 0x21: /* JSIOCSCORR set correction values */ + interposer_log(LOG_INFO, "Intercepted ioctl request %lu -> JSIOCSCORR", request); + va_end(args); + return 0; + + case 0x22: /* JSIOCGCORR get correction values */ + interposer_log(LOG_INFO, "Intercepted ioctl request %lu -> JSIOCGCORR", request); + js_corr_t *corr = va_arg(args, js_corr_t *); + memcpy(corr, &interposer->corr, sizeof(interposer->corr)); + + va_end(args); + return 0; // 0 indicates success + + case 0x31: /* JSIOCSAXMAP set axis mapping */ + interposer_log(LOG_INFO, "Intercepted ioctl request %lu -> JSIOCSAXMAP", request); + va_end(args); + return 0; // 0 indicates success + + case 0x32: /* JSIOCGAXMAP get axis mapping */ + interposer_log(LOG_INFO, "Intercepted ioctl request %lu -> JSIOCGAXMAP", request); + uint8_t *axmap = va_arg(args, uint8_t *); + memcpy(axmap, interposer->js_config.axes_map, interposer->js_config.num_axes * sizeof(uint8_t)); + va_end(args); + return 0; // 0 indicates success + + case 0x33: /* JSIOCSBTNMAP set button mapping */ + interposer_log(LOG_INFO, "Intercepted ioctl request %lu -> JSIOCSBTNMAP", request); + va_end(args); + return 0; // 0 indicates success + + case 0x34: /* JSIOCGBTNMAP get button mapping */ + interposer_log(LOG_INFO, "Intercepted ioctl request %lu -> JSIOCGBTNMAP", request); + uint16_t *btn_map = va_arg(args, uint16_t *); + memcpy(btn_map, interposer->js_config.btn_map, interposer->js_config.num_btns * sizeof(uint16_t)); + va_end(args); + return 0; // 0 indicates success + + default: + interposer_log(LOG_WARN, "Unhandled Intercepted ioctl request %lu", request); + void *arg = va_arg(args, void *); + va_end(args); + return real_ioctl(fd, request, arg); + } + + // Handle other ioctl requests as needed + return -ENOTTY; // Not a valid ioctl request for this character device emulation +} diff --git a/addons/js-interposer/js-interposer-test.py b/addons/js-interposer/js-interposer-test.py new file mode 100644 index 00000000..04eaad92 --- /dev/null +++ b/addons/js-interposer/js-interposer-test.py @@ -0,0 +1,267 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +# write joystick events to an fd + +# import ctypes +import os +import struct +import time +import asyncio +import socket + +# Types from https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/include/uapi/linux/input-event-codes.h#n380 +BTN_MISC = 0x100 +BTN_0 = 0x100 +BTN_1 = 0x101 +BTN_2 = 0x102 +BTN_3 = 0x103 +BTN_4 = 0x104 +BTN_5 = 0x105 +BTN_6 = 0x106 +BTN_7 = 0x107 +BTN_8 = 0x108 +BTN_9 = 0x109 + +BTN_MOUSE = 0x110 +BTN_LEFT = 0x110 +BTN_RIGHT = 0x111 +BTN_MIDDLE = 0x112 +BTN_SIDE = 0x113 +BTN_EXTRA = 0x114 +BTN_FORWARD = 0x115 +BTN_BACK = 0x116 +BTN_TASK = 0x117 + +BTN_JOYSTICK = 0x120 +BTN_TRIGGER = 0x120 +BTN_THUMB = 0x121 +BTN_THUMB2 = 0x122 +BTN_TOP = 0x123 +BTN_TOP2 = 0x124 +BTN_PINKIE = 0x125 +BTN_BASE = 0x126 +BTN_BASE2 = 0x127 +BTN_BASE3 = 0x128 +BTN_BASE4 = 0x129 +BTN_BASE5 = 0x12a +BTN_BASE6 = 0x12b +BTN_DEAD = 0x12f + +BTN_GAMEPAD = 0x130 +BTN_SOUTH = 0x130 +BTN_A = BTN_SOUTH +BTN_EAST = 0x131 +BTN_B = BTN_EAST +BTN_C = 0x132 +BTN_NORTH = 0x133 +BTN_X = BTN_NORTH +BTN_WEST = 0x134 +BTN_Y = BTN_WEST +BTN_Z = 0x135 +BTN_TL = 0x136 +BTN_TR = 0x137 +BTN_TL2 = 0x138 +BTN_TR2 = 0x139 +BTN_SELECT = 0x13a +BTN_START = 0x13b +BTN_MODE = 0x13c +BTN_THUMBL = 0x13d +BTN_THUMBR = 0x13e + +ABS_X = 0x00 +ABS_Y = 0x01 +ABS_Z = 0x02 +ABS_RX = 0x03 +ABS_RY = 0x04 +ABS_RZ = 0x05 +ABS_THROTTLE = 0x06 +ABS_RUDDER = 0x07 +ABS_WHEEL = 0x08 +ABS_GAS = 0x09 +ABS_BRAKE = 0x0a +ABS_HAT0X = 0x10 +ABS_HAT0Y = 0x11 +ABS_HAT1X = 0x12 +ABS_HAT1Y = 0x13 +ABS_HAT2X = 0x14 +ABS_HAT2Y = 0x15 +ABS_HAT3X = 0x16 +ABS_HAT3Y = 0x17 +ABS_PRESSURE = 0x18 +ABS_DISTANCE = 0x19 +ABS_TILT_X = 0x1a +ABS_TILT_Y = 0x1b +ABS_TOOL_WIDTH = 0x1c +ABS_VOLUME = 0x20 +ABS_PROFILE = 0x21 + +SOCKET_PATH = "/tmp/selkies_js0.sock" + +# From /usr/include/linux/joystick.h +JS_EVENT_BUTTON = 0x01 +JS_EVENT_AXIS = 0x02 + +# Max num of buttons and axes +MAX_BTNS = 512 +MAX_AXES = 64 + +# Joystick event struct +# https://www.kernel.org/doc/Documentation/input/joystick-api.txt +# struct js_event { +# __u32 time; /* event timestamp in milliseconds */ +# __s16 value; /* value */ +# __u8 type; /* event type */ +# __u8 number; /* axis/button number */ +# }; + +# Map of client file descriptors to sockets. +clients = {} + +XPAD_CONFIG = { + "name": "Xbox 360 Controller", + "btn_map": [ + BTN_A, + BTN_B, + BTN_X, + BTN_Y, + BTN_TL, + BTN_TR, + BTN_SELECT, + BTN_START, + BTN_MODE, + BTN_THUMBL, + BTN_THUMBR + ], + "axes_map": [ + ABS_X, + ABS_Y, + ABS_Z, + ABS_RX, + ABS_RY, + ABS_RZ, + ABS_HAT0X, + ABS_HAT0Y + ] +} + + +def get_btn_event(btn_num, btn_val): + ts = int((time.time() * 1000) % 1000000000) + + # see js_event struct definition above. + # https://docs.python.org/3/library/struct.html + struct_format = 'IhBB' + event = struct.pack(struct_format, ts, btn_val, JS_EVENT_BUTTON, btn_num) + + # debug + print(struct.unpack(struct_format, event)) + + return event + + +def get_axis_event(axis_num, axis_val): + ts = int((time.time() * 1000) % 1000000000) + + # see js_event struct definition above. + # https://docs.python.org/3/library/struct.html + struct_format = 'IhBB' + event = struct.pack(struct_format, ts, axis_val, JS_EVENT_AXIS, axis_num) + + # debug + print(struct.unpack(struct_format, event)) + + return event + + +def make_config(): + cfg = XPAD_CONFIG + num_btns = len(cfg["btn_map"]) + num_axes = len(cfg["axes_map"]) + + # zero fill array to max lenth. + btn_map = [i for i in cfg["btn_map"]] + axes_map = [i for i in cfg["axes_map"]] + + btn_map[num_btns:MAX_BTNS] = [0 for i in range(num_btns, MAX_BTNS)] + axes_map[num_axes:MAX_AXES] = [0 for i in range(num_axes, MAX_AXES)] + + struct_fmt = "255sHH%dH%dB" % (MAX_BTNS, MAX_AXES) + data = struct.pack(struct_fmt, + cfg["name"].encode(), + num_btns, + num_axes, + *btn_map, + *axes_map + ) + return data + + +async def send_events(): + loop = asyncio.get_event_loop() + btn_num = 0 + btn_val = 0 + while True: + if len(clients) < 1: + await asyncio.sleep(0.1) + continue + + closed_clients = [] + for fd in clients: + try: + client = clients[fd] + print("Sending event to client: %d" % fd) + await loop.sock_sendall(client, get_btn_event(btn_num, btn_val)) + except BrokenPipeError: + print("Client %d disconnected" % fd) + closed_clients.append(fd) + client.close() + + for fd in closed_clients: + del clients[fd] + + await asyncio.sleep(0.5) + btn_val = 0 if btn_val == 1 else 1 + + if btn_val == 1: + btn_num = (btn_num + 1) % 11 + + +async def run_server(): + server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + server.bind(SOCKET_PATH) + server.listen(1) + server.setblocking(False) + + loop = asyncio.get_event_loop() + + print('Listening for connections on %s' % SOCKET_PATH) + + # Create task that sends events to all connected clients. + loop.create_task(send_events()) + + try: + while True: + client, _ = await loop.sock_accept(server) + fd = client.fileno() + print("Client connected with fd: %d" % fd) + + # Send client the joystick configuration + await loop.sock_sendall(client, make_config()) + + # Add client to dictionary to receive events. + clients[fd] = client + finally: + server.shutdown(1) + server.close() + +if __name__ == "__main__": + # remove the socket file if it already exists + try: + os.unlink(SOCKET_PATH) + except OSError: + if os.path.exists(SOCKET_PATH): + raise + + asyncio.run(run_server()) diff --git a/src/selkies_gstreamer/__main__.py b/src/selkies_gstreamer/__main__.py index 2b76bc69..f0ef6d88 100644 --- a/src/selkies_gstreamer/__main__.py +++ b/src/selkies_gstreamer/__main__.py @@ -383,9 +383,9 @@ def main(): parser.add_argument('--uinput_mouse_socket', default=os.environ.get('UINPUT_MOUSE_SOCKET', ''), help='Path to the uinput mouse socket provided by the uinput-device-plugin, if not provided uinput is used directly.') - parser.add_argument('--uinput_js_socket', - default=os.environ.get('UINPUT_JS_SOCKET', ''), - help='Path to the uinput joystick socket provided by the uinput-device-plugin, if not provided uinput is used directly.') + parser.add_argument('--js_socket_path', + default=os.environ.get('SELKIES_JS_SOCKET_PATH', '/tmp'), + help='Directory to write the selkies joystick interposer communication sockets to, default: /tmp results in socket files: /tmp/selkies_js{0-3}.sock') parser.add_argument('--encoder', default=os.environ.get('WEBRTC_ENCODER', 'x264enc'), help='GStreamer encoder plugin to use') @@ -574,7 +574,14 @@ def on_session_handler(meta=None): # Initialize the Xinput instance cursor_scale = 1.0 - webrtc_input = WebRTCInput(args.uinput_mouse_socket, args.uinput_js_socket, args.enable_clipboard.lower(), enable_cursors, cursor_size, cursor_scale, cursor_debug) + webrtc_input = WebRTCInput( + args.uinput_mouse_socket, + args.js_socket_path, + args.enable_clipboard.lower(), + enable_cursors, + cursor_size, + cursor_scale, + cursor_debug) # Handle changed cursors webrtc_input.on_cursor_change = lambda data: app.send_cursor_data(data) @@ -709,6 +716,7 @@ def on_sysmon_timer(t): # [START main_start] # Connect to the signalling server and process messages. loop = asyncio.get_event_loop() + webrtc_input.loop = loop # Initialize the signaling and web server options = argparse.Namespace() @@ -781,8 +789,9 @@ def mon_rtc_config(stun_servers, turn_servers, rtc_config): while True: asyncio.ensure_future(app.handle_bus_calls(), loop=loop) loop.run_until_complete(signalling.connect()) - loop.run_until_complete(signalling.start()) + loop.run_until_complete(signalling.start()) app.stop_pipeline() + webrtc_input.stop_js_server() except Exception as e: logger.error("Caught exception: %s" % e) sys.exit(1) @@ -790,6 +799,7 @@ def mon_rtc_config(stun_servers, turn_servers, rtc_config): app.stop_pipeline() webrtc_input.stop_clipboard() webrtc_input.stop_cursor_monitor() + webrtc_input.stop_js_server() webrtc_input.disconnect() gpu_mon.stop() hmac_turn_mon.stop() diff --git a/src/selkies_gstreamer/gamepad.py b/src/selkies_gstreamer/gamepad.py new file mode 100644 index 00000000..e96d6c32 --- /dev/null +++ b/src/selkies_gstreamer/gamepad.py @@ -0,0 +1,395 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import asyncio +import os +import struct +import socket +import time +from queue import Queue +from input_event_codes import * +from signal import ( + signal, + SIGINT, +) + +import logging +logger = logging.getLogger("selkies_gamepad") +logger.setLevel(logging.INFO) + +STANDARD_XPAD_CONFIG = { + # Browser detects xpad as 4 axes 17 button controller. + # Linux xpad has 11 buttons and 8 axes. + + "name": "Selkies Controller", + "btn_map": [ + BTN_A, # 0 + BTN_B, # 1 + BTN_X, # 2 + BTN_Y, # 3 + BTN_TL, # 4 + BTN_TR, # 5 + BTN_SELECT, # 6 + BTN_START, # 7 + BTN_MODE, # 8 + BTN_THUMBL, # 9 + BTN_THUMBR # 10 + ], + "axes_map": [ + ABS_X, # 0 + ABS_Y, # 1 + ABS_Z, # 2 + ABS_RX, # 3 + ABS_RY, # 4 + ABS_RZ, # 5 + ABS_HAT0X, # 6 + ABS_HAT0Y # 7 + ], + + + # Input mapping from javascript: + # Axis 0: Left thumbstick X + # Axis 1: Left thumbstick Y + # Axis 2: Right thumbstick X + # Axis 3: Right thumbstick Y + # Button 0: A + # Button 1: B + # Button 2: X + # Button 3: Y + # Button 4: L1 + # Button 5: R1 + # Button 6: L2 (abs) + # Button 7: R2 (abs) + # Button 8: Select + # Button 9: Start + # Button 10: L3 + # Button 11: R3 + # Button 12: DPad Up + # Button 13: DPad Down + # Button 14: DPad Left + # Button 15: DPad Right + # Button 16: Xbox Button + "mapping": { + # Remap some buttons to axes + "axes_to_btn": { + 2: (6,), # ABS_Z to L2 + 5: (7,), # ABS_RZ to R2 + 6: (15, 14), # ABS_HAT0X to DPad Left and DPad Right + 7: (13, 12) # ABS_HAT0Y to DPad Down and DPad Up + }, + # Remap axis, done in conjunction with axes_to_btn_map + "axes": { + 2: 3, # Right Thumbstick X to ABS_RX + 3: 4, # Right Thumbstick Y to ABS_RY + }, + # Because some buttons are remapped to axis, remap the other buttons to match target mapping. + "btns": { + 8: 6, # Select to BTN_SELECT + 9: 7, # Start to BTN_START + 10: 9, # L3 to BTN_THUMBL + 11: 10, # R2 to BTN_THUMBR + 16: 8 # BTN_MODE + }, + # Treat triggers as full range single axes + "trigger_axes": [ + 2, # ABS_Z + 5 # ABS_RZ + ] + } +} + +# Vendor and product IDs to configs. +XPAD_CONFIG_MAP = { + ("045e", "0b12"): STANDARD_XPAD_CONFIG, # Xbox Series S/X +} + +# From /usr/include/linux/joystick.h +JS_EVENT_BUTTON = 0x01 +JS_EVENT_AXIS = 0x02 + +# Max num of buttons and axes +MAX_BTNS = 512 +MAX_AXES = 64 + +# Range for axis values +ABS_MIN = -32767 +ABS_MAX = 32767 + +# Joystick event struct +# https://www.kernel.org/doc/Documentation/input/joystick-api.txt +# struct js_event { +# __u32 time; /* event timestamp in milliseconds */ +# __s16 value; /* value */ +# __u8 type; /* event type */ +# __u8 number; /* axis/button number */ +# }; + +def get_btn_event(btn_num, btn_val): + ts = int((time.time() * 1000) % 1000000000) + + # see js_event struct definition above. + # https://docs.python.org/3/library/struct.html + struct_format = 'IhBB' + event = struct.pack(struct_format, ts, btn_val, + JS_EVENT_BUTTON, btn_num) + + logger.debug(struct.unpack(struct_format, event)) + + return event + + +def get_axis_event(axis_num, axis_val): + ts = int((time.time() * 1000) % 1000000000) + + # see js_event struct definition above. + # https://docs.python.org/3/library/struct.html + struct_format = 'IhBB' + event = struct.pack(struct_format, ts, axis_val, + JS_EVENT_AXIS, axis_num) + + logger.debug(struct.unpack(struct_format, event)) + + return event + +def detect_gamepad_config(name): + # TODO switch mapping based on name. + return STANDARD_XPAD_CONFIG + +def get_num_btns_for_mapping(cfg): + num_mapped_btns = len( + [i for j in cfg["axes_to_btn_map"].values() for i in j]) + return len(cfg["btn_map"]) + num_mapped_btns + + +def get_num_axes_for_mapping(cfg): + return len(cfg["axes_map"]) + + +def normalize_axis_val(val): + return round(ABS_MIN + ((val+1) * (ABS_MAX - ABS_MIN)) / 2) + + +def normalize_trigger_val(val): + return round(val * (ABS_MAX - ABS_MIN)) + ABS_MIN + +class SelkiesGamepad: + def __init__(self, socket_path, loop): + self.socket_path = socket_path + self.loop = loop + + # Gamepad input mapper instance + # created when calling set_config() + self.mapper = None + self.name = None + + # socket server + self.server = None + + # Joystick config, set dynamically. + self.config = None + + # Map of client file descriptors to sockets. + self.clients = {} + + # queue of events to send. + self.events = Queue() + + # flag indicating that loop is running. + self.running = False + + def set_config(self, name, num_btns, num_axes): + self.name = name + self.config = detect_gamepad_config(name) + self.mapper = GamepadMapper(self.config, name, num_btns, num_axes) + + def __make_config(self): + ''' + Build config message to be sent to new clients. + Requires that self.config has been set first. + ''' + if not self.config: + logger.error("could not make js config becuase it has not yet been set.") + return None + + num_btns = len(self.config["btn_map"]) + num_axes = len(self.config["axes_map"]) + + # zero fill array to max lenth. + btn_map = [i for i in self.config["btn_map"]] + axes_map = [i for i in self.config["axes_map"]] + + btn_map[num_btns:MAX_BTNS] = [0 for i in range(num_btns, MAX_BTNS)] + axes_map[num_axes:MAX_AXES] = [0 for i in range(num_axes, MAX_AXES)] + + struct_fmt = "255sHH%dH%dB" % (MAX_BTNS, MAX_AXES) + data = struct.pack(struct_fmt, + self.config["name"].encode(), + num_btns, + num_axes, + *btn_map, + *axes_map + ) + return data + + async def __send_events(self): + while self.running: + if self.events.empty(): + await asyncio.sleep(0.001) + continue + while self.running and not self.events.empty(): + await self.send_event(self.events.get()) + + def send_btn(self, btn_num, btn_val): + if not self.mapper: + logger.warning("failed to send js button event because mapper was not set") + return + event = self.mapper.get_mapped_btn(btn_num, btn_val) + if event is not None: + self.events.put(event) + + def send_axis(self, axis_num, axis_val): + if not self.mapper: + logger.warning("failed to send js axis event because mapper was not set") + return + event = self.mapper.get_mapped_axis(axis_num, axis_val) + if event is not None: + self.events.put(event) + + async def send_event(self, event): + if len(self.clients) < 1: + return + + closed_clients = [] + for fd in self.clients: + try: + client = self.clients[fd] + logger.debug("Sending event to client with fd: %d" % fd) + await self.loop.sock_sendall(client, event) + except BrokenPipeError: + logger.info("Client %d disconnected" % fd) + closed_clients.append(fd) + client.close() + + for fd in closed_clients: + del self.clients[fd] + + async def setup_client(self, client): + logger.info("Sending config to client with fd: %d" % client.fileno()) + try: + config_data = self.__make_config() + if not config_data: + return + await self.loop.sock_sendall(client, config_data) + await asyncio.sleep(0.5) + # Send zero values for all buttons and axis. + for btn_num in range(len(self.config["btn_map"])): + self.send_btn(btn_num, 0) + for axis_num in range(len(self.config["axes_map"])): + self.send_axis(axis_num, 0) + + except BrokenPipeError: + client.close() + logger.info("Client disconnected") + + async def run_server(self): + try: + os.unlink(self.socket_path) + except OSError: + if os.path.exists(self.socket_path): + raise + + self.server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self.server.bind(self.socket_path) + self.server.listen(1) + self.server.setblocking(False) + + logger.info('Listening for connections on %s' % self.socket_path) + + # start loop to process event queue. + self.loop.create_task(self.__send_events()) + + self.running = True + try: + while self.running: + try: + client, _ = await asyncio.wait_for(self.loop.sock_accept(self.server), timeout=1) + except asyncio.TimeoutError: + continue + + fd = client.fileno() + logger.info("Client connected with fd: %d" % fd) + + # Send client the joystick configuration + await self.setup_client(client) + + # Add client to dictionary to receive events. + self.clients[fd] = client + finally: + self.server.close() + try: + os.unlink(self.socket_path) + except: + pass + + logger.info("Stopped gamepad socket server for %s" % self.socket_path) + + def stop_server(self): + self.running = False + self.server.close() + try: + os.unlink(self.socket_path) + except: + pass + +class GamepadMapper: + def __init__(self, config, name, num_btns, num_axes): + self.config = config + self.input_name = name + self.input_num_btns = num_btns + self.input_num_axes = num_axes + + def get_mapped_btn(self, btn_num, btn_val): + ''' + return either a button or axis event based on mapping. + ''' + + # Check to see if button is mapped to an axis + axis_num = None + axis_sign = 1 + for axis, mapping in self.config["mapping"]["axes_to_btn"].items(): + if btn_num in mapping: + axis_num = axis + if len(mapping) > 1: + axis_sign = 1 if mapping[0] == btn_num else -1 + break + + if axis_num is not None: + # Remap button to axis + # Normalize for input between -1 and 1 + axis_val = normalize_axis_val(btn_val*axis_sign) + + if axis_num in self.config["mapping"]["trigger_axes"]: + # Normalize to full range for input between 0 and 1. + axis_val = normalize_trigger_val(btn_val) + + return get_axis_event(axis_num, axis_val) + + # Perform button mapping. + mapped_btn = self.config["mapping"]["btns"].get(btn_num, btn_num) + if mapped_btn >= len(self.config["btn_map"]): + logger.error("cannot send button num %d, max num buttons is %d" % ( + mapped_btn, len(self.config["btn_map"]) - 1)) + return None + + return get_btn_event(mapped_btn, int(btn_val)) + + def get_mapped_axis(self, axis_num, axis_val): + mapped_axis = self.config["mapping"]["axes"].get(axis_num, axis_num) + if mapped_axis >= len(self.config["axes_map"]): + logger.error("cannot send axis %d, max axis num is %d" % + (mapped_axis, len(self.config["axes_map"]) - 1)) + return None + + # Normalize axis value to be within range. + return get_axis_event(mapped_axis, normalize_axis_val(axis_val)) diff --git a/src/selkies_gstreamer/input_event_codes.py b/src/selkies_gstreamer/input_event_codes.py new file mode 100644 index 00000000..40004196 --- /dev/null +++ b/src/selkies_gstreamer/input_event_codes.py @@ -0,0 +1,117 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +# Types from https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/include/uapi/linux/input-event-codes.h#n380 +''' +Event types +''' +EV_SYN = 0x00 +EV_KEY = 0x01 +EV_REL = 0x02 +EV_ABS = 0x03 +EV_MSC = 0x04 +EV_SW = 0x05 +EV_LED = 0x11 +EV_SND = 0x12 +EV_REP = 0x14 +EV_FF = 0x15 +EV_PWR = 0x16 +EV_FF_STATUS = 0x17 +EV_MAX = 0x1f +EV_CNT = (EV_MAX+1) + +''' +Synchronization events. +''' +SYN_REPORT = 0 +SYN_CONFIG = 1 +SYN_MT_REPORT = 2 +SYN_DROPPED = 3 +SYN_MAX = 0xf +SYN_CNT = (SYN_MAX+1) + +BTN_MISC = 0x100 +BTN_0 = 0x100 +BTN_1 = 0x101 +BTN_2 = 0x102 +BTN_3 = 0x103 +BTN_4 = 0x104 +BTN_5 = 0x105 +BTN_6 = 0x106 +BTN_7 = 0x107 +BTN_8 = 0x108 +BTN_9 = 0x109 + +BTN_MOUSE = 0x110 +BTN_LEFT = 0x110 +BTN_RIGHT = 0x111 +BTN_MIDDLE = 0x112 +BTN_SIDE = 0x113 +BTN_EXTRA = 0x114 +BTN_FORWARD = 0x115 +BTN_BACK = 0x116 +BTN_TASK = 0x117 + +BTN_JOYSTICK = 0x120 +BTN_TRIGGER = 0x120 +BTN_THUMB = 0x121 +BTN_THUMB2 = 0x122 +BTN_TOP = 0x123 +BTN_TOP2 = 0x124 +BTN_PINKIE = 0x125 +BTN_BASE = 0x126 +BTN_BASE2 = 0x127 +BTN_BASE3 = 0x128 +BTN_BASE4 = 0x129 +BTN_BASE5 = 0x12a +BTN_BASE6 = 0x12b +BTN_DEAD = 0x12f + +BTN_GAMEPAD = 0x130 +BTN_SOUTH = 0x130 +BTN_A = BTN_SOUTH +BTN_EAST = 0x131 +BTN_B = BTN_EAST +BTN_C = 0x132 +BTN_NORTH = 0x133 +BTN_X = BTN_NORTH +BTN_WEST = 0x134 +BTN_Y = BTN_WEST +BTN_Z = 0x135 +BTN_TL = 0x136 +BTN_TR = 0x137 +BTN_TL2 = 0x138 +BTN_TR2 = 0x139 +BTN_SELECT = 0x13a +BTN_START = 0x13b +BTN_MODE = 0x13c +BTN_THUMBL = 0x13d +BTN_THUMBR = 0x13e + +ABS_X = 0x00 +ABS_Y = 0x01 +ABS_Z = 0x02 +ABS_RX = 0x03 +ABS_RY = 0x04 +ABS_RZ = 0x05 +ABS_THROTTLE = 0x06 +ABS_RUDDER = 0x07 +ABS_WHEEL = 0x08 +ABS_GAS = 0x09 +ABS_BRAKE = 0x0a +ABS_HAT0X = 0x10 +ABS_HAT0Y = 0x11 +ABS_HAT1X = 0x12 +ABS_HAT1Y = 0x13 +ABS_HAT2X = 0x14 +ABS_HAT2Y = 0x15 +ABS_HAT3X = 0x16 +ABS_HAT3Y = 0x17 +ABS_PRESSURE = 0x18 +ABS_DISTANCE = 0x19 +ABS_TILT_X = 0x1a +ABS_TILT_Y = 0x1b +ABS_TOOL_WIDTH = 0x1c +ABS_VOLUME = 0x20 +ABS_PROFILE = 0x21 diff --git a/src/selkies_gstreamer/webrtc_input.py b/src/selkies_gstreamer/webrtc_input.py index f2900b43..16f73da4 100644 --- a/src/selkies_gstreamer/webrtc_input.py +++ b/src/selkies_gstreamer/webrtc_input.py @@ -21,6 +21,7 @@ from Xlib import display from Xlib.ext import xfixes +import asyncio import base64 import pynput import io @@ -31,38 +32,15 @@ import subprocess from subprocess import Popen, PIPE, STDOUT import socket +import struct import time from PIL import Image +from gamepad import SelkiesGamepad import logging logger = logging.getLogger("webrtc_input") logger.setLevel(logging.INFO) -JS_BTNS = ( - uinput.BTN_GAMEPAD, - uinput.BTN_EAST, - uinput.BTN_NORTH, - uinput.BTN_WEST, - uinput.BTN_TL, - uinput.BTN_TR, - uinput.BTN_SELECT, - uinput.BTN_START, - uinput.BTN_THUMBL, - uinput.BTN_THUMBR, - uinput.BTN_MODE, -) - -JS_AXES = ( - uinput.ABS_X + (-32768, 32767, 0, 0), - uinput.ABS_Y + (-32768, 32767, 0, 0), - uinput.ABS_RX + (-32768, 32767, 0, 0), - uinput.ABS_RY + (-32768, 32767, 0, 0), - uinput.ABS_Z + (-32768, 32767, 0, 0), - uinput.ABS_RZ + (-32768, 32767, 0, 0), - uinput.ABS_HAT0X + (-1, 1, 0, 0), - uinput.ABS_HAT0Y + (-1, 1, 0, 0), -) - # Local enumerations for mouse actions. MOUSE_POSITION = 10 MOUSE_MOVE = 11 @@ -91,20 +69,26 @@ }, } + class WebRTCInputError(Exception): pass class WebRTCInput: - def __init__(self, uinput_mouse_socket_path="", uinput_js_socket_path="", enable_clipboard="", enable_cursors=True, cursor_size=24, cursor_scale=1.0, cursor_debug=False): + def __init__(self, uinput_mouse_socket_path="", js_socket_path="", enable_clipboard="", enable_cursors=True, cursor_size=24, cursor_scale=1.0, cursor_debug=False): """Initializes WebRTC input instance """ + self.loop = None + self.clipboard_running = False self.uinput_mouse_socket_path = uinput_mouse_socket_path self.uinput_mouse_socket = None - self.uinput_js_socket_path = uinput_js_socket_path - self.uinput_js_socket = None + # Map of gamepad numbers to socket paths + self.js_socket_path_map = {i: os.path.join(js_socket_path, "selkies_js%d.sock" % i) for i in range(4)} + + # Map of gamepad number to SelkiesGamepad objects + self.js_map = {} self.enable_clipboard = enable_clipboard @@ -156,8 +140,10 @@ def __keyboard_connect(self): def __mouse_connect(self): if self.uinput_mouse_socket_path: # Proxy uinput mouse commands through unix domain socket. - logger.info("Connecting to uinput mouse socket: %s" % self.uinput_mouse_socket_path) - self.uinput_mouse_socket = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) + logger.info("Connecting to uinput mouse socket: %s" % + self.uinput_mouse_socket_path) + self.uinput_mouse_socket = socket.socket( + socket.AF_UNIX, socket.SOCK_DGRAM) self.mouse = pynput.mouse.Controller() @@ -170,50 +156,65 @@ def __mouse_emit(self, *args, **kwargs): if self.uinput_mouse_socket_path: cmd = {"args": args, "kwargs": kwargs} data = msgpack.packb(cmd, use_bin_type=True) - self.uinput_mouse_socket.sendto(data, self.uinput_mouse_socket_path) - - def __js_connect(self, num_axes, num_buttons): - """Connect virtual joystick + self.uinput_mouse_socket.sendto( + data, self.uinput_mouse_socket_path) - Arguments: - num_axes {integer} -- number of joystick axes - num_buttons {integer} -- number of joystick buttons + def __js_connect(self, js_num, name, num_btns, num_axes): + """Connect virtual joystick using Selkies Joystick Interposer """ + assert self.loop is not None - if self.uinput_js_socket_path: - # Proxy uinput joystick commands through unix domain socket - logger.info("Connecting to uinput joystick socket: %s" % self.uinput_js_socket_path) - self.uinput_js_socket = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) - else: - logger.info("initializing joystick with %d buttons and %d axes" % - (num_buttons, num_axes)) - axes = JS_AXES[:min(len(JS_AXES), num_axes)] - btns = JS_BTNS[:min(len(JS_BTNS), num_buttons)] - self.joystick = uinput.Device(btns + axes, - vendor=0x045e, - product=0x028e, - version=0x110, - name="Microsoft X-Box 360 pad") - - def __js_disconnect(self): - if self.joystick: - del self.joystick - - def __js_emit(self, *args, **kwargs): - if self.uinput_js_socket_path: - cmd = {"args": args, "kwargs": kwargs} - data = msgpack.packb(cmd, use_bin_type=True) - self.uinput_js_socket.sendto(data, self.uinput_js_socket_path) - else: - if self.joystick is not None: - self.joystick.emit(*args, **kwargs) + logger.info("creating selkies gamepad for js%d, name: '%s', buttons: %d, axes: %d" % (js_num, name, num_btns, num_axes)) - async def connect(self): - """Connects to X server + socket_path = self.js_socket_path_map.get(js_num, None) + if socket_path is None: + logger.error("failed to connect js%d because socket_path was not found" % js_num) + return - The target X server is determined by the DISPLAY environment variable. - """ + # Create the gamepad and button config. + js = SelkiesGamepad(socket_path, self.loop) + js.set_config(name, num_btns, num_axes) + asyncio.ensure_future(js.run_server(), loop=self.loop) + + self.js_map[js_num] = js + + def __js_disconnect(self, js_num=None): + if js_num is None: + # stop all gamepads. + for js in self.js_map.values(): + js.stop_server() + self.js_map = {} + return + + js = self.js_map.get(js_num, None) + if js is not None: + logger.info("stopping gamepad %d" % js_num) + js.stop_server() + del self.js_map[js_num] + + def __js_emit_btn(self, js_num, btn_num, btn_val): + js = self.js_map.get(js_num, None) + if js is None: + logger.error("cannot send button because js%d is not connected" % js_num) + return + + logger.debug("sending js%d button num %d with val %d" % (js_num, btn_num, btn_val)) + + js.send_btn(btn_num, btn_val) + + def __js_emit_axis(self, js_num, axis_num, axis_val): + js = self.js_map.get(js_num, None) + if js is None: + logger.error("cannot send axis because js%d is not connected" % js_num) + return + + logger.debug("sending js%d axis num %d with val %d" % (js_num, axis_num, axis_val)) + + js.send_axis(axis_num, axis_val) + + async def connect(self): + # Create connection to the X11 server provided by the DISPLAY env var. self.xdisplay = display.Display() self.__keyboard_connect() @@ -396,7 +397,8 @@ def start_clipboard(self): while self.clipboard_running: curr_data = self.read_clipboard() if curr_data and curr_data != last_data: - logger.info("sending clipboard content, length: %d" % len(curr_data)) + logger.info( + "sending clipboard content, length: %d" % len(curr_data)) self.on_clipboard_read(curr_data) last_data = curr_data time.sleep(0.5) @@ -411,7 +413,8 @@ def stop_clipboard(self): def start_cursor_monitor(self): if not self.xdisplay.has_extension('XFIXES'): if self.xdisplay.query_extension('XFIXES') is None: - logger.error('XFIXES extension not supported, cannot watch cursor changes') + logger.error( + 'XFIXES extension not supported, cannot watch cursor changes') return xfixes_version = self.xdisplay.xfixes_query_version() @@ -424,13 +427,15 @@ def start_cursor_monitor(self): self.cursor_cache = {} self.cursors_running = True screen = self.xdisplay.screen() - self.xdisplay.xfixes_select_cursor_input(screen.root, xfixes.XFixesDisplayCursorNotifyMask) + self.xdisplay.xfixes_select_cursor_input( + screen.root, xfixes.XFixesDisplayCursorNotifyMask) logger.info("watching for cursor changes") # Fetch initial cursor try: image = self.xdisplay.xfixes_get_cursor_image(screen.root) - self.cursor_cache[image.cursor_serial] = self.cursor_to_msg(image, self.cursor_scale, self.cursor_size) + self.cursor_cache[image.cursor_serial] = self.cursor_to_msg( + image, self.cursor_scale, self.cursor_size) self.on_cursor_change(self.cursor_cache[image.cursor_serial]) except Exception as e: logger.warning("exception from fetching cursor image: %s" % e) @@ -444,19 +449,24 @@ def start_cursor_monitor(self): cache_key = event.cursor_serial if cache_key in self.cursor_cache: if self.cursor_debug: - logger.warning("cursor changed to cached serial: {}".format(cache_key)) + logger.warning( + "cursor changed to cached serial: {}".format(cache_key)) else: try: # Request the cursor image. - cursor = self.xdisplay.xfixes_get_cursor_image(screen.root) + cursor = self.xdisplay.xfixes_get_cursor_image( + screen.root) # Convert cursor image and cache. - self.cursor_cache[cache_key] = self.cursor_to_msg(cursor, self.cursor_scale, self.cursor_size) + self.cursor_cache[cache_key] = self.cursor_to_msg( + cursor, self.cursor_scale, self.cursor_size) if self.cursor_debug: - logger.warning("New cursor: position={},{}, size={}x{}, length={}, xyhot={},{}, cursor_serial={}".format(cursor.x, cursor.y, cursor.width,cursor.height, len(cursor.cursor_image), cursor.xhot, cursor.yhot, cursor.cursor_serial)) + logger.warning("New cursor: position={},{}, size={}x{}, length={}, xyhot={},{}, cursor_serial={}".format( + cursor.x, cursor.y, cursor.width, cursor.height, len(cursor.cursor_image), cursor.xhot, cursor.yhot, cursor.cursor_serial)) except Exception as e: - logger.warning("exception from fetching cursor image: %s" % e) + logger.warning( + "exception from fetching cursor image: %s" % e) self.on_cursor_change(self.cursor_cache.get(cache_key)) @@ -478,7 +488,8 @@ def cursor_to_msg(self, cursor, scale=1.0, cursor_size=-1): xhot_scaled = int(cursor.xhot * scale) yhot_scaled = int(cursor.yhot * scale) - png_data_b64 = base64.b64encode(self.cursor_to_png(cursor, target_width, target_height)) + png_data_b64 = base64.b64encode( + self.cursor_to_png(cursor, target_width, target_height)) override = None if sum(cursor.cursor_image) == 0: @@ -497,10 +508,12 @@ def cursor_to_msg(self, cursor, scale=1.0, cursor_size=-1): def cursor_to_png(self, cursor, resize_width, resize_height): with io.BytesIO() as f: # Extract each component to RGBA bytes. - s = [((i >> b) & 0xFF) for i in cursor.cursor_image for b in [16,8,0,24]] + s = [((i >> b) & 0xFF) + for i in cursor.cursor_image for b in [16, 8, 0, 24]] # Create raw image from pixel bytes - im = Image.frombytes('RGBA', (cursor.width,cursor.height), bytes(s), 'raw') + im = Image.frombytes( + 'RGBA', (cursor.width, cursor.height), bytes(s), 'raw') if cursor.width != resize_width or cursor.height != resize_height: # Resize cursor to target size @@ -515,6 +528,9 @@ def cursor_to_png(self, cursor, resize_width, resize_height): debugf.write(data) return data + def stop_js_server(self): + self.__js_disconnect() + def on_message(self, msg): """Handles incoming input messages @@ -581,26 +597,27 @@ def on_message(self, msg): self.on_audio_encoder_bit_rate(bitrate) elif toks[0] == "js": # Joystick - # init: i,,, # button: b,, # axis: a,, if toks[1] == 'c': - num_axes = int(toks[2]) - num_btns = int(toks[3]) - try: - self.__js_connect(num_axes, num_btns) - except Exception as e: - logger.error("Failed to initialize joystick: %s", e) + js_num = int(toks[2]) + name = base64.b64decode(toks[3]).decode()[:255] + num_axes = int(toks[4]) + num_btns = int(toks[5]) + self.__js_connect(js_num, name, num_btns, num_axes) elif toks[1] == 'd': - self.__js_disconnect() + js_num = int(toks[2]) + self.__js_disconnect(js_num) elif toks[1] == 'b': - btn_num = int(toks[2]) - btn_on = toks[3] == '1' - self.__js_emit((uinput.BTN_0[0], btn_num), btn_on) + js_num = int(toks[2]) + btn_num = int(toks[3]) + btn_val = float(toks[4]) + self.__js_emit_btn(js_num, btn_num, btn_val) elif toks[1] == 'a': - axis_num = int(toks[2]) - axis_val = int(toks[3]) - self.__js_emit((uinput.ABS_X[0], axis_num), axis_val) + js_num = int(toks[2]) + axis_num = int(toks[3]) + axis_val = float(toks[4]) + self.__js_emit_axis(js_num, axis_num, axis_val) else: logger.warning('unhandled joystick command: %s' % toks[1]) elif toks[0] == "cr": @@ -608,12 +625,14 @@ def on_message(self, msg): if self.enable_clipboard in ["true", "out"]: data = self.read_clipboard() if data: - logger.info("read clipboard content, length: %d" % len(data)) + logger.info("read clipboard content, length: %d" % + len(data)) self.on_clipboard_read(data) else: logger.warning("no clipboard content to send") else: - logger.warning("rejecting clipboard read because outbound clipboard is disabled.") + logger.warning( + "rejecting clipboard read because outbound clipboard is disabled.") elif toks[0] == "cw": # Clipboard write if self.enable_clipboard in ["true", "in"]: @@ -621,23 +640,26 @@ def on_message(self, msg): self.write_clipboard(data) logger.info("set clipboard content, length: %d" % len(data)) else: - logger.warning("rejecting clipboard write because inbound clipboard is disabled.") + logger.warning( + "rejecting clipboard write because inbound clipboard is disabled.") elif toks[0] == "r": # resize event res = toks[1] if re.match(re.compile(r'^\d+x\d+$'), res): # Make sure resolution is divisible by 2 - w, h = [int(i) + int(i)%2 for i in res.split("x")] + w, h = [int(i) + int(i) % 2 for i in res.split("x")] self.on_resize("%dx%d" % (w, h)) else: - logger.warning("rejecting resolution change, invalid WxH resolution: %s" % res) + logger.warning( + "rejecting resolution change, invalid WxH resolution: %s" % res) elif toks[0] == "s": # scaling info scale = toks[1] if re.match(re.compile(r'^\d+(\.\d+)?$'), scale): self.on_scaling_ratio(float(scale)) else: - logger.warning("rejecting scaling change, invalid scale ratio: %s" % scale) + logger.warning( + "rejecting scaling change, invalid scale ratio: %s" % scale) elif toks[0] == "_arg_fps": # Set framerate fps = int(toks[1]) @@ -650,7 +672,7 @@ def on_message(self, msg): self.on_set_enable_audio(enabled) elif toks[0] == "_arg_resize": if len(toks) != 3: - logger.error("invalid _arg_resize commnad, expected 2 arguments ,") + logger.error("invalid _arg_resize command, expected 2 arguments ,") else: # Set resizing enabled enabled = toks[1].lower() == "true" @@ -659,10 +681,11 @@ def on_message(self, msg): res = toks[2] if re.match(re.compile(r'^\d+x\d+$'), res): # Make sure resolution is divisible by 2 - w, h = [int(i) + int(i)%2 for i in res.split("x")] + w, h = [int(i) + int(i) % 2 for i in res.split("x")] enable_res = "%dx%d" % (w, h) else: - logger.warning("rejecting enable resize with resolution change to invalid resolution: %s" % res) + logger.warning( + "rejecting enable resize with resolution change to invalid resolution: %s" % res) enable_res = None self.on_set_enable_resize(enabled, enable_res) @@ -679,6 +702,7 @@ def on_message(self, msg): latencty_ms = int(toks[1]) self.on_client_latency(latencty_ms) except: - logger.error("failed to parse latency report from client" + str(toks)) + logger.error( + "failed to parse latency report from client" + str(toks)) else: logger.info('unknown data channel message: %s' % msg) diff --git a/static/index.html b/static/index.html new file mode 100644 index 00000000..a43a8385 --- /dev/null +++ b/static/index.html @@ -0,0 +1,24 @@ + + + + + WebSocket Test + + + + + + + \ No newline at end of file diff --git a/web.py b/web.py new file mode 100644 index 00000000..063059f5 --- /dev/null +++ b/web.py @@ -0,0 +1,28 @@ +import asyncio +from aiohttp import web +import websockets +import pathlib + +async def websocket_handler(request): + ws = web.WebSocketResponse() + await ws.prepare(request) + + async for msg in ws: + if msg.type == web.WSMsgType.TEXT: + await ws.send_str("Echo: " + msg.data) + elif msg.type == web.WSMsgType.CLOSED: + break + + return ws + +app = web.Application() +app.router.add_get("/ws", websocket_handler) + +# Serve static files +current_path = pathlib.Path(__file__).parent +static_path = current_path / "static" +print(static_path) +app.router.add_static("/", path=static_path, name="static") + +if __name__ == '__main__': + web.run_app(app, port=8080) \ No newline at end of file