Skip to content

Commit

Permalink
Import of Selkies JS Interposer for WebRTC to Linux Joystick support
Browse files Browse the repository at this point in the history
- LD_PRELOAD syscall interposer to emulate gamepads.
- Intercepts open() requests to /dev/input/jsX devices.
- Interposer connects to selkies unix domain socket.
- Interposer writes log to /tmp/selkies_js.log
- Selkies WebRTC app sends Javascript gamepad events to socket.
- Up to 4 simultaneous gamepads supported from WebRTC.
- Build as debian package and installed to /usr/local/lib/selkies-js-interposer/joystick_interposer.so
- Dev container updated to auto-load interposer.
- Update to github workflow to build and publish image and release asset.
- Add joystick interposer to example Dockerfile
  • Loading branch information
danisla committed Sep 22, 2023
1 parent 1a4bebd commit be3c2f0
Show file tree
Hide file tree
Showing 27 changed files with 1,653 additions and 201 deletions.
5 changes: 5 additions & 0 deletions .devcontainer/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -71,3 +71,8 @@ RUN cd /tmp && sudo apt-get update && \
sudo DEBIAN_FRONTEND=noninteractive apt-get install -y xdg-utils ./google-chrome-stable_current_amd64.deb && \
sudo rm -f google-chrome-stable_current_amd64.deb && \
xdg-settings set default-web-browser google-chrome.desktop

# Install other dev utils
RUN sudo apt-get update && sudo apt-get install -y \
jstest-gtk \
netcat
10 changes: 10 additions & 0 deletions .devcontainer/features/desktop-selkies/src/start-selkies.sh
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,16 @@ export PULSE_SERVER=tcp:127.0.0.1:4713
sudo /usr/bin/pulseaudio -k >/dev/null 2>&1
sudo /usr/bin/pulseaudio --daemonize --system --verbose --log-target=file:/tmp/pulseaudio.log --realtime=true --disallow-exit -L 'module-native-protocol-tcp auth-ip-acl=127.0.0.0/8 port=4713 auth-anonymous=1'

# Create /dev/input/jsX if they don't already exists
sudo mkdir -p /dev/input
sudo touch /dev/input/{js0,js1,js2,js3}

# If installed, add the joystick interposer to the LD_PRELOAD environment
if [[ -e /usr/local/lib/selkies-js-interposer/joystick_interposer.so ]]; then
export LD_PRELOAD=${LD_PRELOAD}:/usr/local/lib/selkies-js-interposer/joystick_interposer.so
export SDL_JOYSTICK_DEVICE=/dev/input/js0
fi

# Start desktop environment
case ${DESKTOP:-XFCE} in
FLUXBOX)
Expand Down
17 changes: 15 additions & 2 deletions .github/workflows/build_and_publish_all_images.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,18 @@ jobs:
version_suffix: -ubuntu22.04
build_args: UBUNTU_RELEASE=22.04
source_directory: addons/gstreamer

- name: js-interposer
version_suffix: -ubuntu20.04
build_args: DISTRIB_RELEASE=20.04;DEBFULLNAME="$GITHUB_ACTOR";DEBEMAIL="[email protected]";PKG_NAME=selkies-js-interposer;PKG_VERSION=0.0.0
source_directory: addons/js-interposer
dockerfile: Dockerfile.ubuntu_debpkg

- name: js-interposer
version_suffix: -ubuntu22.04
build_args: DISTRIB_RELEASE=22.04;DEBFULLNAME="$GITHUB_ACTOR";DEBEMAIL="[email protected]";PKG_NAME=selkies-js-interposer;PKG_VERSION=0.0.0
source_directory: addons/js-interposer
dockerfile: Dockerfile.ubuntu_debpkg

- name: infra-gcp-installer
source_directory: infra/gce/installer-image
Expand All @@ -59,6 +71,7 @@ jobs:
image_name: ${{ matrix.name }}
image_source_directory: ${{ matrix.source_directory }}
image_version_1: $GITHUB_REF_NAME${{ matrix.version_suffix }}
dockerfile: ${{ matrix.dockerfile || 'Dockerfile' }}

# Note: When modifying this job, copy modifications to all other workflows' image jobs.
all_example_images:
Expand All @@ -69,13 +82,13 @@ jobs:
include:
- name: gst-py-example
version_suffix: -ubuntu20.04
build_args: PACKAGE_VERSION=0.0.0.dev0;UBUNTU_RELEASE=20.04;GSTREAMER_BASE_IMAGE_RELEASE=${{ github.ref_name }};PY_BUILD_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/py-build:${{ github.ref_name }};WEB_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/gst-web:${{ github.ref_name }}
build_args: PACKAGE_VERSION=0.0.0.dev0;UBUNTU_RELEASE=20.04;GSTREAMER_BASE_IMAGE_RELEASE=${{ github.ref_name }};PY_BUILD_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/py-build:${{ github.ref_name }};WEB_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/gst-web:${{ github.ref_name }};JS_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/js-interposer:${{ github.ref_name }}
dockerfile: Dockerfile.example
source_directory: .

- name: gst-py-example
version_suffix: -ubuntu22.04
build_args: PACKAGE_VERSION=0.0.0.dev0;UBUNTU_RELEASE=22.04;GSTREAMER_BASE_IMAGE_RELEASE=${{ github.ref_name }};PY_BUILD_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/py-build:${{ github.ref_name }};WEB_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/gst-web:${{ github.ref_name }}
build_args: PACKAGE_VERSION=0.0.0.dev0;UBUNTU_RELEASE=22.04;GSTREAMER_BASE_IMAGE_RELEASE=${{ github.ref_name }};PY_BUILD_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/py-build:${{ github.ref_name }};WEB_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/gst-web:${{ github.ref_name }};JS_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/js-interposer:${{ github.ref_name }}
dockerfile: Dockerfile.example
source_directory: .

Expand Down
14 changes: 12 additions & 2 deletions .github/workflows/build_changed_images.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,16 @@ jobs:
build_args: UBUNTU_RELEASE=22.04
source_directory: addons/gstreamer
source_files_for_diff: addons/gstreamer

- name: js-interposer
version_suffix: -ubuntu20.04
build_args: DISTRIB_RELEASE=20.04;DEBFULLNAME="$GITHUB_ACTOR";DEBEMAIL="[email protected]";PKG_NAME=selkies-js-interposer;PKG_VERSION=0.0.0
source_directory: addons/js-interposer

- name: js-interposer
version_suffix: -ubuntu22.04
build_args: DISTRIB_RELEASE=22.04;DEBFULLNAME="$GITHUB_ACTOR";DEBEMAIL="[email protected]";PKG_NAME=selkies-js-interposer;PKG_VERSION=0.0.0
source_directory: addons/js-interposer

- name: infra-gcp-installer
source_directory: infra/gce/installer-image
Expand Down Expand Up @@ -95,7 +105,7 @@ jobs:
- name: gst-py-example
version_suffix: -ubuntu20.04
push_image: "false"
build_args: PACKAGE_VERSION=0.0.0.dev0;UBUNTU_RELEASE=20.04;GSTREAMER_BASE_IMAGE_RELEASE=pr;PY_BUILD_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/py-build:pr;WEB_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/gst-web:pr
build_args: PACKAGE_VERSION=0.0.0.dev0;UBUNTU_RELEASE=20.04;GSTREAMER_BASE_IMAGE_RELEASE=pr${{ matrix.version_suffix }};PY_BUILD_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/py-build:pr${{ matrix.version_suffix }};WEB_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/gst-web:pr${{ matrix.version_suffix }};JS_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/js-interposer:pr${{ matrix.version_suffix }}
dockerfile: Dockerfile.example
source_directory: .
source_files_for_diff: |
Expand All @@ -110,7 +120,7 @@ jobs:
- name: gst-py-example
version_suffix: -ubuntu22.04
push_image: "false"
build_args: PACKAGE_VERSION=0.0.0.dev0;UBUNTU_RELEASE=22.04;GSTREAMER_BASE_IMAGE_RELEASE=pr;PY_BUILD_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/py-build:pr;WEB_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/gst-web:pr
build_args: PACKAGE_VERSION=0.0.0.dev0;UBUNTU_RELEASE=22.04;GSTREAMER_BASE_IMAGE_RELEASE=pr${{ matrix.version_suffix }};PY_BUILD_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/py-build:pr${{ matrix.version_suffix }};WEB_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/gst-web:pr${{ matrix.version_suffix }};JS_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/js-interposer:pr${{ matrix.version_suffix }}
dockerfile: Dockerfile.example
source_directory: .
source_files_for_diff: |
Expand Down
72 changes: 70 additions & 2 deletions .github/workflows/publish_release.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,16 @@ jobs:
version_suffix: -ubuntu22.04
build_args: UBUNTU_RELEASE=22.04
source_directory: addons/gstreamer

- name: js-interposer
version_suffix: -ubuntu20.04
build_args: DISTRIB_RELEASE=20.04;DEBFULLNAME="$GITHUB_ACTOR";DEBEMAIL="[email protected]";PKG_NAME=selkies-js-interposer;PKG_VERSION=${{ needs.get_semver.outputs.semver }}
source_directory: addons/js-interposer

- name: js-interposer
version_suffix: -ubuntu22.04
build_args: DISTRIB_RELEASE=22.04;DEBFULLNAME="$GITHUB_ACTOR";DEBEMAIL="[email protected]";PKG_NAME=selkies-js-interposer;PKG_VERSION=${{ needs.get_semver.outputs.semver }}
source_directory: addons/js-interposer

- name: infra-gcp-installer
source_directory: infra/gce/installer-image
Expand Down Expand Up @@ -86,13 +96,13 @@ jobs:
include:
- name: gst-py-example
version_suffix: -ubuntu20.04
build_args: PACKAGE_VERSION=${{ needs.get_semver.outputs.semver }};UBUNTU_RELEASE=20.04;GSTREAMER_BASE_IMAGE_RELEASE=$GITHUB_REF_NAME;PY_BUILD_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/py-build:$GITHUB_REF_NAME;WEB_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/gst-web:$GITHUB_REF_NAME
build_args: PACKAGE_VERSION=${{ needs.get_semver.outputs.semver }};UBUNTU_RELEASE=20.04;GSTREAMER_BASE_IMAGE_RELEASE=$GITHUB_REF_NAME;PY_BUILD_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/py-build:$GITHUB_REF_NAME;WEB_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/gst-web:$GITHUB_REF_NAME;JS_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/js-interposer:$GITHUB_REF_NAME
dockerfile: Dockerfile.example
source_directory: .

- name: gst-py-example
version_suffix: -ubuntu22.04
build_args: PACKAGE_VERSION=${{ needs.get_semver.outputs.semver }};UBUNTU_RELEASE=22.04;GSTREAMER_BASE_IMAGE_RELEASE=$GITHUB_REF_NAME;PY_BUILD_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/py-build:$GITHUB_REF_NAME;WEB_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/gst-web:$GITHUB_REF_NAME
build_args: PACKAGE_VERSION=${{ needs.get_semver.outputs.semver }};UBUNTU_RELEASE=22.04;GSTREAMER_BASE_IMAGE_RELEASE=$GITHUB_REF_NAME;PY_BUILD_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/py-build:$GITHUB_REF_NAME;WEB_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/gst-web:$GITHUB_REF_NAME;JS_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/js-interposer:$GITHUB_REF_NAME
dockerfile: Dockerfile.example
source_directory: .

Expand Down Expand Up @@ -126,6 +136,14 @@ jobs:
gst22_mimetype: ${{ steps.extract.outputs.gst22_mimetype }}
gst22_name: ${{ steps.extract.outputs.gst22_name }}
gst22_path: ${{ steps.extract.outputs.gst22_path }}
js20_cache_key: ${{ steps.extract.outputs.js20_cache_key }}
js20_mimetype: ${{ steps.extract.outputs.js20_mimetype }}
js20_name: ${{ steps.extract.outputs.js20_name }}
js20_path: ${{ steps.extract.outputs.js20_path }}
js22_cache_key: ${{ steps.extract.outputs.js22_cache_key }}
js22_mimetype: ${{ steps.extract.outputs.js22_mimetype }}
js22_name: ${{ steps.extract.outputs.js22_name }}
js22_path: ${{ steps.extract.outputs.js22_path }}
py_cache_key: ${{ steps.extract.outputs.py_cache_key }}
py_mimetype: ${{ steps.extract.outputs.py_mimetype }}
py_name: ${{ steps.extract.outputs.py_name }}
Expand Down Expand Up @@ -156,6 +174,26 @@ jobs:
target_directory: /tmp
target_name: selkies-gstreamer-${{ github.ref_name }}-ubuntu22.04.tgz
upload_bucket_path: gs://selkies-project-releases/selkies-gstreamer/${{ github.ref_name }}/

- id: js20
cache_key: js-interposer-asset-ubuntu2004
description: JS Interposr Ubuntu 20.04
image_tag: ghcr.io/selkies-project/selkies-gstreamer/js-interposer:${{ github.ref_name }}-ubuntu20.04
mimetype: application/octet-stream
source_path: /opt/selkies-js-interposer_${{ needs.get_semver.outputs.semver }}.deb
target_directory: /tmp
target_name: selkies-js-interposer-${{ github.ref_name }}-ubuntu20.04.deb
upload_bucket_path: gs://selkies-project-releases/selkies-gstreamer/${{ github.ref_name }}/

- id: js22
cache_key: js-interposer-asset-ubuntu2204
description: JS Interposr Ubuntu 22.04
image_tag: ghcr.io/selkies-project/selkies-gstreamer/js-interposer:${{ github.ref_name }}-ubuntu22.04
mimetype: application/octet-stream
source_path: /opt/selkies-js-interposer_${{ needs.get_semver.outputs.semver }}.deb
target_directory: /tmp
target_name: selkies-js-interposer-${{ github.ref_name }}-ubuntu22.04.deb
upload_bucket_path: gs://selkies-project-releases/selkies-gstreamer/${{ github.ref_name }}/

- id: py
cache_key: gst-py-asset
Expand Down Expand Up @@ -226,6 +264,18 @@ jobs:
key: ${{ needs.all_assets.outputs.gst20_cache_key }}
path: ${{ needs.all_assets.outputs.gst20_path }}

- name: Ubuntu 22.04 cache read
uses: actions/cache@v3
with:
key: ${{ needs.all_assets.outputs.gst22_cache_key }}
path: ${{ needs.all_assets.outputs.gst22_path }}

- name: JS Interposer Ubuntu 20.04 cache read
uses: actions/cache@v3
with:
key: ${{ needs.all_assets.outputs.js20_cache_key }}
path: ${{ needs.all_assets.outputs.js20_path }}

- name: Ubuntu 22.04 cache read
uses: actions/cache@v3
with:
Expand Down Expand Up @@ -261,6 +311,24 @@ jobs:
file: ${{ needs.all_assets.outputs.gst22_path }}
asset_name: ${{ needs.all_assets.outputs.gst22_name }}
overwrite: true

- name: JS Interceptor Ubuntu 20.04 upload
uses: svenstaro/upload-release-action@v2
with:
repo_token: ${{ secrets.GITHUB_TOKEN }}
tag: ${{ github.ref }}
file: ${{ needs.all_assets.outputs.js20_path }}
asset_name: ${{ needs.all_assets.outputs.js20_name }}
overwrite: true

- name: JS Interceptor Ubuntu 22.04 upload
uses: svenstaro/upload-release-action@v2
with:
repo_token: ${{ secrets.GITHUB_TOKEN }}
tag: ${{ github.ref }}
file: ${{ needs.all_assets.outputs.js22_path }}
asset_name: ${{ needs.all_assets.outputs.js22_name }}
overwrite: true

- name: Python upload
uses: svenstaro/upload-release-action@v2
Expand Down
3 changes: 2 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,5 @@ addons/_archive
dist
coturn_env
src/*.egg-info
NOTES.md
NOTES.md
**/__pycache__/
24 changes: 24 additions & 0 deletions .vscode/c_cpp_properties.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
{
"env": {
"myDefaultIncludePath": [
"/usr/include/**",
"${workspaceFolder}/**",
]
},
"configurations": [
{
"name": "Dev",
"includePath": [
"${myDefaultIncludePath}"
],
"browse": {
"path": [
"${myDefaultIncludePath}"
],
"limitSymbolsToIncludedHeaders": true,
"databaseFilename": ""
}
}
],
"version": 4
}
6 changes: 6 additions & 0 deletions .vscode/settings.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
{
"[python]": {
"editor.defaultFormatter": "ms-python.python"
},
"python.formatting.provider": "none"
}
10 changes: 10 additions & 0 deletions .vscode/tasks.json
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,15 @@
// for the documentation about the tasks.json format
"version": "2.0.0",
"tasks": [
{
"label": "[build] build joystick interposer library",
"type": "shell",
"command": "sudo make install",
"problemMatcher": [],
"options": {
"cwd": "${workspaceFolder}/addons/js-interposer"
}
},
{
"label": "[build] build python package",
"type": "shell",
Expand Down Expand Up @@ -34,6 +43,7 @@
"problemMatcher": [],
"dependsOrder": "sequence",
"dependsOn": [
"[build] build joystick interposer library",
"[build] build python package",
"[install] re-install python package",
"[run] Start selkies-gstreamer"
Expand Down
10 changes: 10 additions & 0 deletions Dockerfile.example
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,11 @@ ARG GSTREAMER_BASE_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/gstreamer
ARG GSTREAMER_BASE_IMAGE_RELEASE=main
ARG PY_BUILD_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/py-build:main
ARG WEB_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/gst-web:main
ARG JS_IMAGE=ghcr.io/selkies-project/selkies-gstreamer/js-interposer:main
FROM ${GSTREAMER_BASE_IMAGE}:${GSTREAMER_BASE_IMAGE_RELEASE}-ubuntu${UBUNTU_RELEASE} as selkies-gstreamer
FROM ${PY_BUILD_IMAGE} as selkies-build
FROM ${WEB_IMAGE} as selkies-web
FROM ${JS_IMAGE}-ubuntu${UBUNTU_RELEASE} as selkies-js-interposer
FROM ubuntu:${UBUNTU_RELEASE}
ARG UBUNTU_RELEASE

Expand Down Expand Up @@ -120,6 +122,10 @@ COPY --from=selkies-gstreamer /opt/gstreamer ./gstreamer
# Install web application
COPY --from=selkies-web /usr/share/nginx/html ./gst-web

# Install Joystick Interposer
COPY --from=selkies-js-interposer /opt/*.deb /opt/selkies-js-interposer.deb
RUN apt-get install -y /opt/selkies-js-interposer.deb

# Update PWA manifest.json with application information and route.
ARG PWA_APP_NAME="Selkies WebRTC"
ARG PWA_APP_SHORT_NAME="selkies"
Expand Down Expand Up @@ -174,6 +180,10 @@ export DISPLAY=:0\n\
export GST_DEBUG=*:2\n\
export GSTREAMER_PATH=/opt/gstreamer\n\
source /opt/gstreamer/gst-env\n\
export LD_PRELOAD=/usr/local/lib/selkies-js-interposer/joystick_interposer.so\n\
export SDL_JOYSTICK_DEVICE=/dev/input/js0\n\
sudo mkdir -p /dev/input\n\
sudo touch /dev/input/{js0,js1,js2,js3}\n\
Xvfb -screen :0 8192x4096x24 +extension RANDR +extension GLX +extension MIT-SHM -nolisten tcp -noreset -shmem 2>&1 >/tmp/Xvfb.log &\n\
until [[ -S /tmp/.X11-unix/X0 ]]; do sleep 1; done && echo 'X Server is ready'\n\
export PULSE_SERVER=unix:/run/pulse/native\n\
Expand Down
2 changes: 2 additions & 0 deletions addons/gst-web/src/app.js
Original file line number Diff line number Diff line change
Expand Up @@ -439,12 +439,14 @@ webrtc.onconnectionstatechange = (state) => {
webrtc.ondatachannelopen = () => {
// Bind gamepad connected handler.
webrtc.input.ongamepadconnected = (gamepad_id) => {
webrtc._setStatus('Gamepad connected: ' + gamepad_id);
app.gamepadState = "connected";
app.gamepadName = gamepad_id;
}

// Bind gamepad disconnect handler.
webrtc.input.ongamepaddisconnected = () => {
webrtc._setStatus('Gamepad disconnected: ' + gamepad_id);
app.gamepadState = "disconnected";
app.gamepadName = "none";
}
Expand Down
Loading

0 comments on commit be3c2f0

Please sign in to comment.