From 0e2765001f0067344dac2232ad175d08cef3202c Mon Sep 17 00:00:00 2001 From: Olliver Schinagl Date: Tue, 20 Jun 2023 13:08:20 +0200 Subject: [PATCH 1/3] resolve symlinks for serial ports It is not uncommon to setup smart symlinks for hotpluggable/USB adapters, so they always have a known name. PySerial however seems to refuse to follow symlinks Signed-off-by: Olliver Schinagl --- se/env.py | 2 +- se/files.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/se/env.py b/se/env.py index bd00a3e..4319e06 100644 --- a/se/env.py +++ b/se/env.py @@ -117,7 +117,7 @@ def validated_ports(ports_str): elif args.datasource != "stdin": # figure out the list of valid serial ports on this server # this is either a list of tuples or ListPortInfo objects - serial_ports = serial.tools.list_ports.comports() + serial_ports = serial.tools.list_ports.comports(include_links=True) serial_port_names = map(lambda p: p.device if isinstance(p, serial.tools.list_ports_common.ListPortInfo) else p[0], serial_ports) serialDevice = args.datasource in serial_port_names diff --git a/se/files.py b/se/files.py index f41e6ee..18d7288 100644 --- a/se/files.py +++ b/se/files.py @@ -7,6 +7,8 @@ import logging import se.logutils +from pathlib import Path + logger = logging.getLogger(__name__) socketTimeout = 120.0 @@ -52,7 +54,7 @@ def openDataSocket(ports): # open serial device def openSerial(inFileName, baudRate): - return serial.Serial(inFileName, baudrate=baudRate) + return serial.Serial(Path(inFileName).resolve(), baudrate=baudRate) def openInFile(inFileName): if inFileName == "stdin": From 77bfc3eafa12624c3b48f127d6ba75fe8cfc684b Mon Sep 17 00:00:00 2001 From: Olliver Schinagl Date: Tue, 20 Jun 2023 19:28:17 +0200 Subject: [PATCH 2/3] se/msg: Be a little bit more carefull with the hardcoded path The path replacement function tries to remove `/se` from the full path. However, if the path already contains `/se`, such as for example in `/solaredge/semonitor/se/msg.py` the path gets completely messed up. Instead, by using python's `removesuffix`, we get what we wanted. Signed-off-by: Olliver Schinagl --- se/msg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/se/msg.py b/se/msg.py index f3e23db..e5ae48f 100644 --- a/se/msg.py +++ b/se/msg.py @@ -16,7 +16,7 @@ sleepInterval = .1 # Hard coded last0503.msg file. os module used to find full path to calling msg.py file, then removes the se part so it's essentially the root of solaredge (where semonitor.py lives) -LAST0503FILE = os.path.dirname(os.path.realpath(__file__)).replace('/'+ __name__.split(".")[0], '') + "/last0503.msg" +LAST0503FILE = os.path.dirname(os.path.realpath(__file__)).removesuffix(__name__.split(".")[0]) + "last0503.msg" class SECrypto: def __init__(self, key, msg0503): From 481c55c3089319aa82ebeb527d734b642906af11 Mon Sep 17 00:00:00 2001 From: Olliver Schinagl Date: Tue, 20 Jun 2023 15:00:13 +0200 Subject: [PATCH 3/3] Containerize semonitor To be able to track dependencies better, make development easier and have a more reliable means of distributing and managing semonitor, it is good to put it isolate it within a container. Everything should still be useable as before, but care needs to be taken when running pipelines, as if these are not properly passed into the container, one is piping the outside of the container. The result would be the same, but either two docker instances would need to be run or the second command needs to be available on the host. Both are not ideal solutions. Signed-off-by: Olliver Schinagl --- .dockerignore | 3 + .github/workflows/container-build.yaml | 85 ++++++++++++++++++++++ Containerfile | 39 +++++++++++ README.Docker.md | 97 ++++++++++++++++++++++++++ scripts/semonitor.sh | 13 ++++ services/container-entrypoint.sh | 24 +++++++ 6 files changed, 261 insertions(+) create mode 100644 .dockerignore create mode 100644 .github/workflows/container-build.yaml create mode 100644 Containerfile create mode 100644 README.Docker.md create mode 100755 scripts/semonitor.sh create mode 100755 services/container-entrypoint.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..3347a18 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,3 @@ +.circleci/ +.git/ +test/ diff --git a/.github/workflows/container-build.yaml b/.github/workflows/container-build.yaml new file mode 100644 index 0000000..37a1886 --- /dev/null +++ b/.github/workflows/container-build.yaml @@ -0,0 +1,85 @@ +name: Create and publish Container image + +on: + push: + branches: + - master + tags: + - 'v*' + pull_request: + branches: + - master + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + TEST_TAG: ${{ github.repository }}:test_tag + +jobs: + build-and-push-image: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + with: + lfs: true + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to Container registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Docker meta + id: meta + uses: docker/metadata-action@v4 + with: + images: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=ref,event=branch + type=ref,event=pr + type=edge + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + flavor: | + latest=auto + + - name: Build and export + uses: docker/build-push-action@v4 + with: + context: . + file: Containerfile + load: true + tags: ${{ env.TEST_TAG }} + + - name: Test + run: > + docker container run \ + --rm \ + --user 'root:root' \ + --volume "$(pwd)/test:/usr/local/src/semonitor/test" \ + ${{ env.TEST_TAG }} \ + '/bin/sh' -c 'apk add --no-cache bash runuser && runuser --user semonitor -- ./test/test.sh' + + - name: Build and push + uses: docker/build-push-action@v4 + with: + platforms: linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6 + context: . + file: Containerfile + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/Containerfile b/Containerfile new file mode 100644 index 0000000..ba5f112 --- /dev/null +++ b/Containerfile @@ -0,0 +1,39 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +# +# Copyright (C) 2023 Olliver Schinagl + +ARG PYTHON_VERSION="3-alpine" + +FROM index.docker.io/library/python:${PYTHON_VERSION} + + +COPY requirements.txt /tmp/ + +RUN apk add --no-cache --virtual .build-deps \ + gcc \ + musl-dev \ + linux-headers \ + && \ + pip --no-cache-dir install --requirement '/tmp/requirements.txt' && \ + rm '/tmp/requirements.txt' && \ + apk del .build-deps && \ + install -d -m 775 '/usr/local/src' && \ + addgroup -S 'semonitor' && \ + adduser -D -G 'semonitor' -h '/usr/local/src/semonitor' -s '/bin/nologin' -S 'semonitor' && \ + adduser 'semonitor' 'usb' && \ + install -d -m 775 -g 'semonitor' -o 'semonitor' '/var/lib/semonitor' + +VOLUME /var/lib/semonitor + +WORKDIR /usr/local/src/semonitor/ + +COPY conversion /usr/local/src/semonitor/conversion +COPY scripts/semonitor.sh /usr/local/bin/semonitor.sh +COPY se /usr/local/src/semonitor/se +COPY semonitor.py /usr/local/src/semonitor/semonitor.py +COPY services/container-entrypoint.sh /init +COPY utilities /usr/local/src/semonitor/utilities + +USER semonitor + +ENTRYPOINT [ "/init" ] diff --git a/README.Docker.md b/README.Docker.md new file mode 100644 index 0000000..c9854eb --- /dev/null +++ b/README.Docker.md @@ -0,0 +1,97 @@ +# Running semonitor in docker +TVHeadend can be run within a Docker container. This provides isolation from +other processes by running it in a containerized environment. As this is not +and in-depth tutorial on docker, those with Docker, containers or cgroups see +[docker.com][docker]. + +This guide is not a comprehensive guide, but just lists the most basic things! + + +## Building the container +Building the container is only needed if the official one is not sufficient, +or when developing on semonitor. + +To build the image, the following can be used, where `ISSUE-123` is just used +as an example. It is important in that it will be re-used later. + +```sh +docker image build \ + --file 'Containerfile' \ + --rm \ + --tag 'semonitor:ISSUE-123' \ + './' +``` + + +## Running tests +The tests are not included in the container image, so we volume mount them to +make them available to the installed semonitor application. + +```sh +docker container run \ + --interactive \ + --rm \ + --tty \ + --user 'root:root' \ + --volume "$(pwd)/test:/usr/local/src/semonitor/test" \ + 'semonitor:ISSUE-123' \ + '/bin/sh' -c 'apk add --no-cache bash runuser && runuser --user semonitor -- ./test/test.sh' +``` + + +## Running serial device +Running the semonitor on a serial device using the official latest image can +be done as follows. + +```sh +docker container run \ + --device '/dev/ttyUSB0:/dev/solaredge' \ + --interactive \ + --rm \ + --tty \ + --volume "$(pwd)/semonitor_logs/:/semonitor/" \ + 'ghcr.io/jbuel/solaredge:latest' \ + semonitor.sh \ + -a \ + -b 115200 \ + -m \ + -o "/semonitor/json/$(date +%Y%m%d).json" \ + -r "/semonitor/rec/$(date +%Y%m%d).rec" \ + -s '1234567' \ + -t 4 \ + '/dev/solaredge' +``` + +## Using compose +It is also possible to run the container using `docker compose`. Here an +example. The device used is a udev symlinked serial to USB adapter, whith +the appropriate permissions. + +```yaml +networks: + semonitor: {} + +volumes: + semonitor: + +services: + sslh: + image: ghcr.io/jbhuel/solaredge:master + cap_drop: + - all + ulimits: + nproc: 64 + nofile: + soft: 4194304 + hard: 16777216 + devices: + - /dev/solaredge + env_file: + - common.env + volumes: + - semonitor:/var/lib/semonitor:rw + networks: + - semonitor + command: -a -b 115200 -m -o "/var/lib/semonitor/json/$(date +%Y%m%d).json" -r "/var/lib/semonitor/rec/$(date +%Y%m%d).rec" -s '1234567' -t 4 '/dev/solaredge' + restart: unless-stopped +``` diff --git a/scripts/semonitor.sh b/scripts/semonitor.sh new file mode 100755 index 0000000..c72372d --- /dev/null +++ b/scripts/semonitor.sh @@ -0,0 +1,13 @@ +#!/bin/sh +# SPDX-License-Identifier: AGPL-3.0-or-later + +set -eu +if [ -n "${DEBUG_TRACE_SH:-}" ] && \ + [ "${DEBUG_TRACE_SH:-}" != "${DEBUG_TRACE_SH#*"$(basename "${0}")"*}" ] || \ + [ "${DEBUG_TRACE_SH:-}" = 'all' ]; then + set -x +fi + +exec '/usr/local/src/solaredge/semonitor.py' "${@}" + +exit 0 diff --git a/services/container-entrypoint.sh b/services/container-entrypoint.sh new file mode 100755 index 0000000..3fffe72 --- /dev/null +++ b/services/container-entrypoint.sh @@ -0,0 +1,24 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-or-later +# +# Copyright (C) 2023 Olliver Schinagl +# +# A beginning user should be able to docker run image bash (or sh) without +# needing to learn about --entrypoint +# https://github.com/docker-library/official-images#consistency + +set -eu + +bin='semonitor.sh' + +# run command if it is not starting with a "-" and is an executable in PATH +if [ "${#}" -le 0 ] || \ + [ "${1#-}" != "${1}" ] || \ + [ -d "${1}" ] || \ + ! command -v "${1}" > '/dev/null' 2>&1; then + entrypoint='true' +fi + +exec ${entrypoint:+${bin:?}} "${@}" + +exit 0