From 8fb0ffe6a7fc0449c12aaae4891bf72c2086c3ac Mon Sep 17 00:00:00 2001 From: Dmitri Khokhlov Date: Wed, 15 Feb 2023 21:23:30 +0000 Subject: [PATCH] CI build - added docker based local build on linux - added Appveyor CI build --- Dockerfile | 37 +++++++++++ Jenkinsfile | 12 ++-- Makefile | 40 ++++++++++-- appveyor.yml | 30 +++++++++ pyproject.toml | 170 ++++++++++++++++++++++++------------------------- 5 files changed, 195 insertions(+), 94 deletions(-) create mode 100644 Dockerfile create mode 100644 appveyor.yml diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..8518982 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,37 @@ +FROM ubuntu:focal +ARG USER_ID=5000 +ARG GROUP_ID=5000 +USER root +ARG DEBIAN_FRONTEND=noninteractive +RUN apt-get update +RUN apt-get install -y wget software-properties-common gnupg2 winbind xvfb +RUN dpkg --add-architecture i386 +RUN wget -nc https://dl.winehq.org/wine-builds/winehq.key +RUN apt-key add winehq.key +RUN add-apt-repository 'deb https://dl.winehq.org/wine-builds/ubuntu/ focal main' +RUN apt-get update +RUN apt-get install -y winehq-stable +RUN apt-get install -y winetricks +RUN apt-get clean -y +RUN apt-get autoremove -y +ENV WINEDEBUG=-all +ENV WINEPREFIX=/home/logdna/.wine +ENV DISPLAY= +RUN groupadd -g $GROUP_ID logdna && useradd -m -u $USER_ID -g logdna logdna +USER logdna +WORKDIR /home/logdna +RUN mkdir -p ./workdir && chmod 777 ./workdir +RUN wineboot -u +RUN winetricks cmd +RUN winetricks win10 +ARG MINICONDA_INSTALL=Miniconda3-py310_23.1.0-1-Windows-x86_64.exe +RUN wget -q https://repo.anaconda.com/miniconda/$MINICONDA_INSTALL && \ + xvfb-run wine ./$MINICONDA_INSTALL /InstallationType=JustMe /RegisterPython=1 /S /D=%UserProfile%\\Miniconda3 && \ + rm $MINICONDA_INSTALL +RUN rm Miniconda3-py310_23.1.0-1-Windows-x86_64.exe +ENV WINEPATH=C:\\users\\logdna\\miniconda3\\condabin;C:\\users\\logdna\\miniconda3\\Scripts;C:\\users\\logdna\\miniconda3\\Library\\bin;C:\\users\\logdna\\miniconda3\\Library\\usr\\bin +RUN wine cmd /C conda.bat init cmd.exe +RUN wine cmd /C conda.bat install -y pip git make -c conda-forge +RUN wine cmd /C pip.exe install poetry +RUN wine cmd /C git.exe config --global --add safe.directory ./workdir +WORKDIR /home/logdna/workdir diff --git a/Jenkinsfile b/Jenkinsfile index 2d482ef..271f01e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -23,11 +23,13 @@ pipeline { stages { stage('Test') { - + steps { + sh 'echo done' + } +/* TODO - run on Windows agent steps { sh 'make install lint test' } - post { always { junit 'coverage/test.xml' @@ -41,10 +43,11 @@ pipeline { ] } } +*/ } +/* TODO - run on Windows agent stage('Release') { - stages { stage('dry run') { when { @@ -68,7 +71,6 @@ pipeline { } stage('publish') { - environment { GH_TOKEN = credentials('github-api-token') PYPI_TOKEN = credentials('pypi-token') @@ -81,11 +83,13 @@ pipeline { changelog '\\[skip ci\\]' } } + steps { sh 'make release' } } } } +*/ } } diff --git a/Makefile b/Makefile index 8a883fa..7b428d7 100644 --- a/Makefile +++ b/Makefile @@ -3,11 +3,45 @@ # Source in repository specific environment variables include .config.mk +# Define commands via docker +DOCKER = docker +DOCKER_RUN := $(DOCKER) run --rm -i +WORKDIR :=/workdir +DOCKER_COMMAND := $(DOCKER_RUN) -v $(PWD):$(WORKDIR):Z -w $(WORKDIR) -u $(shell id -u):$(shell id -g) \ + -e XDG_CONFIG_HOME=$(WORKDIR)/build/xdg_cache \ + -e XDG_CACHE_HOME=$(WORKDIR)/build/xdg_cache \ + -e POETRY_CACHE_DIR=$(WORKDIR)/build/poetry_cache \ + -e POETRY_VIRTUALENV_IN_PROJECT=true \ + -e PYPI_TOKEN \ + -e GH_TOKEN \ + -e JENKINS_URL \ + -e BRANCH_NAME \ + -e CHANGE_ID \ + -e GIT_AUTHOR_NAME \ + -e GIT_AUTHOR_EMAIL \ + -e GIT_COMMITTER_NAME \ + -e GIT_COMMITTER_EMAIL \ + logdna-poetry:local + +ifeq ($(OS),Windows_NT) +# Windows host, that was configured per readme.md POETRY_COMMAND := poetry +.PHONY:build-image + echo skip +else +# x86_64 docker host +POETRY_COMMAND := $(DOCKER_COMMAND) wine poetry + +.PHONY:build-image +build-image: + DOCKER_BUILDKIT=1 $(DOCKER) build --progress=plain --build-arg USER_ID=$(shell id -u) --build-arg GROUP_ID=$(shell id -g) -t logdna-poetry:local . +endif + # Exports the variables for shell use export + # This helper function makes debugging much easier. .PHONY:debug-% debug-%: ## Debug a variable by calling `make debug-VARIABLE` @@ -18,10 +52,6 @@ debug-%: ## Debug a variable by calling `make debug-VARIABLE` help: ## Show this help, includes list of all actions. @awk 'BEGIN {FS = ":.*?## "}; /^.+: .*?## / && !/awk/ {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' ${MAKEFILE_LIST} -.PHONY:run -run: install ## purge build time artifacts - $(DOCKER_COMMAND) bash - .PHONY:clean clean: ## purge build time artifacts rm -rf dist/ build/ coverage/ pypoetry/ pip/ **/__pycache__/ .pytest_cache/ .cache .coverage @@ -31,7 +61,7 @@ changelog: install ## print the next version of the change log to stdout $(POETRY_COMMAND) run semantic-release changelog --unreleased .PHONY:install -install: ## install development and build time dependencies +install: build-image ## install development and build time dependencies $(POETRY_COMMAND) install --no-interaction .PHONY:lint diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 0000000..cb9dee8 --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,30 @@ +os: Visual Studio 2019 + +# Disable msbuild +build: false + +skip_branch_with_pr: true + +environment: + PATH: C:\Users\appveyor\Miniconda3\condabin;C:\Users\appveyor\Miniconda3\Scripts;%PATH% + +cache: + +install: + - echo Installing Miniconda + - appveyor-retry appveyor DownloadFile https://repo.anaconda.com/miniconda/Miniconda3-py310_23.1.0-1-Windows-x86_64.exe + - Miniconda3-py310_23.1.0-1-Windows-x86_64.exe /InstallationType=JustMe /RegisterPython=1 /S /D=C:\Users\appveyor\Miniconda3 + - del Miniconda3-py310_23.1.0-1-Windows-x86_64.exe + - activate.bat + - conda install -y pip git make -c conda-forge + - pip install poetry + +build_script: + - make build + +test_script: + - .\build\dist\winevt-tailer.exe -v + +artifacts: + - path: .\build\dist\winevt-tailer.exe + name: Executable diff --git a/pyproject.toml b/pyproject.toml index c70f69a..b98a225 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,85 +1,85 @@ -[tool.poetry] -name = "winevt-tailer" -version = "0.3.4" -description = 'Windows Event Log Tailer' -authors = ["Dmitri Khokhlov "] -maintainers = ["Dmitri Khokhlov "] -license = "MIT" -repository = "https://github.com/logdna/winevt-tailer" -documentation = "https://github.com/logdna/winevt-tailer" -readme = "README.md" - -[tool.semantic_release] -version_toml = "pyproject.toml:tool.poetry.version" -version_pattern = "winevt-tailer/VERSION:(\\d+\\.\\d+\\.\\d+)" -branch = "master" -commit_subject = "release: Version {version} [skip ci]" -commit_author = "LogDNA Bot " - -[tool.poetry.dependencies] -python = ">=3.10,<3.12" -pywin32 = "^304" -lxml = "^4.9.1" -PyYAML = "^6.0" -pydantic = "^1.10.2" -single-source = "^0.3.0" -google-crc32c = "^1.5.0" -psutil = "^5.9.4" -lazy-string = "^1.0.0" - -[tool.poetry.dev-dependencies] -coverage = "^5.4" -flake8 = "^3.8.4" -yapf = "^0.30.0" -pytest = "^6.2.2" -pytest-cov = "^2.11.1" -taskipy = "^1.6.0" -python-semantic-release = "^7.16.1" -tomlkit = "^0.11.5" -pyinstaller = "^5.6.2" -pyinstaller-versionfile = "^2.0.0" -tinyaes = "^1.0.4" -tomli = "^2.0.1" -pyinstaller-hooks-contrib = "^2022.13" -cffi = "^1.15.1" - -[tool.taskipy.tasks] -pre_test = "rm -rf coverage;mkdir coverage" -test = "pytest --junitxml=coverage/test.xml --cov=./ --cov-report=html --verbose tests/" -post_test = "python scripts/json_coverage.py" -lint = "flake8 --doctests --ignore=E402,F841,F401,E302,E305 --max-line-length=120" -"lint:fix" = "yapf -r -i winevt-tailer scripts tests" -"post_lint:fix" = "task lint" -release = "semantic-release publish" -create_version_file = "python packaging/pyinstaller_utils.py ./pyproject.toml ./build/version_file.txt" -create_exe = "pyinstaller.exe --clean --key Mezmo --specpath ./build --distpath ./build/dist --version-file=version_file.txt --icon ../packaging/images/Mezmo.ico -y --name winevt-tailer --copy-metadata winevt_tailer --hidden-import winevt_tailer.transforms --onefile winevt_tailer/main.py" - - -[build-system] -requires = ["poetry>=0.12"] -build-backend = "poetry.masonry.api" - -[tool.pytest.ini_options] -minversion = "6.0" -testpaths = "tests" - -[tool.coverage.run] -branch = true -source = ["winevt-tailer"] - -[tool.coverage.report] -fail_under = 54 -show_missing = true - -[tool.coverage.json] -output = "coverage/coverage.json" - -[tool.coverage.html] -directory = "coverage" -show_contexts = true - -[tool.poetry.scripts] -run = "winevt_tailer.main:main" - -[tool.poetry.urls] -issues = "https://github.com/logdna/winevt-tailer/issues" +[tool.poetry] +name = "winevt-tailer" +version = "0.3.4" +description = 'Windows Event Log Tailer' +authors = ["Dmitri Khokhlov "] +maintainers = ["Dmitri Khokhlov "] +license = "MIT" +repository = "https://github.com/logdna/winevt-tailer" +documentation = "https://github.com/logdna/winevt-tailer" +readme = "README.md" + +[tool.semantic_release] +version_toml = "pyproject.toml:tool.poetry.version" +version_pattern = "pyproject.toml:version = \"(\\d+\\.\\d+\\.\\d+)\"" +branch = "master" +commit_subject = "release: Version {version} [skip ci]" +commit_author = "LogDNA Bot " + +[tool.poetry.dependencies] +python = ">=3.10,<3.12" +pywin32 = "^304" +lxml = "^4.9.1" +PyYAML = "^6.0" +pydantic = "^1.10.2" +single-source = "^0.3.0" +google-crc32c = "^1.5.0" +psutil = "^5.9.4" +lazy-string = "^1.0.0" + +[tool.poetry.dev-dependencies] +coverage = "^5.4" +flake8 = "^3.8.4" +yapf = "^0.30.0" +pytest = "^6.2.2" +pytest-cov = "^2.11.1" +taskipy = "^1.6.0" +python-semantic-release = "^7.16.1" +tomlkit = "^0.11.5" +pyinstaller = "^5.6.2" +pyinstaller-versionfile = "^2.0.0" +tinyaes = "^1.0.4" +tomli = "^2.0.1" +pyinstaller-hooks-contrib = "^2022.13" +cffi = "^1.15.1" + +[tool.taskipy.tasks] +pre_test = "rm -rf coverage;mkdir coverage" +test = "pytest --junitxml=coverage/test.xml --cov=./ --cov-report=html --verbose tests/" +post_test = "python scripts/json_coverage.py" +lint = "flake8 --doctests --ignore=E402,F841,F401,E302,E305 --max-line-length=120" +"lint:fix" = "yapf -r -i winevt-tailer scripts tests" +"post_lint:fix" = "task lint" +release = "semantic-release publish" +create_version_file = "python packaging/pyinstaller_utils.py ./pyproject.toml ./build/version_file.txt" +create_exe = "pyinstaller.exe --clean --key Mezmo --specpath ./build --distpath ./build/dist --version-file=version_file.txt --icon ../packaging/images/Mezmo.ico -y --name winevt-tailer --copy-metadata winevt_tailer --hidden-import winevt_tailer.transforms --onefile winevt_tailer/main.py" + + +[build-system] +requires = ["poetry>=0.12"] +build-backend = "poetry.masonry.api" + +[tool.pytest.ini_options] +minversion = "6.0" +testpaths = "tests" + +[tool.coverage.run] +branch = true +source = ["winevt-tailer"] + +[tool.coverage.report] +fail_under = 54 +show_missing = true + +[tool.coverage.json] +output = "coverage/coverage.json" + +[tool.coverage.html] +directory = "coverage" +show_contexts = true + +[tool.poetry.scripts] +run = "winevt_tailer.main:main" + +[tool.poetry.urls] +issues = "https://github.com/logdna/winevt-tailer/issues"