diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 00000000..3116ec73 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,143 @@ +name: CI + +on: + push: + branches: + - master + tags: + - '*' + pull_request: + branches: + - master + +jobs: + cpp_edlib: + name: "Check that CPP edlib correctly builds and passes tests." + strategy: + matrix: + include: + # If you add another compiler, make sure to define its CC and CXX env vars in the step below. + - { os: ubuntu-20.04, compiler: gcc-10 } + - { os: ubuntu-20.04, compiler: clang-11 } + - { os: macos-12, compiler: gcc-10 } + - { os: macos-12, compiler: clang-11 } + - { os: ubuntu-22.04, compiler: gcc-11 } + - { os: ubuntu-22.04, compiler: clang-15 } + - { os: macos-14, compiler: gcc-11 } + - { os: macos-14, compiler: clang-15 } + runs-on: ${{ matrix.os }} + continue-on-error: true + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install system dependencies (Linux) + if: runner.os == 'Linux' + run: | + sudo apt-get update + sudo apt-get install -y ${{ matrix.compiler.package }} valgrind + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + + - name: Install Python packages + run: | + python -m pip install --upgrade pip setuptools meson ninja + + - name: Set C/CPP compiler to use + run: | + if [[ "${{ matrix.compiler }}" == "gcc-10" ]]; then export CC=gcc-10 CXX=g++-10; fi + if [[ "${{ matrix.compiler }}" == "gcc-11" ]]; then export CC=gcc-11 CXX=g++-11; fi + if [[ "${{ matrix.compiler }}" == "clang-11" ]]; then export CC=clang-11 CXX=clang++-11; fi + if [[ "${{ matrix.compiler }}" == "clang-15" ]]; then export CC=clang-15 CXX=clang++-15; fi + + - name: Build binaries and libraries and test them (with Meson) + run: | + make CXXFLAGS="-Werror" LIBRARY_TYPE=static BUILD_DIR=meson-build-static + make CXXFLAGS="-Werror" LIBRARY_TYPE=shared BUILD_DIR=meson-build-shared + + # Check for memory leaks. + # I run this only on linux because osx returns errors from + # system libraries, which I would have to supress. + if [ ${{ runner.os }} == "Linux" ]; then + make check-memory-leaks BUILD_DIR=meson-build-static + fi + + - name: Build binaries and libraries and test them (with CMake) + run: | + mkdir -p build && cd build + CXXFLAGS="-Werror" cmake -GNinja .. + ninja -v + bin/runTests + + # TODO: I should have this step produce artifacts (wheels and sdist), but not deploy them. + # Then, I should have another step that deploys them, therefore not deploying unless all jobs pass, + # making sure we don't deploy only half of the wheels. + python_edlib: + name: "Build, test and possibly deploy python bindings for edlib" + strategy: + matrix: + include: + - os: ubuntu-22.04 + deploy-sdist: true + - os: macos-13 # intel runner + # TODO: Get macos-14 building, currently I have an error with arch mismatch. + # - os: macos-14 # apple silicon + runs-on: ${{ matrix.os }} + defaults: + run: + working-directory: bindings/python + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + + - name: Install Python deps + run: | + python -m pip install setuptools + + - name: Build edlib python module + run: | + make build + + - name: Test edlib python module + run: | + python test.py + + - name: Build sdist + run: | + make sdist + # To ensure it doesn't get cleaned up by the `make wheels` or some other step. + mv dist sdist + + - name: Build wheels + run: | + if [ ${{ matrix.os }} == "macos-13" ]; then + # Default is x86-64 only, but this way we also build universal2 wheels, + # which work on both intel (x86_64) and apple silicon (arm64). + export CIBW_ARCHS_MACOS="x86_64 universal2" + fi + make wheels + + - name: Deploy sdist and Linux and Mac wheels to PyPI + if: github.ref_type == 'tag' + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} + run: | + python -m pip install twine + + python -m twine upload --skip-existing wheelhouse/*.whl + + # While I do want to upload wheels for both Mac and Linux, + # it makes no sense to upload sdist twice. + if [ ${{ matrix.deploy-sdist }} == "true" ]; then + python -m twine upload --skip-existing sdist/edlib-*.tar.gz + fi + diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 681f9535..00000000 --- a/.travis.yml +++ /dev/null @@ -1,156 +0,0 @@ -# This ensures only commits on master, PRs towards master, and all tags are built. -# Otherwise, build does not start. -if: branch = master OR tag IS present - -language: cpp - -os: linux - -dist: focal - -stages: - - ccpp - - python - -env: - global: - - TWINE_USERNAME=Martinsos - # Encrypted TWINE_PASSWORD: - - secure: "VfgRi/hcUMLDYzATLS8JOuZo+wg0VNCmEojIJ3gpy89LAXgPN1Qx/2n3c52PPgFWgWFdDgns2SdxhQ1/810qb+65choy2fOgTULnyFcH3ZFTM5nINVHlGB5dLDUesKjrqqfIKHnwB6LAskHqHHPm/qv7WxUjQbcpR6mICsKB7h3NI7yUSc3tRPmd8tVBc/mTUXpMRVznTxGFCo/rC4f2y57GI9odjDxS4VqGS8T9Igvgyteg+ougXYbl84FIi3Bij6TQITa1rP9cFw/lfXMTM2m1czTonzAbgGPC6q7798MCGhLpEdxr+Zu9NPCS8aWFYdxMUYmsisjffDtTdIagZFh/14YrnQb2RxqnLTqSi20E+xXx16GcHhnMOxw/sSOh6HUEaXcbLNtx/mqMEa7mQEcIFBdXHWs/czTHz8xEoGnHIVHI5zqItoK3nUD0W/X6XUex8GP8obvjSKVw2k8X/0bZUaqSyNVbTfYcLmOk53Kx8vfXd82BbTOWiJVZmvuugqrshIEaLOyRhSaoC4Bj5KjfijM/gKPa+n8wVEQ/4hF4HpJKr/33cAm1URDGsDnfvHfxfeEwizTq8gCIxnbpOzSsd1M6+3MQLr48DE8uYM+wkG2F+KsCKe3OexLiaqHG7rVZh1R99nOykoqfjhxek3H1KGfx7qFbdK80AaVBiQo=" - -.ccpp: &ccpp - stage: ccpp # Builds C/CPP code. - - before_install: - - sudo -H pip3 install -U pip # Needed for Meson. - - sudo -H pip3 install -U setuptools # Needed for Meson. - - eval "${MATRIX_EVAL}" - - install: - - sudo -H pip3 install meson ninja # We use Ninja both with Meson and CMake. - - script: - # Build library and binaries using Meson and test them. - # Static library. - - make CXXFLAGS="-Werror" LIBRARY_TYPE=static BUILD_DIR=meson-build-static - # Shared library. - - make CXXFLAGS="-Werror" LIBRARY_TYPE=shared BUILD_DIR=meson-build-shared - - # Check for memory leaks. - # I run this only on linux because osx returns errors from - # system libraries, which I would have to supress. - - | - if [ $TRAVIS_OS_NAME == "linux" ]; then - make check-memory-leaks BUILD_DIR=meson-build-static - fi - - # Build library and binaries with CMake and test them. - - mkdir -p build && cd build - - CXXFLAGS="-Werror" cmake -GNinja .. - - ninja -v - - bin/runTests - - cd .. - -.python: &python - stage: python # Builds python binding. - - install: - - wget -qO- https://bootstrap.pypa.io/get-pip.py | python3 - - python3 -m pip install cibuildwheel==2.10.2 - - before_script: - - cd bindings/python - - script: - # Build the wheels, put them into './wheelhouse'. - # cibuildwheel needs to be run from python directory. - # The python directory is mounted in the docker build containers, - # so we need to copy the edlib source before running cibuild wheel. - - make edlib # Copies edlib source to python dir. - - | - CIBW_SKIP="cp27-* pp* *-manylinux_i686" \ - CIBW_BEFORE_BUILD="make sdist" \ - CIBW_TEST_COMMAND="python3 {project}/test.py" \ - python3 -m cibuildwheel --output-dir wheelhouse - - if [ $DEPLOY_SDIST_TO_PYPI ]; then make sdist; fi - - # Deploy wheel(s) and source (sdist) to PYPI (if commit is tagged). - # NOTE: We are doing this in script section, and not in after_success, - # because we want build to fail if deploying fails, which does not - # happen if deployment is done inside after_success (that is how - # travis works). - # NOTE: We check TRAVIS_TEST_RESULT because we don't want to deploy - # if building failed. - - | - if [ $TRAVIS_TEST_RESULT -eq 0 ]; then - if [ $TRAVIS_TAG ]; then - # upgrade and upgrade-strategy-eager are here because of https://travis-ci.community/t/cant-deploy-to-pypi-anymore-pkg-resources-contextualversionconflict-importlib-metadata-0-18/10494/14 . - python3 -m pip install --upgrade --upgrade-strategy eager twine - python3 -m twine upload wheelhouse/*.whl - if [ $DEPLOY_SDIST_TO_PYPI ]; then - python3 -m twine upload dist/edlib-*.tar.gz - fi - else - echo "Skipping twine upload because not a tag, files built:" - ls -l wheelhouse - ls -l dist - fi - fi - -jobs: - include: - # In order to catch as many weird edge cases in the code with -Werror as - # possible, we want to test a large range of old and new compilers on both - # Linux and macOS. This gives us the best possible coverage while maintaining - # compatibility with a large number of compilers. - - name: "C/CPP (Linux; GCC 10)" - os: linux - addons: - apt: - sources: - - ubuntu-toolchain-r-test # For g++-10 - packages: - - g++-10 - - valgrind - - python3 - - python3-pip - env: - - MATRIX_EVAL="export CC=gcc-10 && export CXX=g++-10" - <<: *ccpp - - - name: "C/CPP (Linux; Clang 10.0)" - os: linux - addons: - apt: - packages: - - clang-10 - - valgrind - - python3 - - python3-pip - env: - - MATRIX_EVAL="export CC=clang-10 && export CXX=clang++-10" - <<: *ccpp - - - name: "C/CPP (OSX; XCode 11; Clang)" - os: osx - osx_image: xcode11 - <<: *ccpp - - - name: "Python wheel (Linux)" - os: linux - language: python - python: - - "3.6" - services: docker - env: - # We need to upload sdist only once, since it is same for all jobs, - # so with this flag we specify which job will take care of that (this one). - - DEPLOY_SDIST_TO_PYPI="true" - <<: *python - - - name: "Python wheel (OSX)" - os: osx - language: shell - <<: *python - - # TODO: Add windows x64 and linux aarch64 python wheel builds? Edlib users said they need them. diff --git a/bindings/python/.gitignore b/bindings/python/.gitignore index 2f22afa8..7ee6660c 100644 --- a/bindings/python/.gitignore +++ b/bindings/python/.gitignore @@ -1,5 +1,6 @@ build/ dist/ +wheelhouse/ *.egg-info/ edlib/ edlib.c @@ -7,4 +8,5 @@ edlib.*.so .eggs/ *.bycython.* README.rst -README.html \ No newline at end of file +README.html +.venv diff --git a/bindings/python/Makefile b/bindings/python/Makefile index 940cd5bb..1316a6f1 100644 --- a/bindings/python/Makefile +++ b/bindings/python/Makefile @@ -1,41 +1,82 @@ +.PHONY: default buildWithoutREADME.rst build sdist publish wheels clean + default: build -.PHONY: -edlib: $(shell find ../../edlib) - # create a clean (maybe updated) copy of edlib src - rm -rf edlib && cp -r ../../edlib . +# Copies edlib cpp source files into the project, if not copied yet. +edlib: + if [ ! -d edlib ]; then \ + cp -r ../../edlib . ; \ + fi pyedlib.bycython.cpp: edlib.pyx cedlib.pxd python -m pip install cython cython --cplus edlib.pyx -o edlib.bycython.cpp +BUILD_SOURCE_FILES=edlib pyedlib.bycython.cpp setup.py + # To build package, README.rst is needed, because it goes into long description of package, # which is what is visible on PyPI. # However, to generate README.rst from README-tmpl.rst, built package is needed (for `import edlib` in cog)! # Therefore, we first build package without README.rst, use it to generate README.rst, # and then finally build package again but with README.rst. - -BUILD_SOURCE_FILES=edlib pyedlib.bycython.cpp setup.py - buildWithoutREADME.rst: ${BUILD_SOURCE_FILES} - EDLIB_OMIT_README_RST=1 python setup.py build_ext -i + EDLIB_OMIT_README_RST=1 \ + python -m pip install -e . README.rst: buildWithoutREADME.rst README-tmpl.rst python -m pip install cogapp - cog -d -o README.rst README-tmpl.rst + cog -d -o README.rst README-tmpl.rst BUILD_FILES=${BUILD_SOURCE_FILES} README.rst build: ${BUILD_FILES} - python setup.py build_ext -i + python -m pip install -e . + +sdist: ${BUILD_FILES} MANIFEST.in + python -m pip install build + python -m build --sdist -sdist: edlib pyedlib.bycython.cpp setup.py README.rst MANIFEST.in - python setup.py sdist +# cibuildwheel builds wheels in docker containers. These containers are chosen and +# configured in such way that generated wheels are as robust as they can be in regards +# to glibc, musl and similar. +# They also ensure wheels are built in a reproducible way, independent of the host machine. +# +# For cibuildwheel to be able to build the wheels, we need to prepare / prebuild some +# some stuff for it in advance. Good way to handle this is to run `make sdist` which +# certainly builds everything that cibuildwheel might need to build the wheels. +# +# However, we can't just run `make sdist` manually before starting the cibuildwheel, because then +# `make sdist` will build its stuff on our local machine, and once cibuildwheel copies +# those in its container and tries to use them to build wheels, problems can arise due to +# those having been built with different tooling than what is now used to build the wheels! +# +# Instead, we want `make sdist` to run in cibuildwheel's docker containers also, +# so it uses the same tooling (e.g. gcc) as will be used to build the wheels, and we accomplish +# this by using CIBW_BEFORE_BUILD option, which will run given cmd in the container +# before building the wheel. +# +# There is though one thing we do need to "prebuild" on our machine in advance: +# copying edlib/ dir from ../../edlib to here. We can't do it inside the container +# because Docker context's root is . and not ../../, so ../../edlib is not available +# in docker container. +# That is why it is important that we have `edlib` here as dep for `wheels` Makefile target. +# We also `clean` first to remove everything else, so we have clean start in the container, +# since as we said, we want to make sure everything is built in there, with same tooling. +# +# So, first we clean any old build artifacts, then we ensure ../../edlib is copied to +# ./edlib, and then we call cibuildwheel which builds wheels in its docker containers +# running `make sdist` in the containers at the start of every wheel build. +wheels: clean edlib + python -m pip install cibuildwheel==2.20.0 + CIBW_SKIP="pp* *-manylinux_i686" \ + CIBW_BEFORE_BUILD="make sdist" \ + CIBW_TEST_COMMAND="python3 {project}/test.py" \ + python -m cibuildwheel --output-dir wheelhouse -publish: clean sdist - twine upload dist/* +publish: clean sdist wheels + python -m twine upload dist/edlib-*.tar.gz wheelhouse/*.whl clean: - rm -rf edlib dist edlib.egg-info build + rm -rf edlib dist edlib.egg-info build wheelhouse rm -f edlib.c *.bycython.* edlib.*.so rm -f README.rst diff --git a/bindings/python/README.md b/bindings/python/README.md index 5f0e1d20..2198cab6 100644 --- a/bindings/python/README.md +++ b/bindings/python/README.md @@ -4,10 +4,29 @@ This README contains only development information, you can check out full README README.rst is not commited to git because it is generated from [README-tmpl.rst](./README-tmpl.rst). +## Setup + +Ensure you have the version of python you want to use installed. You can use `pyenv` to manage python versions and pick specific one, if needed. + +Let's now say that `python` is pointing to the python version you want to use. +What you will most likely want to do now is run (from this dir, bindings/python/): +```sh +python -m venv .venv +``` +to create the virtual environment if you don't have it yet, and then +```sh +source .venv/bin/activate +``` +to activate it. + +This virtual environment will ensure all the python packages are installed locally for this project, and that local python packages are used. +You will want to keep this virtual environment activated for the rest of the commands in this README. + +Actual installation of python deps (packages) is not done by the "requirements.txt" as is typical for python projects, but by the `Makefile`: read on for the details on how to build with it. ## Building -Run `make build` to generate an extension module as .so file. +Run `make build` to build and install the package locally. You can test it then by importing it from python interpreter `import edlib` and running `edlib.align(...)` (you have to be positioned in the directory where .so was built). This is useful for testing while developing. @@ -21,9 +40,17 @@ README.rst is auto-generated from [README-tmpl.rst](./README-tmpl.rst), to run r README.rst is also automatically regenerated when building package (e.g. `make build`). This enables us to always have up to date results of code execution and help documentation of edlib methods in readme. +You can build wheels manually by running `build wheels`. + ## Publishing Remember to update version in setup.py before publishing. -To trigger automatic publish to PyPI, create a tag and push it to Github -> Travis will create sdist, build wheels, and push them all to PyPI while publishing new version. +To trigger automatic publish to PyPI, create a tag and push it to Github -> Github Action will create sdist, build wheels, and push them all to PyPI while publishing new version. + +You can also publish new version manually if needed: run `make publish` to create a source distribution and wheels and publish it to the PyPI. This will however produce wheels only for your platform / os. + +## Common tasks for maintainers +We use a pinned down version of cibuildwheel, which means that when new stable version of Python comes out, we won't be building wheels for it. Nor will we be publishing them. +Therefore, every so and so, when new Python version comes out, we should update the version of cibuildwheel and then also publish a new version of edlib python package. +Usually best way to do this is to update edlib version with `.postX` suffix and push to Github with a tag so that CI does all the work for us. -You can also publish new version manually if needed: run `make publish` to create a source distribution and publish it to the PyPI. diff --git a/bindings/python/setup.py b/bindings/python/setup.py index fff5d6f8..136fd8f3 100644 --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -30,7 +30,7 @@ name = "edlib", description = "Lightweight, super fast library for sequence alignment using edit (Levenshtein) distance.", long_description = long_description, - version = "1.3.9", + version = "1.3.9.post1", url = "https://github.com/Martinsos/edlib", author = "Martin Sosic", author_email = "sosic.martin@gmail.com", @@ -42,6 +42,7 @@ include_dirs=["edlib/include"], depends=["edlib/include/edlib.h"], language="c++", + compiler_directives={'language_level': '3'}, extra_compile_args=["-O3", "-std=c++11"])], cmdclass = cmdclass )