ci: Release process updates (#385)

- see #356

.github/workflows/_publish_to_pypi.yaml

@@ -2,6 +2,10 @@ name: Publish to PyPI
 
 on:
   workflow_call:
+    inputs:
+      version_number:
+        required: true
+        type: string
 
 jobs:
   publish_to_pypi:
@@ -23,24 +27,37 @@ jobs:
         with:
           python-version: 3.12
 
+      - name: Set up Poetry
+        run: pipx install poetry
+
       - name: Install dependencies
         run: make install-dev
 
-      # Determines the release type based on the event that triggered the workflow.
-      - name: Determine release type
-        id: determine-release-type
+      # Updates the version number in the project's configuration.
+      - name: Set version in pyproject.toml
         run: |
-          if [[ ${{ github.event_name }} = workflow_dispatch ]]; then
-            release_type="final"
-          elif [[ ${{ github.event_name }} = push ]]; then
-            release_type="beta"
+          if [[ ${{ github.event_name }} = push ]]; then
+            # Make a pre-release on push
+            next_beta=$(
+              curl "https://pypi.org/pypi/crawlee/json" | jq --raw-output '
+                [
+                  .releases |
+                    keys |
+                    .[] |
+                    select(startswith("${{ inputs.version_number }}b")) |
+                    split("b") |
+                    .[1] |
+                    tonumber
+                ] |
+                sort_by(- .) |
+                .[0] + 1
+              '
+            )
+            poetry version "${{ inputs.version_number }}b$next_beta"
+          else
+            # ...and a regular release otherwise
+            poetry version '${{ inputs.version_number }}'
           fi
-          echo "release_type=${release_type}" >> $GITHUB_OUTPUT
-
-      # Updates the version number for pre-releases in the project's configuration.
-      - name: Set pre-release version
-        if: steps.determine-release-type.outputs.release_type != 'final'
-        run: python ./scripts/update_version_for_prerelease.py ${{ steps.determine-release-type.outputs.release_type }}
 
       # Builds the package.
       - name: Build package

.github/workflows/run_code_checks.yaml

@@ -23,10 +23,6 @@ jobs:
     name: Run unit tests
     uses: ./.github/workflows/_unit_tests.yaml
 
-  check_version_conflict:
-    name: Check version conflict
-    uses: ./.github/workflows/_check_version_conflict.yaml
-
   check_docs_build:
     name: Check docs build
     uses: ./.github/workflows/_check_docs_build.yaml

.github/workflows/run_release.yaml

@@ -38,8 +38,9 @@ jobs:
       github.event_name == 'workflow_dispatch' ||
       (
         github.event_name == 'push' &&
-        !startsWith(github.event.head_commit.message, 'docs') &&
-        !startsWith(github.event.head_commit.message, 'chore')
+        !startsWith(github.event.head_commit.message, 'docs:') &&
+        !startsWith(github.event.head_commit.message, 'chore:') &&
+        !startsWith(github.event.head_commit.message, 'ci:')
       )
     runs-on: ubuntu-latest
     outputs:
@@ -74,7 +75,9 @@ jobs:
       contents: write
       id-token: write # Required for OIDC authentication.
     outputs:
-      changelog_commitish: ${{ steps.push.commitish }}
+      changelog_commitish: ${{ steps.commit.commit_long_sha || github.sha }}
+    env:
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -89,11 +92,6 @@ jobs:
       - name: Install git-cliff
         run: pip install git-cliff
 
-      - name: Set up git
-        run: |
-          git config --global user.name "Apify Release Bot"
-          git config --global user.email "[email protected]"
-
       - name: Generate changelog with git-cliff
         run: |
           if [[ ${{ github.event_name }} = workflow_dispatch ]]; then
@@ -102,18 +100,27 @@ jobs:
             git-cliff --with-tag-message v${{ needs.should_release.outputs.version_number }} > CHANGELOG.md
           fi
 
-      - name: Commit and push the result
-        id: push
-        run: |
-          git commit -am "chore(release): Update changelog [skip ci]" || true
-          git push origin master || true
-          echo commitish=$(git rev-parse HEAD) | tee -a $GITHUB_OUTPUT
+      - name: Install poetry
+        run: pipx install poetry
+
+      - name: Update package version in pyproject.toml
+        run: poetry version ${{ needs.should_release.outputs.version_number }}
+
+      - name: Commit changes
+        id: commit
+        uses: EndBug/add-and-commit@v9
+        with:
+          author_name: Apify Release Bot
+          author_email: [email protected]
+          message: 'chore(release): Update changelog and package version [skip ci]'
 
   create_github_release:
     name: Create github release
     if: github.event_name == 'workflow_dispatch'
     needs: [update_changelog, should_release]
     runs-on: ubuntu-latest
+    env:
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -140,8 +147,10 @@ jobs:
 
   publish_to_pypi:
     name: Publish to PyPI
-    needs: [create_github_release]
+    needs: [should_release]
     uses: ./.github/workflows/_publish_to_pypi.yaml
+    with:
+      version_number: ${{ needs.should_release.outputs.version_number }}
 
   # TODO: add job for publish package to Conda
   # https://github.com/apify/crawlee-py/issues/104

.pre-commit-config.yaml

@@ -18,9 +18,3 @@ repos:
         entry: make unit-tests
         language: system
         pass_filenames: false
-
-      - id: check-version-conflict
-        name: Check version conflict
-        entry: make check-version-conflict
-        language: system
-        pass_filenames: false