ql-for-ql-dataset_measure.yml

name: Collect database stats for QL for QL

on:
  push:
    branches: [main]
    paths:
      - ql/ql/src/ql.dbscheme
  pull_request:
    branches: [main]
    paths:
      - ql/ql/src/ql.dbscheme
  workflow_dispatch:

jobs:
  measure:
    env:
      CODEQL_THREADS: 4 # TODO: remove this once it's set by the CLI
    strategy:
      matrix:
        repo:
          - github/codeql
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3

      - name: Find codeql
        id: find-codeql
        uses: github/codeql-action/init@aa93aea877e5fb8841bcb1193f672abf6e9f2980
        with:
          languages: javascript # does not matter
      - uses: actions/cache@v3
        with:
          path: |
            ~/.cargo/registry
            ~/.cargo/git
            ql/target
          key: ${{ runner.os }}-qltest-cargo-${{ hashFiles('ql/**/Cargo.lock') }}
      - name: Build Extractor
        run: cd ql; env "PATH=$PATH:`dirname ${CODEQL}`" ./scripts/create-extractor-pack.sh
        env:
          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
      - name: Checkout ${{ matrix.repo }}
        uses: actions/checkout@v3
        with:
          repository: ${{ matrix.repo }}
          path: ${{ github.workspace }}/repo
      - name: Create database
        run: |
          "${CODEQL}" database create \
            --search-path "ql/extractor-pack" \
            --threads 4 \
            --language ql --source-root "${{ github.workspace }}/repo" \
            "${{ runner.temp }}/database"
        env:
          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
      - name: Measure database
        run: |
          mkdir -p "stats/${{ matrix.repo }}"
          "${CODEQL}" dataset measure --threads 4 --output "stats/${{ matrix.repo }}/stats.xml" "${{ runner.temp }}/database/db-ql"
        env:
          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
      - uses: actions/upload-artifact@v3
        with:
          name: measurements
          path: stats
          retention-days: 1

  merge:
    runs-on: ubuntu-latest
    needs: measure
    steps:
      - uses: actions/checkout@v3
      - uses: actions/download-artifact@v3
        with:
          name: measurements
          path: stats
      - run: |
          python -m pip install --user lxml
          find stats -name 'stats.xml' -print0 | sort -z | xargs -0 python ruby/scripts/merge_stats.py --output ql/ql/src/ql.dbscheme.stats --normalise ql_tokeninfo
      - uses: actions/upload-artifact@v3
        with:
          name: ql.dbscheme.stats
          path: ql/ql/src/ql.dbscheme.stats