Skip to content

Commit

Permalink
Merge branch 'main' into fix-#199
Browse files Browse the repository at this point in the history
  • Loading branch information
louis030195 authored Sep 11, 2024
2 parents abecb2f + a0366ca commit 2dad0f7
Show file tree
Hide file tree
Showing 113 changed files with 7,237 additions and 1,886 deletions.
5 changes: 5 additions & 0 deletions .cargo/config.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
[target.x86_64-pc-windows-msvc]
rustflags = [
"-C", "target-feature=-crt-static",
"-C", "link-args=/NODEFAULTLIB:libcmt.lib"
]
47 changes: 47 additions & 0 deletions .github/scripts/purge.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
#!/bin/bash

# Check if the CN_API_KEY is set or throw an error
if [ -z "$CN_API_KEY" ]; then
echo "Error: CN_API_KEY is not set"
exit 1
fi

# Extract the current version from Cargo.toml
current_version=$(sed -n 's/^version = "\(.*\)"/\1/p' examples/apps/screenpipe-app-tauri/src-tauri/Cargo.toml | head -n 1)

# Get the version to purge up to from the first argument
purge_up_to=${1:-$current_version}

echo "Current version: $current_version"
echo "Purging versions up to: $purge_up_to"

# Check if current_version is empty
if [ -z "$current_version" ]; then
echo "Error: Could not extract version from Cargo.toml"
exit 1
fi

# Generate versions dynamically
major=$(echo $current_version | cut -d. -f1)
minor=$(echo $current_version | cut -d. -f2)
patch=$(echo $current_version | cut -d. -f3)

# Check if we got valid numbers
if ! [[ "$major" =~ ^[0-9]+$ ]] || ! [[ "$minor" =~ ^[0-9]+$ ]] || ! [[ "$patch" =~ ^[0-9]+$ ]]; then
echo "Error: Invalid version format in Cargo.toml"
exit 1
fi

for ((m=0; m<=$minor; m++)); do
for ((p=0; p<=70; p++)); do
version="$major.$m.$p"
if [[ "$(echo -e "$version\n$purge_up_to" | sort -V | head -n1)" == "$version" && "$version" != "$purge_up_to" ]]; then
echo "Purging version $version"
cn release purge screenpipe "$version"
else
echo "Skipping version $version"
fi
done
done

echo "Purge complete"
59 changes: 54 additions & 5 deletions .github/workflows/benchmark.yml
Original file line number Diff line number Diff line change
Expand Up @@ -60,16 +60,46 @@ jobs:

- name: Run Windows OCR benchmarks
run: |
cargo bench --bench ocr_benchmark -- --output-format criterion | Tee-Object -Append -FilePath ocr_output.txt
cargo bench --bench ocr_benchmark -- --output-format criterion
# TODO broken
# | Tee-Object -Append -FilePath ocr_output.txt
- name: Upload OCR benchmark artifact
uses: actions/upload-artifact@v3
with:
name: ocr-benchmark-data
path: ocr_output.txt

stt_benchmark:
name: Run STT benchmark
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: dtolnay/rust-toolchain@stable

- name: Install dependencies
run: |
sudo apt-get update
sudo apt-get install -y ffmpeg tesseract-ocr libtesseract-dev libavformat-dev libavfilter-dev libavdevice-dev ffmpeg libasound2-dev libgtk-3-dev libsoup-3.0-dev libjavascriptcoregtk-4.1-dev libwebkit2gtk-4.1-dev
- name: Run STT benchmarks
run: |
cargo bench --bench stt_benchmark -- --output-format bencher | tee -a stt_output.txt
- name: Upload STT benchmark artifact
uses: actions/upload-artifact@v3
with:
name: stt-benchmark-data
path: stt_output.txt

analyze_benchmarks:
needs: [apple_ocr_benchmark, tesseract_ocr_benchmark, windows_ocr_benchmark]
needs:
[
apple_ocr_benchmark,
tesseract_ocr_benchmark,
windows_ocr_benchmark,
stt_benchmark,
]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
Expand All @@ -78,17 +108,36 @@ jobs:
uses: actions/download-artifact@v3
with:
name: ocr-benchmark-data
path: ./cache
path: ./cache/ocr

- name: Download STT benchmark data
uses: actions/download-artifact@v3
with:
name: stt-benchmark-data
path: ./cache/stt

- name: List contents of cache directory
run: ls -R ./cache

- name: Analyze all OCR benchmarks
- name: Analyze OCR benchmarks
uses: benchmark-action/github-action-benchmark@v1
with:
name: OCR Benchmarks
tool: "cargo"
output-file-path: ./cache/ocr_output.txt
output-file-path: ./cache/ocr/ocr_output.txt
github-token: ${{ secrets.GH_PAGES_TOKEN }}
auto-push: true
alert-threshold: "200%"
comment-on-alert: true
fail-on-alert: true
alert-comment-cc-users: "@louis030195"

- name: Analyze STT benchmarks
uses: benchmark-action/github-action-benchmark@v1
with:
name: STT Benchmarks
tool: "cargo"
output-file-path: ./cache/stt/stt_output.txt
github-token: ${{ secrets.GH_PAGES_TOKEN }}
auto-push: true
alert-threshold: "200%"
Expand Down
6 changes: 3 additions & 3 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ on:
jobs:
test-ubuntu:
runs-on: ubuntu-latest
env:
RUSTFLAGS: "-C link-arg=-Wl,--allow-multiple-definition"
steps:
- uses: actions/checkout@v3
- uses: actions/cache@v3
Expand All @@ -27,6 +29,7 @@ jobs:
with:
profile: minimal
toolchain: stable
override: true

- name: Install dependencies
run: |
Expand Down Expand Up @@ -65,9 +68,6 @@ jobs:
run: bun ./scripts/pre_build.js
working-directory: ./examples/apps/screenpipe-app-tauri

# - name: Run tests
# run: cargo test

- name: Copy test image
shell: bash
run: |
Expand Down
109 changes: 109 additions & 0 deletions .github/workflows/perf-long-running-end-to-end.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
name: Linux Memory Test

on:
workflow_dispatch:

jobs:
test-ubuntu:
runs-on: ubuntu-latest
timeout-minutes: 20

steps:
- uses: actions/checkout@v3

- uses: actions/cache@v3
with:
path: |
~/.cargo/bin/
~/.cargo/registry/index/
~/.cargo/registry/cache/
~/.cargo/git/db/
target/
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}

- uses: actions-rs/toolchain@v1
with:
profile: minimal
toolchain: stable

- name: Install dependencies
run: |
sudo apt-get update
sudo apt-get install -y tesseract-ocr libtesseract-dev libavformat-dev libavfilter-dev libavdevice-dev ffmpeg libasound2-dev libgtk-3-dev libsoup-3.0-dev libjavascriptcoregtk-4.1-dev libwebkit2gtk-4.1-dev xvfb pulseaudio gnuplot
- name: Set up virtual display
run: |
Xvfb :99 -ac &
echo "DISPLAY=:99" >> $GITHUB_ENV
- name: Set up virtual audio
run: |
pulseaudio --start
pacmd load-module module-null-sink sink_name=virtual_speaker
pacmd load-module module-virtual-source source_name=virtual_mic
pacmd set-default-sink virtual_speaker
pacmd set-default-source virtual_mic
- name: Build screenpipe
run: cargo build --release

- name: Run screenpipe and monitor memory
run: |
export DISPLAY=:99
export PULSE_SERVER=unix:/tmp/pulse-socket
./target/release/screenpipe --debug &
SCREENPIPE_PID=$!
echo "Timestamp,RSS (KB),VSZ (KB)" > memory_usage.csv
for i in {1..90}; do
RSS=$(ps -o rss= -p $SCREENPIPE_PID)
VSZ=$(ps -o vsz= -p $SCREENPIPE_PID)
echo "$(date +%s),$RSS,$VSZ" >> memory_usage.csv
sleep 10
done
kill $SCREENPIPE_PID
- name: Generate memory usage graph
run: |
gnuplot <<EOF
set terminal png size 800,600
set output 'memory_usage_graph.png'
set title 'Screenpipe Memory Usage Over Time'
set xlabel 'Time (seconds)'
set ylabel 'Memory Usage (KB)'
set key outside
plot 'memory_usage.csv' using 1:2 with lines title 'RSS', \
'' using 1:3 with lines title 'VSZ'
EOF
- name: Upload memory usage data
uses: actions/upload-artifact@v3
with:
name: memory-usage-data
path: memory_usage.csv

- name: Upload memory usage graph
uses: actions/upload-artifact@v3
with:
name: memory-usage-graph
path: memory_usage_graph.png

- name: Check logs and data
run: |
echo "Checking logs..."
tail -n 100 ~/.screenpipe/logs/screenpipe.log
echo "Checking database..."
sqlite3 ~/.screenpipe/db.sqlite ".tables"
sqlite3 ~/.screenpipe/db.sqlite "SELECT COUNT(*) FROM ocr_text;"
sqlite3 ~/.screenpipe/db.sqlite "SELECT COUNT(*) FROM audio_transcriptions;"
- name: Upload screenpipe data
uses: actions/upload-artifact@v3
with:
name: screenpipe-data
path: |
~/.screenpipe/logs
~/.screenpipe/db.sqlite
43 changes: 38 additions & 5 deletions .github/workflows/release-app.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ jobs:
- uses: actions/checkout@v4

- name: create draft release
uses: crabnebula-dev/cloud-release@v0.2.0
uses: crabnebula-dev/cloud-release@v0
with:
command: release draft ${{ secrets.CN_APP_SLUG }} --framework tauri
api-key: ${{ secrets.CN_API_KEY }}
Expand Down Expand Up @@ -163,15 +163,48 @@ jobs:
run: bun ./scripts/pre_build.js ${{ matrix.pre-build-args }}
working-directory: ./examples/apps/screenpipe-app-tauri

- name: Identify esaxx usage (Windows)
if: matrix.platform == 'windows-latest'
run: |
cargo tree > cargo_tree_output.txt
findstr /i "esaxx" cargo_tree_output.txt
continue-on-error: true

- name: Install vcpkg
if: matrix.platform == 'windows-latest'
uses: lukka/run-vcpkg@v11
with:
vcpkgGitCommitId: "5b1214315250939257ef5d62ecdcbca18cf4fb1c"

# - name: Install ONNX Runtime
# if: matrix.platform == 'windows-latest'
# run: vcpkg install onnxruntime:x64-windows

- name: Set up MSVC
if: matrix.platform == 'windows-latest'
uses: ilammy/msvc-dev-cmd@v1

- name: Build
uses: tauri-apps/tauri-action@v0
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# for updater
TAURI_SIGNING_PRIVATE_KEY: ${{ secrets.TAURI_PRIVATE_KEY }}
TAURI_SIGNING_PRIVATE_KEY_PASSWORD: ${{ secrets.TAURI_KEY_PASSWORD }}
# for release
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# for macos signing
APPLE_CERTIFICATE: ${{ secrets.APPLE_CERTIFICATE }}
APPLE_CERTIFICATE_PASSWORD: ${{ secrets.APPLE_CERTIFICATE_PASSWORD }}
APPLE_SIGNING_IDENTITY: ${{ secrets.APPLE_SIGNING_IDENTITY }}
APPLE_ID: ${{ secrets.APPLE_ID }}
APPLE_PASSWORD: ${{ secrets.APPLE_PASSWORD }}
APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }}
# https://tauri.app/v1/guides/distribution/sign-macos
CI: true
# RUSTFLAGS: "-C link-arg=-Wl,--allow-multiple-definition"
with:
tagName: v__VERSION__ # the action automatically replaces __VERSION__ with the app version.
releaseName: "v__VERSION__"
tagName: app-v__VERSION__ # the action automatically replaces __VERSION__ with the app version.
releaseName: "App v__VERSION__"
releaseBody: "What's new? 🎉📣"
releaseDraft: true
prerelease: true
Expand All @@ -180,7 +213,7 @@ jobs:
tauriScript: bunx tauri -v

- name: Upload Assets to CrabNebula Cloud
uses: crabnebula-dev/cloud-release@v0.2.0
uses: crabnebula-dev/cloud-release@v0
with:
command: release upload ${{ secrets.CN_APP_SLUG }} --framework tauri
api-key: ${{ secrets.CN_API_KEY }}
Expand Down
8 changes: 4 additions & 4 deletions .github/workflows/release-cli.yml
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ jobs:
export PKG_CONFIG_PATH="/usr/local/opt/ffmpeg/lib/pkgconfig:$PKG_CONFIG_PATH"
export PKG_CONFIG_ALLOW_CROSS=1
export RUSTFLAGS="-C link-arg=-Wl,-rpath,@executable_path/../lib -C link-arg=-Wl,-rpath,@loader_path/../lib"
cargo build --release --features metal --target ${{ matrix.target }}
cargo build --release --features pipes,metal --target ${{ matrix.target }}
# get the current tag or take latest (in the case of a workflow dispatch)
- name: Set version
Expand Down Expand Up @@ -123,7 +123,7 @@ jobs:
gh pr create --base main --head update-formula-${{ matrix.target }}-${{ github.sha }} --title "Update Homebrew formula for ${{ matrix.target }}" --body "Automated PR to update Homebrew formula for ${{ matrix.target }}"
- name: Upload Artifact
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v3
with:
name: screenpipe-macos-${{ matrix.target }}
path: screenpipe-*.tar.gz
Expand Down Expand Up @@ -172,7 +172,7 @@ jobs:
# tar -czvf screenpipe-linux.tar.gz screenpipe-linux

# - name: Upload Artifact
# uses: actions/upload-artifact@v2
# uses: actions/upload-artifact@v3
# with:
# name: screenpipe-linux
# path: screenpipe-linux.tar.gz
Expand Down Expand Up @@ -243,7 +243,7 @@ jobs:
echo "Set version to: $VERSION"
- name: Download macOS Artifacts
uses: actions/download-artifact@v2
uses: actions/download-artifact@v3
with:
path: artifacts

Expand Down
Loading

0 comments on commit 2dad0f7

Please sign in to comment.