Scrape #538
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Scrape | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| # Run every day | |
| - cron: 36 5 * * * | |
| jobs: | |
| scrape: | |
| name: Run Scrapers | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: | |
| - ubuntu-latest | |
| ocaml-compiler: | |
| - ocaml-base-compiler.4.14.1 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v3 | |
| - name: Use OCaml ${{ matrix.ocaml-compiler }} | |
| uses: ocaml/setup-ocaml@v2 | |
| with: | |
| ocaml-compiler: ${{ matrix.ocaml-compiler }} | |
| dune-cache: ${{ matrix.os != 'macos-latest' }} | |
| opam-repositories: | | |
| pin: git+https://github.com/ocaml/opam-repository#c1fcfbe8369cb5a7cb8d7be63db59856dbd85d19 | |
| opam-disable-sandboxing: true | |
| - name: Install system dependencies | |
| run: sudo apt update && sudo apt-get install libcurl4-openssl-dev | |
| - name: Install opam dependencies | |
| run: opam install --deps-only --with-test . | |
| - name: Build scraper | |
| run: | | |
| opam exec -- dune build tool/ood-gen/bin/scrape.exe | |
| opam exec -- dune build tool/ood-gen/bin/watch_scrape.exe | |
| - name: Run scrapers | |
| run: | | |
| opam exec -- dune exec tool/ood-gen/bin/scrape.exe planet | |
| opam exec -- dune exec tool/ood-gen/bin/watch_scrape.exe > data/watch.yml | |
| opam exec -- dune build src/ocamlorg_data/planet.ml | |
| opam exec -- dune build src/ocamlorg_data/watch.ml | |
| - name: Create Pull Request | |
| uses: peter-evans/create-pull-request@v5 | |
| with: | |
| title: 'Update scraped data' | |
| add-paths: | | |
| data/watch.yml | |
| data/planet/*/*.md |