build-list.yml

name: Build Database Update
on:
  push:
    paths:
    - 'input/**'
    - '.github/workflows/build-list.yml'
  schedule:
    - cron:  '45 8 * * *'
  workflow_dispatch:

jobs:
  build_and_commit:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        list:
         - name: vpn
           validate_low:  1000000
           validate_high: 2000000
         - name: datacenter
           validate_low:  150000000
           validate_high: 250000000
    steps:
    - uses: actions/checkout@v2
    - name: Dependencies
      run: |
        sudo apt-get install ipcalc
    - name: Downloading ASN database
      run: |
        curl https://iptoasn.com/data/ip2asn-v4.tsv.gz | gzip -d > /tmp/asndb.tsv
    - name: ASN -> IP Range
      run: |
        cat "$GITHUB_WORKSPACE/input/${{matrix.list.name}}/ASN.txt" | grep -v '^#' | awk '{print $1}' | grep '^AS' | while read asn; do
          echo "Processing $asn"
          awk '{if($3 == '${asn:2}') print "ipcalc -rn "$1"-"$2" | tail -n+2"}' /tmp/asndb.tsv | bash >> /tmp/asn-processed.txt
        done

        if [[ "${{matrix.list.name}}" == "datacenter" ]]; then
          cat "$GITHUB_WORKSPACE/input/vpn/ASN.txt" | grep -v '^#' | awk '{print $1}' | grep '^AS' | while read asn; do
            echo "Processing $asn"
            awk '{if($3 == '${asn:2}') print "ipcalc -rn "$1"-"$2" | tail -n+2"}' /tmp/asndb.tsv | bash >> /tmp/asn-processed.txt
          done
        fi
    - name: Merge Lists
      run: |
        # Remove too small allocations from ASN list (anything less than a /24 can't be announced)
        # TODO: This might remove some SWIP'ed ranges (needs evaluation)
        perl ./helpers/cleanup.pl /tmp/asn-processed.txt | grep -E '/(2[0-4]|1[0-9]|[0-9])$' > /tmp/asn-cleaned.txt

        # Combine lists
        cat $GITHUB_WORKSPACE/input/${{matrix.list.name}}/ips/*.txt | grep -v '^#' | awk '{print $1}' | sed '/^$/d' > /tmp/manual-processed.txt
        cat /tmp/asn-cleaned.txt /tmp/manual-processed.txt | sort -n > /tmp/ipv4.txt

        # Final cleanup
        perl ./helpers/cleanup.pl /tmp/ipv4.txt > ipv4.txt
    - name: Test generated Lists
      run: |
        if [[ $(grep ^0. ipv4.txt | wc -l) != "0" ]]; then
          echo "Lists can not contain 0.0.0.0/8"
          exit 1
        fi
        if [[ $(grep ^127. ipv4.txt | wc -l) != "0" ]]; then
          echo "Lists can not contain 127.0.0.1/8"
          exit 1
        fi

        ips_covered=$(awk -F/ 'BEGIN{a=0}{if($2==""){a+=1}else{a+=2^(32-$2)}}END{print a}' ipv4.txt)
        if [[ "$ips_covered" -lt ${{matrix.list.validate_low}} ]]; then
          echo "Too few addresses covered ($ips_covered)"
          exit 1
        fi
        if [[ "$ips_covered" -gt ${{matrix.list.validate_high}} ]]; then
          echo "Too many addresses covered ($ips_covered)"
          exit 1
        fi
    - name: Push Output of ipv4.txt
      uses: X4BNet/copy_file_to_another_repo_action@main
      env:
        API_TOKEN_GITHUB: ${{ secrets.MY_GITHUB_TOKEN }}
      with:
        source_file: 'ipv4.txt'
        destination_repo: '${{ github.repository }}'
        destination_folder: '/output/${{matrix.list.name}}/'
        user_email: 'automated@x4b.net'
        user_name: 'listbuilder'
        destination_branch: "main"
    - name: Push Output of ipv4.txt
      uses: X4BNet/copy_file_to_another_repo_action@main
      env:
        API_TOKEN_GITHUB: ${{ secrets.MY_GITHUB_TOKEN }}
      if: ${{ matrix.list.name == 'datacenter' }}
      with:
        source_file: 'ipv4.txt'
        destination_repo: '${{ github.repository }}'
        destination_folder: '/'
        user_email: 'automated@x4b.net'
        user_name: 'listbuilder'
        destination_branch: "main"