Merge branch 'main' into py312

Samsung · Aug 10, 2024 · 9d0dd22 · 9d0dd22
2 parents 2f52bbc + 061d0d5
commit 9d0dd22
Show file tree

Hide file tree

Showing 265 changed files with 12,662 additions and 16,646 deletions.
diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
@@ -39,11 +39,11 @@ jobs:
           path: data
           key: cred-data-${{ hashFiles('checksums.md5') }}
 
-      - name: Set up Python 3.8
+      - name: Set up Python 3.10
         if: steps.cache-data.outputs.cache-hit != 'true'
         uses: actions/setup-python@v4
         with:
-          python-version: "3.8"
+          python-version: "3.10"
 
       - name: Update PIP
         run: python -m pip install --upgrade pip
@@ -97,10 +97,10 @@ jobs:
         if: steps.cache-data.outputs.cache-hit == 'true'
         run: ls -al . && ls -al data
 
-      - name: Set up Python 3.8
+      - name: Set up Python 3.10
         uses: actions/setup-python@v4
         with:
-          python-version: "3.8"
+          python-version: "3.10"
 
       - name: Update PIP
         run: python -m pip install --upgrade pip
@@ -114,19 +114,14 @@ jobs:
           ref: ${{ github.event.pull_request.head.sha }}
           path: temp/CredSweeper
 
-      - name: Patch benchmark for PR work
-        run: |
-          sed -i 's|CREDSWEEPER = "https://github.com/Samsung/CredSweeper.git"|CREDSWEEPER = "dummy://github.com/Samsung/CredSweeper.git"|' benchmark/common/constants.py
-          grep --with-filename --line-number 'dummy://github.com/Samsung/CredSweeper.git' benchmark/common/constants.py
-
       - name: Install CredSweeper
         run: |
           python -m pip install temp/CredSweeper
-          credsweeper_head=
+          python -m credsweeper --banner
 
       - name: Run CredSweeper tool
         run: |
-          credsweeper --banner --jobs $(nproc) --path data --save-json report.${{ github.event.pull_request.head.sha }}.json | tee credsweeper.${{ github.event.pull_request.head.sha }}.log
+          credsweeper --banner --log info --jobs $(nproc) --path data --save-json report.${{ github.event.pull_request.head.sha }}.json | tee credsweeper.${{ github.event.pull_request.head.sha }}.log
 
       - name: Run Benchmark
         run: |
@@ -160,14 +155,14 @@ jobs:
   # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 
   performance_benchmark:
-    # put the benchmark in single job to keep constant environment during test
+    # put the benchmark in single job to keep constant environment during test python 3.8 is not applicable
     needs: [ download_data ]
 
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        python-version: [ "3.8", "3.9", "3.10", "3.11" ]
+        python-version: [ "3.9", "3.10", "3.11" ]
 
     steps:
 
@@ -197,7 +192,7 @@ jobs:
 
       - name: Exclude very huge data
         if: steps.cache-data.outputs.cache-hit == 'true'
-        run: rm -rf data/8* data/7* data/a* data/2* data/0* data/f*  data/b* data/d*
+        run: rm -rf data/0* data/2* data/7* data/8* data/a* data/b* data/d* data/e* data/f*
 
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4
@@ -292,7 +287,7 @@ jobs:
           exit_code=0
           LOW_DELTA=10
           THRESHOLD=250
-          
+
           # RELEASE
           if [ ${RELEASE_TIME} -le ${HEAD_TIME} ]; then
             d=$(( 1000 * ( ${HEAD_TIME} - ${RELEASE_TIME} ) / ${RELEASE_TIME} ))
@@ -316,7 +311,7 @@ jobs:
               echo "Speed-up."
             fi
           fi
-          
+
           # BASE
           if [ ${BASE_TIME} -le ${HEAD_TIME} ]; then
             d=$(( 1000 * ( ${HEAD_TIME} - ${BASE_TIME} ) / ${BASE_TIME} ))
@@ -340,10 +335,11 @@ jobs:
               echo "Speed-up."
             fi
           fi
-          
+
           exit ${exit_code}
 
   # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
   experiment:
     # the ml train test is placed here to use cached data set
     needs: [ download_data ]
@@ -384,11 +380,11 @@ jobs:
           mv data ${{ github.workspace }}/CredData/
           mv meta ${{ github.workspace }}/CredData/
 
-      - name: Set up Python 3.8
+      - name: Set up Python 3.10
         if: steps.cache-data.outputs.cache-hit != 'true'
         uses: actions/setup-python@v3
         with:
-          python-version: "3.8"
+          python-version: "3.10"
 
       - name: Update PIP
         run: python -m pip install --upgrade pip
@@ -419,10 +415,8 @@ jobs:
           # check whether credsweeper is available as module
           python -m credsweeper --banner
           # use only 2 epochs for the test
-          sed -i 's/epochs=42,/epochs=2,/' main.py
+          sed -i 's/max_epochs = .*/max_epochs = 2/' main.py
           python main.py --data ${{ github.workspace }}/CredData -j $(( 2 * $(nproc) ))
-          ls -al results #dbg
-          python -m tf2onnx.convert --saved-model $(find results -mindepth 1 -maxdepth 1 -type d) --output ../credsweeper/ml_model/ml_model.onnx --verbose
           # dbg
           git diff
           # crc32 should be changed
@@ -435,24 +429,34 @@ jobs:
             exit 1
           fi
 
-# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 
   run_doc_benchmark:
     runs-on: ubuntu-latest
-    if: ${{ 'Samsung/CredSweeper' == github.event.pull_request.head.repo.full_name }}
+    if: ${{ 'push' == github.event_name }} or ${{ 'Samsung/CredSweeper' == github.event.pull_request.head.repo.full_name }}
     steps:
-      - name: Checkout CredSweeper
+      - name: Checkout CredSweeper PR
         if: ${{ 'pull_request' == github.event_name }}
         uses: actions/checkout@v4
         with:
           ref: ${{ github.event.pull_request.head.sha }}
 
+      - name: Checkout CredSweeper HEAD
+        if: ${{ 'push' == github.event_name }}
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.head }}
+
       - name: Send cURL request with the commit SHA
-        if: ${{ 'pull_request' == github.event_name }}
         run: |
-          COMMIT_SHA=$(git rev-parse HEAD)
-          curl -X POST ${{ secrets.SLACK_URL }} \
-          --data-urlencode \
-          "payload={'text':'[BMT Request] ${{ github.event.repository.html_url }}/commit/${COMMIT_SHA}'}"
+          if [[ "${{ secrets.SLACK_URL }}" =~ http.*/.*/.* ]]; then
+            COMMIT_SHA=$(git rev-parse HEAD)
+            echo ${COMMIT_SHA}
+            curl -X POST ${{ secrets.SLACK_URL }} \
+            --data-urlencode \
+            "payload={'text':'[BMT Request] ${{ github.event.repository.html_url }}/commit/${COMMIT_SHA}'}"
+          else
+            echo "secrets.SLACK_URL is not available"
+          fi
 
-# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml
@@ -58,7 +58,7 @@ jobs:
     - name: Check ml_model.onnx integrity
       if: ${{ always() && steps.code_checkout.conclusion == 'success' }}
       run: |
-        md5sum --binary credsweeper/ml_model/ml_model.onnx | grep 57ec152f6aa740456c742ecd5e7d9ef5
+        md5sum --binary credsweeper/ml_model/ml_model.onnx | grep 62d92ab2f91a18e861d846a7b8a0c3a7
 
     # # # Python setup
 
@@ -196,9 +196,9 @@ jobs:
             file_crc32_int=$((16#${file_crc32_hex}))
             crc32_int=$(( ${crc32_int} ^ ${file_crc32_int} ))
             done
-        version_with_crc="$(credsweeper --version | head -1) crc32:$(printf '%x' ${crc32_int})"
+        version_with_crc="$(python -m credsweeper --version | head -1) crc32:$(printf '%x' ${crc32_int})"
         echo "version_with_crc = '${version_with_crc}'"
-        banner=$(credsweeper --banner --path requirements.txt | head -1)
+        banner=$(python -m credsweeper --banner | head -1)
         echo "banner = '${banner}'"
         if ! [ -n "${version_with_crc}" ] && [ -n "${banner}" ] && [ "${version_with_crc}" == "${banner}" ]; then
             echo "'${version_with_crc}' != '${banner}'"

diff --git a/README.md b/README.md
@@ -8,6 +8,7 @@
 [![Test](https://github.com/Samsung/CredSweeper/actions/workflows/test.yml/badge.svg)](https://github.com/Samsung/CredSweeper/actions/workflows/test.yml)
 [![codecov](https://codecov.io/gh/Samsung/CredSweeper/branch/main/graph/badge.svg)](https://codecov.io/gh/Samsung/CredSweeper)
 [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/6055/badge)](https://bestpractices.coreinfrastructure.org/projects/6055)
+[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/Samsung/CredSweeper/badge)](https://api.securityscorecards.dev/projects/github.com/Samsung/CredSweeper)
 
 <img src="https://raw.githubusercontent.com/Samsung/CredSweeper/main/docs/images/Logo.png" width="500"/>
 

diff --git a/SECURITY.md b/SECURITY.md
@@ -4,8 +4,8 @@
 
 | Version | Supported          |
 |---------|--------------------|
-| 1.6.x   | :white_check_mark: |
-| <1.6.x  | :x:                |
+| 1.8.x   | :white_check_mark: |
+| <1.8.x  | :x:                |
 
 ## Reporting a Vulnerability