Skip to content

Commit

Permalink
Markup for ip with mask (#81)
Browse files Browse the repository at this point in the history 
* [skip actions] [main] 2024-02-27T13:31:18+02:00

* markup for IPv4 with net mask

* upd

* reduce duplicate lines in markup

* lost one

* upd

* Update benchmark/scanner/scanner.py
  • Loading branch information
@babenek
babenek authored Feb 28, 2024
1 parent d3f28c1 commit 3f3b8a1
Show file tree
Hide file tree
Showing 10 changed files with 71 additions and 3 deletions.
2 changes: 1 addition & 1 deletion benchmark/scanner/scanner.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -244,7 +244,7 @@ def check_line_from_meta(self,
file_id = row["FileID"]
# by default the cred is false positive
approximate = f"{self.next_id},{file_id},GitHub,{project_id},{path}" \
f",{line_num}:{line_num},F,F,{value_start},{value_end},F,F,,,,,0.00,,F,F,F,{rule}"
f",{line_num}:{line_num},F,F,{value_start},{value_end},F,F,,,Info,,0,0,F,F,F,{rule}"
if self._check_line_num(row["LineStart:LineEnd"], line_num):
code = str(project_id) + str(file_id) + str(row["LineStart:LineEnd"])
if code in self.line_checker:
Expand Down
2 changes: 2 additions & 0 deletions meta/0fc802c8.csv
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,3 +106,5 @@ Id,FileID,Domain,RepoName,FilePath,LineStart:LineEnd,GroundTruth,WithWords,Value
113339,297f683e,GitHub,0fc802c8,data/0fc802c8/test/297f683e.example,30:30,F,F,,,F,F,,,,,0.00,,F,F,F,Predefined Pattern
113340,297f683e,GitHub,0fc802c8,data/0fc802c8/test/297f683e.example,31:31,F,F,,,F,F,,,,,0.00,,F,F,F,Predefined Pattern
131928,d3884348,GitHub,0fc802c8,data/0fc802c8/src/d3884348.j2,241:241,T,F,,,F,F,,,,,0.00,,F,F,F,Other
134808,6a2adfaa,GitHub,0fc802c8,data/0fc802c8/src/6a2adfaa.yaml,42:42,T,F,38,45,F,F,,,Info,,0,0,F,F,F,Other

4 changes: 4 additions & 0 deletions meta/6ab520fe.csv
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,3 +69,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart:LineEnd,GroundTruth,WithWords,Value
132035,47a83faf,GitHub,6ab520fe,data/6ab520fe/src/47a83faf.json,73:73,T,F,,,F,F,,,,,0.00,,F,F,F,Other
133419,7eee71d9,GitHub,6ab520fe,data/6ab520fe/src/7eee71d9.js,3497:3497,F,F,,,F,F,,,,,0,0,F,F,F,Generic Secret
133420,eed8d9ef,GitHub,6ab520fe,data/6ab520fe/src/eed8d9ef.js,307:307,F,F,,,F,F,,,,,0,0,F,F,F,Generic Secret
134810,4952053b,GitHub,6ab520fe,data/6ab520fe/src/4952053b.py,880:880,T,F,60,67,F,F,,,Info,,0,0,F,F,F,Other
134811,28a6ce1b,GitHub,6ab520fe,data/6ab520fe/test/28a6ce1b.py,39:39,T,F,30,37,F,F,,,Info,,0,0,F,F,F,Other
134812,28a6ce1b,GitHub,6ab520fe,data/6ab520fe/test/28a6ce1b.py,105:105,T,F,1,8,F,F,,,Info,,0,0,F,F,F,Other
134813,28a6ce1b,GitHub,6ab520fe,data/6ab520fe/test/28a6ce1b.py,106:106,T,F,1,8,F,F,,,Info,,0,0,F,F,F,Other
4 changes: 4 additions & 0 deletions meta/6c73b80a.csv
View file
Original file line number Diff line number Diff line change
Expand Up @@ -824,3 +824,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart:LineEnd,GroundTruth,WithWords,Value
133425,88341713,GitHub,6c73b80a,data/6c73b80a/test/88341713.go,1452:1452,F,F,,,F,F,,,,,0,0,F,F,F,Authentication Credentials
133426,9f7a2e57,GitHub,6c73b80a,data/6c73b80a/test/9f7a2e57.conf,20:20,F,F,,,F,F,,,,,0,0,F,F,F,Generic Secret
133427,e1e39b80,GitHub,6c73b80a,data/6c73b80a/test/e1e39b80.go,1286:1286,F,F,,,F,F,,,,,0,0,F,F,F,Authentication Credentials
134814,8e39a8bd,GitHub,6c73b80a,data/6c73b80a/test/8e39a8bd.go,3915:3915,T,F,41,48,F,F,,,Info,,0,0,F,F,F,Other
134815,8e39a8bd,GitHub,6c73b80a,data/6c73b80a/test/8e39a8bd.go,3916:3916,T,F,40,47,F,F,,,Info,,0,0,F,F,F,Other
134816,8e39a8bd,GitHub,6c73b80a,data/6c73b80a/test/8e39a8bd.go,3918:3918,T,F,40,47,F,F,,,Info,,0,0,F,F,F,Other
134817,8e39a8bd,GitHub,6c73b80a,data/6c73b80a/test/8e39a8bd.go,3919:3919,T,F,41,48,F,F,,,Info,,0,0,F,F,F,Other
53 changes: 53 additions & 0 deletions meta/81cd05d0.csv
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5300,3 +5300,56 @@ Id,FileID,Domain,RepoName,FilePath,LineStart:LineEnd,GroundTruth,WithWords,Value
134790,088d0e81,GitHub,81cd05d0,data/81cd05d0/src/088d0e81.java,184:194,F,F,,,F,F,,,,,0,0,F,F,F,Other
134791,088d0e81,GitHub,81cd05d0,data/81cd05d0/src/088d0e81.java,190:194,F,F,,,F,F,,,,,0,0,F,F,F,Other
134792,bc39da73,GitHub,81cd05d0,data/81cd05d0/src/bc39da73.json,163872:163874,F,F,,,F,F,,,,,0,0,F,F,F,Other
134819,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,129836:129836,T,F,214,225,F,F,,,Info,,0,0,F,F,F,Other
134821,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,129839:129839,T,F,229,240,F,F,,,Info,,0,0,F,F,F,Other
134825,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,129841:129841,T,F,223,234,F,F,,,Info,,0,0,F,F,F,Other
134829,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,129875:129875,T,F,223,234,F,F,,,Info,,0,0,F,F,F,Other
134833,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,129890:129890,T,F,229,240,F,F,,,Info,,0,0,F,F,F,Other
134837,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,129896:129896,T,F,214,225,F,F,,,Info,,0,0,F,F,F,Other
134840,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,130459:130459,T,F,259,269,F,F,,,Info,,0,0,F,F,F,Other
134841,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,130462:130462,T,F,229,240,F,F,,,Info,,0,0,F,F,F,Other
134848,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,130464:130464,T,F,540,550,F,F,,,Info,,0,0,F,F,F,Other
134849,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,130498:130498,T,F,223,234,F,F,,,Info,,0,0,F,F,F,Other
134856,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,130513:130513,T,F,555,565,F,F,,,Info,,0,0,F,F,F,Other
134857,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,130519:130519,T,F,214,225,F,F,,,Info,,0,0,F,F,F,Other
134860,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,140730:140730,T,F,275,285,F,F,,,Info,,0,0,F,F,F,Other
134861,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,140733:140733,T,F,245,256,F,F,,,Info,,0,0,F,F,F,Other
134868,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,140735:140735,T,F,572,582,F,F,,,Info,,0,0,F,F,F,Other
134869,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,140769:140769,T,F,239,250,F,F,,,Info,,0,0,F,F,F,Other
134876,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,140784:140784,T,F,598,608,F,F,,,Info,,0,0,F,F,F,Other
134877,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,140790:140790,T,F,230,241,F,F,,,Info,,0,0,F,F,F,Other
134880,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,141768:141768,T,F,275,285,F,F,,,Info,,0,0,F,F,F,Other
134881,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,141771:141771,T,F,245,256,F,F,,,Info,,0,0,F,F,F,Other
134888,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,141773:141773,T,F,572,582,F,F,,,Info,,0,0,F,F,F,Other
134889,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,141807:141807,T,F,239,250,F,F,,,Info,,0,0,F,F,F,Other
134896,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,141822:141822,T,F,598,608,F,F,,,Info,,0,0,F,F,F,Other
134897,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,141828:141828,T,F,230,241,F,F,,,Info,,0,0,F,F,F,Other
134900,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,148696:148696,T,F,255,265,F,F,,,Info,,0,0,F,F,F,Other
134901,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,148699:148699,T,F,225,236,F,F,,,Info,,0,0,F,F,F,Other
134908,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,148701:148701,T,F,532,542,F,F,,,Info,,0,0,F,F,F,Other
134909,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,148735:148735,T,F,219,230,F,F,,,Info,,0,0,F,F,F,Other
134916,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,148750:148750,T,F,547,557,F,F,,,Info,,0,0,F,F,F,Other
134917,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,148756:148756,T,F,210,221,F,F,,,Info,,0,0,F,F,F,Other
134920,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,149114:149114,T,F,255,265,F,F,,,Info,,0,0,F,F,F,Other
134921,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,149117:149117,T,F,225,236,F,F,,,Info,,0,0,F,F,F,Other
134928,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,149119:149119,T,F,532,542,F,F,,,Info,,0,0,F,F,F,Other
134929,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,149153:149153,T,F,219,230,F,F,,,Info,,0,0,F,F,F,Other
134936,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,149168:149168,T,F,547,557,F,F,,,Info,,0,0,F,F,F,Other
134937,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,149174:149174,T,F,210,221,F,F,,,Info,,0,0,F,F,F,Other
134940,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,175342:175342,T,F,252,262,F,F,,,Info,,0,0,F,F,F,Other
134941,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,175345:175345,T,F,222,233,F,F,,,Info,,0,0,F,F,F,Other
134948,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,175347:175347,T,F,526,536,F,F,,,Info,,0,0,F,F,F,Other
134949,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,175381:175381,T,F,216,227,F,F,,,Info,,0,0,F,F,F,Other
134956,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,175396:175396,T,F,541,551,F,F,,,Info,,0,0,F,F,F,Other
134957,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,175402:175402,T,F,207,218,F,F,,,Info,,0,0,F,F,F,Other
134960,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,175981:175981,T,F,252,262,F,F,,,Info,,0,0,F,F,F,Other
134961,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,175984:175984,T,F,222,233,F,F,,,Info,,0,0,F,F,F,Other
134968,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,175986:175986,T,F,526,536,F,F,,,Info,,0,0,F,F,F,Other
134969,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,176020:176020,T,F,216,227,F,F,,,Info,,0,0,F,F,F,Other
134976,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,176035:176035,T,F,541,551,F,F,,,Info,,0,0,F,F,F,Other
134977,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,176041:176041,T,F,207,218,F,F,,,Info,,0,0,F,F,F,Other
134980,d7225510,GitHub,81cd05d0,data/81cd05d0/src/d7225510.json,9777:9777,T,F,257,267,F,F,,,Info,,0,0,F,F,F,Other
134981,d7225510,GitHub,81cd05d0,data/81cd05d0/src/d7225510.json,10205:10205,T,F,228,239,F,F,,,Info,,0,0,F,F,F,Other
134984,d7225510,GitHub,81cd05d0,data/81cd05d0/src/d7225510.json,10523:10523,T,F,253,263,F,F,,,Info,,0,0,F,F,F,Other
134985,d7225510,GitHub,81cd05d0,data/81cd05d0/src/d7225510.json,11517:11517,T,F,205,216,F,F,,,Info,,0,0,F,F,F,Other

1 change: 1 addition & 0 deletions meta/c2d95db7.csv
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,3 +147,4 @@ Id,FileID,Domain,RepoName,FilePath,LineStart:LineEnd,GroundTruth,WithWords,Value
130009,7ccdfb29,GitHub,c2d95db7,data/c2d95db7/other/7ccdfb29.md,1362:1362,F,F,,,F,F,,,,,0.00,,F,F,F,Password
132707,8e165a9d,GitHub,c2d95db7,data/c2d95db7/src/8e165a9d.sh,372:372,T,F,,,F,F,,,,,0.00,,F,F,F,Password
134369,20a8598d,GitHub,c2d95db7,data/c2d95db7/src/20a8598d.sh,35:35,F,F,,,F,F,,,,,0,0,F,F,F,Predefined Pattern
134987,e45e45ba,GitHub,c2d95db7,data/c2d95db7/src/e45e45ba.yml,1365:1365,T,F,57,66,F,F,,,Info,,0,0,F,F,F,Other
2 changes: 2 additions & 0 deletions meta/cc51a2f0.csv
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,3 +140,5 @@ Id,FileID,Domain,RepoName,FilePath,LineStart:LineEnd,GroundTruth,WithWords,Value
134596,c65d41b8,GitHub,cc51a2f0,data/cc51a2f0/test/c65d41b8.bats,178:178,F,F,,,F,F,,,,,0,0,F,F,F,Authentication Credentials
134597,c65d41b8,GitHub,cc51a2f0,data/cc51a2f0/test/c65d41b8.bats,934:934,F,F,,,F,F,,,,,0,0,F,F,F,Authentication Credentials
134598,c65d41b8,GitHub,cc51a2f0,data/cc51a2f0/test/c65d41b8.bats,971:971,F,F,,,F,F,,,,,0,0,F,F,F,Authentication Credentials
134988,9a13a4f2,GitHub,cc51a2f0,data/cc51a2f0/src/9a13a4f2.sh,40:40,T,F,7,14,F,F,,,Info,,0,0,F,F,F,Other
134989,9a13a4f2,GitHub,cc51a2f0,data/cc51a2f0/src/9a13a4f2.sh,41:41,T,F,7,14,F,F,,,Info,,0,0,F,F,F,Other
2 changes: 2 additions & 0 deletions meta/e5a9c76d.csv
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,3 +59,5 @@ Id,FileID,Domain,RepoName,FilePath,LineStart:LineEnd,GroundTruth,WithWords,Value
113774,c1823fb9,GitHub,e5a9c76d,data/e5a9c76d/other/c1823fb9.md,1121:1121,F,F,,,F,F,,,,,0.00,,F,F,F,Generic Secret
113775,c1823fb9,GitHub,e5a9c76d,data/e5a9c76d/other/c1823fb9.md,2376:2376,F,F,,,F,F,,,,,0.00,,F,F,F,Authentication Credentials
113776,c1823fb9,GitHub,e5a9c76d,data/e5a9c76d/other/c1823fb9.md,935:935,F,F,,,F,F,,,,,0.00,,F,F,F,Generic Secret
134990,c1823fb9,GitHub,e5a9c76d,data/e5a9c76d/other/c1823fb9.md,814:814,T,F,34,47,F,F,,,Info,,0,0,F,F,F,Other
134991,692b71dc,GitHub,e5a9c76d,data/e5a9c76d/src/692b71dc.rb,15:15,T,F,17,30,F,F,,,Info,,0,0,F,F,F,Other
2 changes: 1 addition & 1 deletion review_data.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,7 +174,7 @@ def main(meta_dir, data_dir, data_filter, load_json: Optional[str] = None, categ
except Exception as exc:
print(f"Failure {row}", exc, flush=True)
raise
row_str = f"{file_path},{line_start}:{line_end}"
row_str = f"{file_path},{line_start}:{line_end},{value_start},{value_end}"
if row_str in shown_rows:
print(f"Duplicate row {row}", flush=True)
break
Expand Down
2 changes: 1 addition & 1 deletion snapshot.yaml
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# 20240216
# 20240227_2215
- id: pBfFrJ0X
sha: 2abd898038aa362391856f91fd4f5abe33e1e88a
tag: v3.3.2-1-g2abd898
Expand Down

0 comments on commit 3f3b8a1

Please sign in to comment.