From 3f3b8a16973fffeaa6231a178e9117e10fc8bdc0 Mon Sep 17 00:00:00 2001 From: Roman Babenko Date: Wed, 28 Feb 2024 10:15:29 +0200 Subject: [PATCH] Markup for ip with mask (#81) * [skip actions] [main] 2024-02-27T13:31:18+02:00 * markup for IPv4 with net mask * upd * reduce duplicate lines in markup * lost one * upd * Update benchmark/scanner/scanner.py --- benchmark/scanner/scanner.py | 2 +- meta/0fc802c8.csv | 2 ++ meta/6ab520fe.csv | 4 +++ meta/6c73b80a.csv | 4 +++ meta/81cd05d0.csv | 53 ++++++++++++++++++++++++++++++++++++ meta/c2d95db7.csv | 1 + meta/cc51a2f0.csv | 2 ++ meta/e5a9c76d.csv | 2 ++ review_data.py | 2 +- snapshot.yaml | 2 +- 10 files changed, 71 insertions(+), 3 deletions(-) diff --git a/benchmark/scanner/scanner.py b/benchmark/scanner/scanner.py index e64f25335..03bc90a9f 100644 --- a/benchmark/scanner/scanner.py +++ b/benchmark/scanner/scanner.py @@ -244,7 +244,7 @@ def check_line_from_meta(self, file_id = row["FileID"] # by default the cred is false positive approximate = f"{self.next_id},{file_id},GitHub,{project_id},{path}" \ - f",{line_num}:{line_num},F,F,{value_start},{value_end},F,F,,,,,0.00,,F,F,F,{rule}" + f",{line_num}:{line_num},F,F,{value_start},{value_end},F,F,,,Info,,0,0,F,F,F,{rule}" if self._check_line_num(row["LineStart:LineEnd"], line_num): code = str(project_id) + str(file_id) + str(row["LineStart:LineEnd"]) if code in self.line_checker: diff --git a/meta/0fc802c8.csv b/meta/0fc802c8.csv index 705f7db22..442e4fa06 100644 --- a/meta/0fc802c8.csv +++ b/meta/0fc802c8.csv @@ -106,3 +106,5 @@ Id,FileID,Domain,RepoName,FilePath,LineStart:LineEnd,GroundTruth,WithWords,Value 113339,297f683e,GitHub,0fc802c8,data/0fc802c8/test/297f683e.example,30:30,F,F,,,F,F,,,,,0.00,,F,F,F,Predefined Pattern 113340,297f683e,GitHub,0fc802c8,data/0fc802c8/test/297f683e.example,31:31,F,F,,,F,F,,,,,0.00,,F,F,F,Predefined Pattern 131928,d3884348,GitHub,0fc802c8,data/0fc802c8/src/d3884348.j2,241:241,T,F,,,F,F,,,,,0.00,,F,F,F,Other +134808,6a2adfaa,GitHub,0fc802c8,data/0fc802c8/src/6a2adfaa.yaml,42:42,T,F,38,45,F,F,,,Info,,0,0,F,F,F,Other + diff --git a/meta/6ab520fe.csv b/meta/6ab520fe.csv index 94bfd478b..2255792cc 100644 --- a/meta/6ab520fe.csv +++ b/meta/6ab520fe.csv @@ -69,3 +69,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart:LineEnd,GroundTruth,WithWords,Value 132035,47a83faf,GitHub,6ab520fe,data/6ab520fe/src/47a83faf.json,73:73,T,F,,,F,F,,,,,0.00,,F,F,F,Other 133419,7eee71d9,GitHub,6ab520fe,data/6ab520fe/src/7eee71d9.js,3497:3497,F,F,,,F,F,,,,,0,0,F,F,F,Generic Secret 133420,eed8d9ef,GitHub,6ab520fe,data/6ab520fe/src/eed8d9ef.js,307:307,F,F,,,F,F,,,,,0,0,F,F,F,Generic Secret +134810,4952053b,GitHub,6ab520fe,data/6ab520fe/src/4952053b.py,880:880,T,F,60,67,F,F,,,Info,,0,0,F,F,F,Other +134811,28a6ce1b,GitHub,6ab520fe,data/6ab520fe/test/28a6ce1b.py,39:39,T,F,30,37,F,F,,,Info,,0,0,F,F,F,Other +134812,28a6ce1b,GitHub,6ab520fe,data/6ab520fe/test/28a6ce1b.py,105:105,T,F,1,8,F,F,,,Info,,0,0,F,F,F,Other +134813,28a6ce1b,GitHub,6ab520fe,data/6ab520fe/test/28a6ce1b.py,106:106,T,F,1,8,F,F,,,Info,,0,0,F,F,F,Other diff --git a/meta/6c73b80a.csv b/meta/6c73b80a.csv index 0550a64ca..ac96c8da1 100644 --- a/meta/6c73b80a.csv +++ b/meta/6c73b80a.csv @@ -824,3 +824,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart:LineEnd,GroundTruth,WithWords,Value 133425,88341713,GitHub,6c73b80a,data/6c73b80a/test/88341713.go,1452:1452,F,F,,,F,F,,,,,0,0,F,F,F,Authentication Credentials 133426,9f7a2e57,GitHub,6c73b80a,data/6c73b80a/test/9f7a2e57.conf,20:20,F,F,,,F,F,,,,,0,0,F,F,F,Generic Secret 133427,e1e39b80,GitHub,6c73b80a,data/6c73b80a/test/e1e39b80.go,1286:1286,F,F,,,F,F,,,,,0,0,F,F,F,Authentication Credentials +134814,8e39a8bd,GitHub,6c73b80a,data/6c73b80a/test/8e39a8bd.go,3915:3915,T,F,41,48,F,F,,,Info,,0,0,F,F,F,Other +134815,8e39a8bd,GitHub,6c73b80a,data/6c73b80a/test/8e39a8bd.go,3916:3916,T,F,40,47,F,F,,,Info,,0,0,F,F,F,Other +134816,8e39a8bd,GitHub,6c73b80a,data/6c73b80a/test/8e39a8bd.go,3918:3918,T,F,40,47,F,F,,,Info,,0,0,F,F,F,Other +134817,8e39a8bd,GitHub,6c73b80a,data/6c73b80a/test/8e39a8bd.go,3919:3919,T,F,41,48,F,F,,,Info,,0,0,F,F,F,Other diff --git a/meta/81cd05d0.csv b/meta/81cd05d0.csv index a7fab310e..bc7a265f4 100644 --- a/meta/81cd05d0.csv +++ b/meta/81cd05d0.csv @@ -5300,3 +5300,56 @@ Id,FileID,Domain,RepoName,FilePath,LineStart:LineEnd,GroundTruth,WithWords,Value 134790,088d0e81,GitHub,81cd05d0,data/81cd05d0/src/088d0e81.java,184:194,F,F,,,F,F,,,,,0,0,F,F,F,Other 134791,088d0e81,GitHub,81cd05d0,data/81cd05d0/src/088d0e81.java,190:194,F,F,,,F,F,,,,,0,0,F,F,F,Other 134792,bc39da73,GitHub,81cd05d0,data/81cd05d0/src/bc39da73.json,163872:163874,F,F,,,F,F,,,,,0,0,F,F,F,Other +134819,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,129836:129836,T,F,214,225,F,F,,,Info,,0,0,F,F,F,Other +134821,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,129839:129839,T,F,229,240,F,F,,,Info,,0,0,F,F,F,Other +134825,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,129841:129841,T,F,223,234,F,F,,,Info,,0,0,F,F,F,Other +134829,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,129875:129875,T,F,223,234,F,F,,,Info,,0,0,F,F,F,Other +134833,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,129890:129890,T,F,229,240,F,F,,,Info,,0,0,F,F,F,Other +134837,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,129896:129896,T,F,214,225,F,F,,,Info,,0,0,F,F,F,Other +134840,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,130459:130459,T,F,259,269,F,F,,,Info,,0,0,F,F,F,Other +134841,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,130462:130462,T,F,229,240,F,F,,,Info,,0,0,F,F,F,Other +134848,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,130464:130464,T,F,540,550,F,F,,,Info,,0,0,F,F,F,Other +134849,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,130498:130498,T,F,223,234,F,F,,,Info,,0,0,F,F,F,Other +134856,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,130513:130513,T,F,555,565,F,F,,,Info,,0,0,F,F,F,Other +134857,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,130519:130519,T,F,214,225,F,F,,,Info,,0,0,F,F,F,Other +134860,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,140730:140730,T,F,275,285,F,F,,,Info,,0,0,F,F,F,Other +134861,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,140733:140733,T,F,245,256,F,F,,,Info,,0,0,F,F,F,Other +134868,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,140735:140735,T,F,572,582,F,F,,,Info,,0,0,F,F,F,Other +134869,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,140769:140769,T,F,239,250,F,F,,,Info,,0,0,F,F,F,Other +134876,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,140784:140784,T,F,598,608,F,F,,,Info,,0,0,F,F,F,Other +134877,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,140790:140790,T,F,230,241,F,F,,,Info,,0,0,F,F,F,Other +134880,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,141768:141768,T,F,275,285,F,F,,,Info,,0,0,F,F,F,Other +134881,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,141771:141771,T,F,245,256,F,F,,,Info,,0,0,F,F,F,Other +134888,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,141773:141773,T,F,572,582,F,F,,,Info,,0,0,F,F,F,Other +134889,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,141807:141807,T,F,239,250,F,F,,,Info,,0,0,F,F,F,Other +134896,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,141822:141822,T,F,598,608,F,F,,,Info,,0,0,F,F,F,Other +134897,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,141828:141828,T,F,230,241,F,F,,,Info,,0,0,F,F,F,Other +134900,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,148696:148696,T,F,255,265,F,F,,,Info,,0,0,F,F,F,Other +134901,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,148699:148699,T,F,225,236,F,F,,,Info,,0,0,F,F,F,Other +134908,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,148701:148701,T,F,532,542,F,F,,,Info,,0,0,F,F,F,Other +134909,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,148735:148735,T,F,219,230,F,F,,,Info,,0,0,F,F,F,Other +134916,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,148750:148750,T,F,547,557,F,F,,,Info,,0,0,F,F,F,Other +134917,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,148756:148756,T,F,210,221,F,F,,,Info,,0,0,F,F,F,Other +134920,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,149114:149114,T,F,255,265,F,F,,,Info,,0,0,F,F,F,Other +134921,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,149117:149117,T,F,225,236,F,F,,,Info,,0,0,F,F,F,Other +134928,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,149119:149119,T,F,532,542,F,F,,,Info,,0,0,F,F,F,Other +134929,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,149153:149153,T,F,219,230,F,F,,,Info,,0,0,F,F,F,Other +134936,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,149168:149168,T,F,547,557,F,F,,,Info,,0,0,F,F,F,Other +134937,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,149174:149174,T,F,210,221,F,F,,,Info,,0,0,F,F,F,Other +134940,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,175342:175342,T,F,252,262,F,F,,,Info,,0,0,F,F,F,Other +134941,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,175345:175345,T,F,222,233,F,F,,,Info,,0,0,F,F,F,Other +134948,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,175347:175347,T,F,526,536,F,F,,,Info,,0,0,F,F,F,Other +134949,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,175381:175381,T,F,216,227,F,F,,,Info,,0,0,F,F,F,Other +134956,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,175396:175396,T,F,541,551,F,F,,,Info,,0,0,F,F,F,Other +134957,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,175402:175402,T,F,207,218,F,F,,,Info,,0,0,F,F,F,Other +134960,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,175981:175981,T,F,252,262,F,F,,,Info,,0,0,F,F,F,Other +134961,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,175984:175984,T,F,222,233,F,F,,,Info,,0,0,F,F,F,Other +134968,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,175986:175986,T,F,526,536,F,F,,,Info,,0,0,F,F,F,Other +134969,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,176020:176020,T,F,216,227,F,F,,,Info,,0,0,F,F,F,Other +134976,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,176035:176035,T,F,541,551,F,F,,,Info,,0,0,F,F,F,Other +134977,02fd2d8d,GitHub,81cd05d0,data/81cd05d0/src/02fd2d8d.json,176041:176041,T,F,207,218,F,F,,,Info,,0,0,F,F,F,Other +134980,d7225510,GitHub,81cd05d0,data/81cd05d0/src/d7225510.json,9777:9777,T,F,257,267,F,F,,,Info,,0,0,F,F,F,Other +134981,d7225510,GitHub,81cd05d0,data/81cd05d0/src/d7225510.json,10205:10205,T,F,228,239,F,F,,,Info,,0,0,F,F,F,Other +134984,d7225510,GitHub,81cd05d0,data/81cd05d0/src/d7225510.json,10523:10523,T,F,253,263,F,F,,,Info,,0,0,F,F,F,Other +134985,d7225510,GitHub,81cd05d0,data/81cd05d0/src/d7225510.json,11517:11517,T,F,205,216,F,F,,,Info,,0,0,F,F,F,Other + diff --git a/meta/c2d95db7.csv b/meta/c2d95db7.csv index 3d977fd99..c1f0cd3c5 100644 --- a/meta/c2d95db7.csv +++ b/meta/c2d95db7.csv @@ -147,3 +147,4 @@ Id,FileID,Domain,RepoName,FilePath,LineStart:LineEnd,GroundTruth,WithWords,Value 130009,7ccdfb29,GitHub,c2d95db7,data/c2d95db7/other/7ccdfb29.md,1362:1362,F,F,,,F,F,,,,,0.00,,F,F,F,Password 132707,8e165a9d,GitHub,c2d95db7,data/c2d95db7/src/8e165a9d.sh,372:372,T,F,,,F,F,,,,,0.00,,F,F,F,Password 134369,20a8598d,GitHub,c2d95db7,data/c2d95db7/src/20a8598d.sh,35:35,F,F,,,F,F,,,,,0,0,F,F,F,Predefined Pattern +134987,e45e45ba,GitHub,c2d95db7,data/c2d95db7/src/e45e45ba.yml,1365:1365,T,F,57,66,F,F,,,Info,,0,0,F,F,F,Other diff --git a/meta/cc51a2f0.csv b/meta/cc51a2f0.csv index ae8ce20f3..bf73762e3 100644 --- a/meta/cc51a2f0.csv +++ b/meta/cc51a2f0.csv @@ -140,3 +140,5 @@ Id,FileID,Domain,RepoName,FilePath,LineStart:LineEnd,GroundTruth,WithWords,Value 134596,c65d41b8,GitHub,cc51a2f0,data/cc51a2f0/test/c65d41b8.bats,178:178,F,F,,,F,F,,,,,0,0,F,F,F,Authentication Credentials 134597,c65d41b8,GitHub,cc51a2f0,data/cc51a2f0/test/c65d41b8.bats,934:934,F,F,,,F,F,,,,,0,0,F,F,F,Authentication Credentials 134598,c65d41b8,GitHub,cc51a2f0,data/cc51a2f0/test/c65d41b8.bats,971:971,F,F,,,F,F,,,,,0,0,F,F,F,Authentication Credentials +134988,9a13a4f2,GitHub,cc51a2f0,data/cc51a2f0/src/9a13a4f2.sh,40:40,T,F,7,14,F,F,,,Info,,0,0,F,F,F,Other +134989,9a13a4f2,GitHub,cc51a2f0,data/cc51a2f0/src/9a13a4f2.sh,41:41,T,F,7,14,F,F,,,Info,,0,0,F,F,F,Other diff --git a/meta/e5a9c76d.csv b/meta/e5a9c76d.csv index f37781437..ea33b03ae 100644 --- a/meta/e5a9c76d.csv +++ b/meta/e5a9c76d.csv @@ -59,3 +59,5 @@ Id,FileID,Domain,RepoName,FilePath,LineStart:LineEnd,GroundTruth,WithWords,Value 113774,c1823fb9,GitHub,e5a9c76d,data/e5a9c76d/other/c1823fb9.md,1121:1121,F,F,,,F,F,,,,,0.00,,F,F,F,Generic Secret 113775,c1823fb9,GitHub,e5a9c76d,data/e5a9c76d/other/c1823fb9.md,2376:2376,F,F,,,F,F,,,,,0.00,,F,F,F,Authentication Credentials 113776,c1823fb9,GitHub,e5a9c76d,data/e5a9c76d/other/c1823fb9.md,935:935,F,F,,,F,F,,,,,0.00,,F,F,F,Generic Secret +134990,c1823fb9,GitHub,e5a9c76d,data/e5a9c76d/other/c1823fb9.md,814:814,T,F,34,47,F,F,,,Info,,0,0,F,F,F,Other +134991,692b71dc,GitHub,e5a9c76d,data/e5a9c76d/src/692b71dc.rb,15:15,T,F,17,30,F,F,,,Info,,0,0,F,F,F,Other diff --git a/review_data.py b/review_data.py index 728bbc787..c735d1f40 100644 --- a/review_data.py +++ b/review_data.py @@ -174,7 +174,7 @@ def main(meta_dir, data_dir, data_filter, load_json: Optional[str] = None, categ except Exception as exc: print(f"Failure {row}", exc, flush=True) raise - row_str = f"{file_path},{line_start}:{line_end}" + row_str = f"{file_path},{line_start}:{line_end},{value_start},{value_end}" if row_str in shown_rows: print(f"Duplicate row {row}", flush=True) break diff --git a/snapshot.yaml b/snapshot.yaml index 16af2959a..809206ca3 100644 --- a/snapshot.yaml +++ b/snapshot.yaml @@ -1,4 +1,4 @@ -# 20240216 +# 20240227_2215 - id: pBfFrJ0X sha: 2abd898038aa362391856f91fd4f5abe33e1e88a tag: v3.3.2-1-g2abd898