aboutcode-org · AyanSinhaMahapatra · Feb 15, 2023 · Feb 15, 2023 · Feb 15, 2023 · Feb 15, 2023
diff --git a/docs/source/how-to-guides/add_new_license_detection_rule.rst b/docs/source/how-to-guides/add_new_license_detection_rule.rst
@@ -73,7 +73,7 @@ More (advanced) rules options:
   be present in the result license detections. These just have the license text and a
   `is_false_positive` flag set to True.
 
-- you can specify key phrases by surrounding one or more words between the `{{`
+- you can specify required phrases by surrounding one or more words between the `{{`
   and `}}` tags. Key phrases are words that **must** be matched/present in order
   for a RULE to be considered a match.
 

diff --git a/etc/scripts/licenses/buildrules.py b/etc/scripts/licenses/buildrules.py
@@ -16,6 +16,7 @@
 from licensedcode import models
 from licensedcode import match_hash
 from licensedcode import frontmatter
+from licensedcode.models import get_rule_id_for_text
 from license_expression import Licensing
 
 """
@@ -129,23 +130,6 @@ def load_data(location="00-new-licenses.txt"):
     return rules
 
 
-def rule_exists(text):
-    """
-    Return the matched rule identifier if the text is an existing rule matched
-    exactly, False otherwise.
-    """
-    idx = cache.get_index()
-
-    matches = idx.match(query_string=text)
-    if not matches:
-        return False
-    if len(matches) > 1:
-        return False
-    match = matches[0]
-    if match.matcher == match_hash.MATCH_HASH and match.score() == 100:
-        return match.rule.identifier
-
-
 def all_rule_by_tokens():
     """
     Return a mapping of {tuples of tokens: rule id}, with one item for each
@@ -346,7 +330,7 @@ def cli(licenses_file, dump_to_file_on_errors=False):
 
         text = rule.text
 
-        existing_rule = rule_exists(text)
+        existing_rule = get_rule_id_for_text(text)
         skinny_text = " ".join(text[:80].split()).replace("{", " ").replace("}", " ")
 
         existing_msg = (

diff --git a/etc/scripts/licenses/report_license_rules.py b/etc/scripts/licenses/report_license_rules.py
@@ -62,6 +62,8 @@
     "is_license_reference",
     "is_license_intro",
     "is_license_clue",
+    "is_required_phrase",
+    "skip_collecting_required_phrases",
     "is_deprecated",
     "has_unknown",
     "only_known_words",

diff --git a/setup-mini.cfg b/setup-mini.cfg
@@ -158,6 +158,7 @@ console_scripts =
     scancode-reindex-licenses = licensedcode.reindex:reindex_licenses
     scancode-license-data = licensedcode.license_db:dump_scancode_license_data
     regen-package-docs = packagedcode.regen_package_docs:regen_package_docs
+    add-required-phrases = licensedcode.required_phrases:add_required_phrases
 
 # These are configurations for ScanCode plugins as setuptools entry points.
 # Each plugin entry hast this form:

diff --git a/setup.cfg b/setup.cfg
@@ -158,6 +158,7 @@ console_scripts =
     scancode-reindex-licenses = licensedcode.reindex:reindex_licenses
     scancode-license-data = licensedcode.license_db:dump_scancode_license_data
     regen-package-docs = packagedcode.regen_package_docs:regen_package_docs
+    add-required-phrases = licensedcode.required_phrases:add_required_phrases
 
 # These are configurations for ScanCode plugins as setuptools entry points.
 # Each plugin entry hast this form:

diff --git a/src/licensedcode/data/rules/apache-1.1_45.RULE b/src/licensedcode/data/rules/apache-1.1_45.RULE
@@ -7,7 +7,7 @@ ignorable_urls:
     - http://www.apache.org/
 ---
 
-APACHE 1.1
+{{APACHE 1.1}}
 
   Redistribution and use in source and binary forms, with or without
   modification, are permitted provided that the following conditions

diff --git a/src/licensedcode/data/rules/apache-1.1_82.RULE b/src/licensedcode/data/rules/apache-1.1_82.RULE
@@ -14,7 +14,7 @@ ignorable_emails:
     - [email protected]
 ---
 
-========================= Apache-1.1 =========================
+========================= {{Apache-1.1}} =========================
 
 
 The Apache Software License, Version 1.1

diff --git a/src/licensedcode/data/rules/apache-1.1_89.RULE b/src/licensedcode/data/rules/apache-1.1_89.RULE
@@ -14,7 +14,7 @@ ignorable_emails:
     - [email protected]
 ---
 
-Apache License 1.1
+{{Apache License 1.1}}
 
 Copyright (c) 2000 The Apache Software Foundation. All rights reserved.
 

diff --git a/src/licensedcode/data/rules/apache-1.1_92.RULE b/src/licensedcode/data/rules/apache-1.1_92.RULE
@@ -13,7 +13,7 @@ ignorable_emails:
     - [email protected]
 ---
 
-Apache License 1.1 Copyright (c) 2000 The Apache Software Foundation. All
+{{Apache License 1.1}} Copyright (c) 2000 The Apache Software Foundation. All
 rights reserved.
 
 Redistribution and use in source and binary forms, with or without modification,

diff --git a/src/licensedcode/data/rules/apache-2.0_1009.RULE b/src/licensedcode/data/rules/apache-2.0_1009.RULE
@@ -8,7 +8,7 @@ ignorable_urls:
     - https://www.apache.org/licenses/LICENSE-2.0
 ---
 
-License: Apache-2.0
+License: {{Apache-2.0}}
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

diff --git a/src/licensedcode/data/rules/apache-2.0_1010.RULE b/src/licensedcode/data/rules/apache-2.0_1010.RULE
@@ -7,7 +7,7 @@ ignorable_urls:
     - http://www.apache.org/licenses/LICENSE-2.0
 ---
 
-License: Apache-2.0
+License: {{Apache-2.0}}
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at
@@ -22,4 +22,4 @@ License: Apache-2.0
  .
  On Debian systems, the text of the Apache License, Version 2.0 can be
  found in the file
- `/usr/share/common-licenses/Apache-2.0'.
+ `/usr/share/common-licenses/{{Apache-2.0}}'.
diff --git a/src/licensedcode/data/rules/apache-2.0_1013.RULE b/src/licensedcode/data/rules/apache-2.0_1013.RULE
@@ -7,7 +7,7 @@ ignorable_urls:
     - https://www.apache.org/licenses/LICENSE-2.0
 ---
 
-License: Apache-2.0
+License: {{Apache-2.0}}
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at
@@ -22,4 +22,4 @@ License: Apache-2.0
  .
  On Debian systems, the text of the Apache License, Version 2.0 can be
  found in the file
- `/usr/share/common-licenses/Apache-2.0'.
+ `/usr/share/common-licenses/{{Apache-2.0}}'.
diff --git a/src/licensedcode/data/rules/apache-2.0_1027.RULE b/src/licensedcode/data/rules/apache-2.0_1027.RULE
@@ -6,4 +6,4 @@ ignorable_urls:
     - http://www.apache.org/licenses/LICENSE-2.0.html
 ---
 
-licensed under the http://www.apache.org/licenses/LICENSE-2.0.html[Apache 2.0] license.
+licensed under the http://www.apache.org/licenses/LICENSE-2.0.html[{{Apache 2.0}}] license.
diff --git a/src/licensedcode/data/rules/apache-2.0_1048.RULE b/src/licensedcode/data/rules/apache-2.0_1048.RULE
@@ -8,4 +8,4 @@ ignorable_urls:
     - http://www.apache.org/licenses
 ---
 
-'Apache 2.0 http://www.apache.org/licenses/': 'Apache-2.0',
+'{{Apache 2.0}} http://www.apache.org/licenses/': '{{Apache-2.0}}',
diff --git a/src/licensedcode/data/rules/apache-2.0_1049.RULE b/src/licensedcode/data/rules/apache-2.0_1049.RULE
@@ -8,4 +8,4 @@ ignorable_urls:
     - http://www.apache.org/licenses/
 ---
 
-'Apache 2.0 http://www.apache.org/licenses/
+'{{Apache 2.0}} http://www.apache.org/licenses/
diff --git a/src/licensedcode/data/rules/apache-2.0_1063.RULE b/src/licensedcode/data/rules/apache-2.0_1063.RULE
@@ -6,4 +6,4 @@ ignorable_urls:
     - http://www.apache.org/licenses/LICENSE-2.0
 ---
 
-Redistributed under the the Apache License 2.0 at http://www.apache.org/licenses/LICENSE-2.0
+Redistributed under the the {{Apache License 2.0}} at http://www.apache.org/licenses/LICENSE-2.0
diff --git a/src/licensedcode/data/rules/apache-2.0_1094.RULE b/src/licensedcode/data/rules/apache-2.0_1094.RULE
@@ -7,4 +7,4 @@ ignorable_urls:
     - http://www.apache.org/licenses/LICENSE-2.0
 ---
 
-<license uri="http://www.apache.org/licenses/LICENSE-2.0">Apache 2.0</license>
+<license uri="http://www.apache.org/licenses/LICENSE-2.0">{{Apache 2.0}}</license>
diff --git a/src/licensedcode/data/rules/apache-2.0_1095.RULE b/src/licensedcode/data/rules/apache-2.0_1095.RULE
@@ -4,4 +4,4 @@ is_license_notice: yes
 relevance: 100
 ---
 
-licensed under Apache License 2.0("{{ALv2}}"),
+licensed under {{Apache License 2.0}}("{{ALv2}}"),
diff --git a/src/licensedcode/data/rules/apache-2.0_1101.RULE b/src/licensedcode/data/rules/apache-2.0_1101.RULE
@@ -6,4 +6,4 @@ ignorable_urls:
     - https://spdx.org/licenses/Apache-2.0.html
 ---
 
-- [Apache-2.0](https://spdx.org/licenses/Apache-2.0.html).
+- [{{Apache-2.0}}](https://spdx.org/licenses/Apache-2.0.html).
diff --git a/src/licensedcode/data/rules/apache-2.0_1105.RULE b/src/licensedcode/data/rules/apache-2.0_1105.RULE
@@ -6,4 +6,4 @@ referenced_filenames:
     - LICENSE
 ---
 
-This project uses [the Apache 2.0 license](./LICENSE).
+This project uses [the {{Apache 2.0}} license](./LICENSE).
diff --git a/src/licensedcode/data/rules/apache-2.0_1107.RULE b/src/licensedcode/data/rules/apache-2.0_1107.RULE
@@ -6,4 +6,4 @@ ignorable_urls:
     - http://www.apache.org/licenses/LICENSE-2.0
 ---
 
-library and tools are licenced under Apache 2.0: http://www.apache.org/licenses/LICENSE-2.0
+library and tools are licenced under {{Apache 2.0}}: http://www.apache.org/licenses/LICENSE-2.0
diff --git a/src/licensedcode/data/rules/apache-2.0_1113.RULE b/src/licensedcode/data/rules/apache-2.0_1113.RULE
@@ -6,4 +6,4 @@ referenced_filenames:
     - LICENSE
 ---
 
-released under the Apache 2.0 license. See the [LICENSE](LICENSE) file for details.
+released under the {{Apache 2.0}} license. See the [LICENSE](LICENSE) file for details.
diff --git a/src/licensedcode/data/rules/apache-2.0_1139.RULE b/src/licensedcode/data/rules/apache-2.0_1139.RULE
@@ -7,4 +7,4 @@ referenced_filenames:
 ---
 
 # Li`cense
-  released under the Apache 2.0 license. See the [LICENSE](LICENSE) file for details.
+  released under {{the Apache 2}}.0 license. See the [LICENSE](LICENSE) file for details.
diff --git a/src/licensedcode/data/rules/apache-2.0_1141.RULE b/src/licensedcode/data/rules/apache-2.0_1141.RULE
@@ -6,4 +6,4 @@ referenced_filenames:
     - License.txt
 ---
 
-Licensed under the Apache-2.0 License. See License.txt in the project root for license information.
+Licensed under the {{Apache-2.0}} License. See License.txt in the project root for license information.
diff --git a/src/licensedcode/data/rules/apache-2.0_1147.RULE b/src/licensedcode/data/rules/apache-2.0_1147.RULE
@@ -5,5 +5,5 @@ ignorable_urls:
     - http://www.apache.org/licenses/LICENSE-2.0
 ---
 
-libraries are under the Apache License 2.0
+libraries are under the {{Apache License 2.0}}
 (see @url{http://www.apache.org/licenses/LICENSE-2.0} for details)
diff --git a/src/licensedcode/data/rules/apache-2.0_1153.RULE b/src/licensedcode/data/rules/apache-2.0_1153.RULE
@@ -8,8 +8,8 @@ ignorable_urls:
 <licenses>
     <license>
       <name>
-        All files contained in this JAR are licensed under the Apache
-         2.0 license, unless noted differently in their source (see
+        All files contained in this JAR are licensed under the {{Apache
+         2.0}} license, unless noted differently in their source (see
          swing2swt).
       </name>
       <url>

diff --git a/src/licensedcode/data/rules/apache-2.0_1154.RULE b/src/licensedcode/data/rules/apache-2.0_1154.RULE
@@ -8,8 +8,8 @@ ignorable_urls:
 <licenses>
     <license>
       <name>
-        All files contained in this JAR are licensed under the Apache
-         2.0 license, unless noted differently in their source (see
+        All files contained in this JAR are licensed under the {{Apache
+         2.0}} license, unless noted differently in their source (see
          swing2swt).
       </name>
       <url>

diff --git a/src/licensedcode/data/rules/apache-2.0_1158.RULE b/src/licensedcode/data/rules/apache-2.0_1158.RULE
@@ -6,4 +6,4 @@ ignorable_urls:
     - http://www.opensource.org/licenses/apache2.0.php
 ---
 
-http://www.opensource.org/licenses/apache2.0.php Apache License, 2.0
+http://www.opensource.org/licenses/apache2.0.php {{Apache License, 2.0}}
diff --git a/src/licensedcode/data/rules/apache-2.0_1175.RULE b/src/licensedcode/data/rules/apache-2.0_1175.RULE
@@ -6,4 +6,4 @@ ignorable_urls:
     - http://www.apache.org/licenses/LICENSE-2.0.txt
 ---
 
-Apache License 2.0   (http://www.apache.org/licenses/LICENSE-2.0.txt)
+{{Apache License 2.0}}   (http://www.apache.org/licenses/LICENSE-2.0.txt)
diff --git a/src/licensedcode/data/rules/apache-2.0_1179.RULE b/src/licensedcode/data/rules/apache-2.0_1179.RULE
@@ -6,4 +6,4 @@ ignorable_urls:
     - https://www.apache.org/licenses/LICENSE-2.0.txt
 ---
 
-Apache-2.0   (https://www.apache.org/licenses/LICENSE-2.0.txt)
+{{Apache-2.0}}   (https://www.apache.org/licenses/LICENSE-2.0.txt)
diff --git a/src/licensedcode/data/rules/apache-2.0_1190.RULE b/src/licensedcode/data/rules/apache-2.0_1190.RULE
@@ -7,5 +7,5 @@ ignorable_urls:
 ---
 
 <license>
-<name> Apache-2.0 </name>
+<name> {{Apache-2.0}} </name>
 <url> (https://www.apache.org/licenses/LICENSE-2.0.txt)</url>
diff --git a/src/licensedcode/data/rules/apache-2.0_1193.RULE b/src/licensedcode/data/rules/apache-2.0_1193.RULE
@@ -6,5 +6,5 @@ ignorable_urls:
 ---
 
 <license>
-<name> Apache License 2.0 </name>
+<name> {{Apache License 2.0}} </name>
 <url> (http://www.apache.org/licenses/LICENSE-2.0.txt)</url>
diff --git a/src/licensedcode/data/rules/apache-2.0_1228.RULE b/src/licensedcode/data/rules/apache-2.0_1228.RULE
@@ -7,7 +7,7 @@ ignorable_urls:
 
 <licenses>
     <license>
-      <name>Apache License 2.0</name>
+      <name>{{Apache License 2.0}}</name>
       <url>http://www.apache.org/licenses/LICENSE-2.0</url>
       <distribution>repo</distribution>
     </license>

diff --git a/src/licensedcode/data/rules/apache-2.0_1254.RULE b/src/licensedcode/data/rules/apache-2.0_1254.RULE
@@ -8,7 +8,7 @@ ignorable_urls:
 
 <licenses>
     <license>
-        <name>Apache License 2.0</name>
+        <name>{{Apache License 2.0}}</name>
         <url>http://www.apache.org/licenses/LICENSE-2.0.html</url>
         <distribution>repo</distribution>
     </license>

diff --git a/src/licensedcode/data/rules/apache-2.0_126.RULE b/src/licensedcode/data/rules/apache-2.0_126.RULE
@@ -6,7 +6,7 @@ ignorable_urls:
     - http://www.apache.org/licenses/LICENSE-2.0
 ---
 
-Apache 2.0 License
+{{Apache 2.0}} License
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

diff --git a/src/licensedcode/data/rules/apache-2.0_1271.RULE b/src/licensedcode/data/rules/apache-2.0_1271.RULE
@@ -8,5 +8,5 @@ ignorable_urls:
 
 licenses {
     license {
-        name 'Apache 2.0'
+        name '{{Apache 2.0}}'
         url 'https://www.apache.org/licenses/LICENSE-2.0.html'
diff --git a/src/licensedcode/data/rules/apache-2.0_1287.RULE b/src/licensedcode/data/rules/apache-2.0_1287.RULE
@@ -6,7 +6,7 @@ ignorable_urls:
     - http://www.apache.org/licenses/LICENSE-2.0
 ---
 
-*  Apache License 2.0:
+*  {{Apache License 2.0}}:
  *
  *  Licensed under the Apache License, Version 2.0 (the "License"); you may
  *  not use this file except in compliance with the License.

diff --git a/src/licensedcode/data/rules/apache-2.0_1298.RULE b/src/licensedcode/data/rules/apache-2.0_1298.RULE
@@ -7,7 +7,7 @@ ignorable_urls:
 
 Apache License Version 2.0, January 2004
 
-The following applies to all products licensed under the Apache 2.0
+The following applies to all products licensed under the {{Apache 2.0}}
 License: You may not use the identified files except in compliance
 with the Apache License, Version 2.0 (the "License.") You may obtain a
 copy of the License at http://www.apache.org/licenses/LICENSE-2.0. A

diff --git a/src/licensedcode/data/rules/apache-2.0_13.RULE b/src/licensedcode/data/rules/apache-2.0_13.RULE
@@ -6,4 +6,4 @@ ignorable_urls:
     - http://www.apache.org/licenses/LICENSE-2.0
 ---
 
-- Apache 2.0        : http://www.apache.org/licenses/LICENSE-2.0
+- {{Apache 2.0}}        : http://www.apache.org/licenses/LICENSE-2.0
diff --git a/src/licensedcode/data/rules/apache-2.0_1311.RULE b/src/licensedcode/data/rules/apache-2.0_1311.RULE
@@ -6,6 +6,6 @@ ignorable_urls:
 ---
 
 <license>
-      <name>Apache-2.0
+      <name>{{Apache-2.0}}
       <url>https://opensource.org/licenses/Apache-2.0
       <comments>Apache License, Version 2.0
diff --git a/src/licensedcode/data/rules/apache-2.0_1312.RULE b/src/licensedcode/data/rules/apache-2.0_1312.RULE
@@ -5,6 +5,6 @@ ignorable_urls:
     - https://opensource.org/licenses/Apache-2.0
 ---
 
-<name>Apache-2.0
+<name>{{Apache-2.0}}
       <url>https://opensource.org/licenses/Apache-2.0
       <comments>Apache License, Version 2.0