fix indentation and docmd

GenBench · Aug 2, 2023 · 7958b66 · 7958b66
1 parent aedd50a
commit 7958b66
Show file tree

Hide file tree

Showing 20 changed files with 174 additions and 304 deletions.
diff --git a/src/genbench/tasks/nl_codesearch_clf/codesearchnet_adv/config.jsonnet b/src/genbench/tasks/nl_codesearch_clf/codesearchnet_adv/config.jsonnet
@@ -1,17 +1,15 @@
 {
     name: 'Natural Language Codesearch Classification (codesearchnet_adv)',
 
-    // @TODO: Add a description of the task
     description: 'Natural Language Codesearch Classification (codesearchnet_adv) aims to measure the generalization capabilites of language models in code understanding. This subtasks measures robustness in covariate shift',
 
-    // @TODO: Add a list of keywords that describe the task
     keywords: [
         'codesearch',
         'natural language query',
-		'binary classification',
-		'python',
-		'robustness',
-		'covariate shift',
+        'binary classification',
+        'python',
+        'robustness',
+        'covariate shift',
     ],
 
     authors: [
@@ -24,7 +22,7 @@
     data_source: {
         type: 'manual',
         test: 'https://raw.githubusercontent.com/drndr/genbench_ds/master/sample_data/clf/codesearchnet_adv/test_sample_cbt.jsonl',
-		train:'https://raw.githubusercontent.com/drndr/genbench_ds/master/sample_data/clf/codesearchnet_adv/train_sample_cbt.jsonl',
+        train:'https://raw.githubusercontent.com/drndr/genbench_ds/master/sample_data/clf/codesearchnet_adv/train_sample_cbt.jsonl',
     },
 
     has_validation_set: false,
@@ -35,26 +33,20 @@
     evaluation_metrics: [
         {
             hf_id: 'accuracy',
-			git_commit_sha: '34d6add55811828baef83e0d7c6826e2193f7b6a',
-			best_score: 1.0,
-		},
+            git_commit_sha: '34d6add55811828baef83e0d7c6826e2193f7b6a',
+            best_score: 1.0,
+        },
     ],
 
     preparation_strategies: {
-        // A recipe for preparing the model to perform the task by configuring its prompt.
-        // This recipe is suitable for generative LMs such as GPT-3, OPT, T5, etc.
-        // We provide a few options for configuring the prompt. But, the task creator can
-        // also provide a custom prompt preparation in the task's Python class.
-		finetuning: {
+        finetuning: {
             objective: 'maximum_likelihood',
         },
 
         prompt_based_testing: {
             prompt_builder: {
-                // Currently, we follow BIG-bench options for prompt construction: 
-                // https://github.com/google/BIG-bench/blob/main/docs/doc.md#optional-fields
-                instruction_zero_shot: 'Given a code comment and an R programming language code snippet, determine if the comment accurately represents the function of the code. Respond with True if the code matches the comment and False if it does not. The input format is defined as: comment [SEP] code',
-				input_prefix: '',
+                instruction_zero_shot: 'Given a code comment and a Python programming language code snippet, determine if the comment accurately represents the function of the code. Respond with True if the code matches the comment and False if it does not. The input format is defined as: comment [SEP] code',
+                input_prefix: '',
                 output_prefix: '',
                 choices_prefix: '',
                 append_choices_to_input: false,

diff --git a/src/genbench/tasks/nl_codesearch_clf/codesearchnet_go/config.jsonnet b/src/genbench/tasks/nl_codesearch_clf/codesearchnet_go/config.jsonnet
@@ -1,16 +1,14 @@
 {
     name: 'Natural Language Codesearch Classification (codesearchnet_go)',
-
-    // @TODO: Add a description of the task
+
     description: 'Natural Language Codesearch Classification (codesearchnet_go) aims to measure the generalization capabilites of language models in code understanding. This subtasks measures cross-lingual generalization',
 
-    // @TODO: Add a list of keywords that describe the task
     keywords: [
         'codesearch',
         'natural language query',
-		'binary classification',
-		'go',
-		'cross-lingual'
+        'binary classification',
+        'go',
+        'cross-lingual'
     ],
 
     authors: [
@@ -23,7 +21,7 @@
     data_source: {
         type: 'manual',
         test: 'https://raw.githubusercontent.com/drndr/genbench_ds/master/sample_data/clf/codesearchnet_go/test_sample_cbt.jsonl',
-		train:'https://raw.githubusercontent.com/drndr/genbench_ds/master/sample_data/clf/codesearchnet_adv/train_sample_cbt.jsonl',
+        train:'https://raw.githubusercontent.com/drndr/genbench_ds/master/sample_data/clf/codesearchnet_adv/train_sample_cbt.jsonl',
     },
 
     has_validation_set: false,
@@ -34,24 +32,19 @@
     evaluation_metrics: [
         {
             hf_id: 'accuracy',
-			git_commit_sha: '34d6add55811828baef83e0d7c6826e2193f7b6a',
-			best_score: 1.0,
+            git_commit_sha: '34d6add55811828baef83e0d7c6826e2193f7b6a',
+            best_score: 1.0,
 	    },
     ],
 
     preparation_strategies: {
-
-	    finetuning: {
+        finetuning: {
             objective: 'maximum_likelihood',
         },
-        // A recipe for preparing the model to perform the task by configuring its prompt.
-        // This recipe is suitable for generative LMs such as GPT-3, OPT, T5, etc.
-        // We provide a few options for configuring the prompt. But, the task creator can
-        // also provide a custom prompt preparation in the task's Python class.
         prompt_based_testing: {
             prompt_builder: {
-                instruction_zero_shot: 'Given a code comment and an R programming language code snippet, determine if the comment accurately represents the function of the code. Respond with True if the code matches the comment and False if it does not. The input format is defined as: comment [SEP] code',
-				input_prefix: '',
+                instruction_zero_shot: 'Given a code comment and a Go programming language code snippet, determine if the comment accurately represents the function of the code. Respond with True if the code matches the comment and False if it does not. The input format is defined as: comment [SEP] code',
+                input_prefix: '',
                 output_prefix: '',
                 choices_prefix: '',
                 append_choices_to_input: false,

diff --git a/src/genbench/tasks/nl_codesearch_clf/codesearchnet_java/config.jsonnet b/src/genbench/tasks/nl_codesearch_clf/codesearchnet_java/config.jsonnet
@@ -1,16 +1,14 @@
 {
     name: 'Natural Language Codesearch Classification (codesearchnet_java)',
-
-    // @TODO: Add a description of the task
+
     description: 'Natural Language Codesearch Classification (codesearchnet_java) aims to measure the generalization capabilites of language models in code understanding. This subtasks measures cross-lingual generalization',
-
-    // @TODO: Add a list of keywords that describe the task
+
     keywords: [
         'codesearch',
         'natural language query',
-		'binary classification',
-		'java',
-		'cross-lingual'
+        'binary classification',
+        'java',
+        'cross-lingual'
     ],
 
     authors: [
@@ -23,7 +21,7 @@
     data_source: {
         type: 'manual',
         test: 'https://raw.githubusercontent.com/drndr/genbench_ds/master/sample_data/clf/codesearchnet_java/test_sample_cbt.jsonl',
-		train:'https://raw.githubusercontent.com/drndr/genbench_ds/master/sample_data/clf/codesearchnet_adv/train_sample_cbt.jsonl',
+        train:'https://raw.githubusercontent.com/drndr/genbench_ds/master/sample_data/clf/codesearchnet_adv/train_sample_cbt.jsonl',
     },
 
     has_validation_set: false,
@@ -34,24 +32,19 @@
     evaluation_metrics: [
         {
             hf_id: 'accuracy',
-			git_commit_sha: '34d6add55811828baef83e0d7c6826e2193f7b6a',
-			best_score: 1.0,
-		},
+            git_commit_sha: '34d6add55811828baef83e0d7c6826e2193f7b6a',
+            best_score: 1.0,
+        },
     ],
 
     preparation_strategies: {
-
-	    finetuning: {
+        finetuning: {
             objective: 'maximum_likelihood',
         },
-        // A recipe for preparing the model to perform the task by configuring its prompt.
-        // This recipe is suitable for generative LMs such as GPT-3, OPT, T5, etc.
-        // We provide a few options for configuring the prompt. But, the task creator can
-        // also provide a custom prompt preparation in the task's Python class.
         prompt_based_testing: {
             prompt_builder: {
-                instruction_zero_shot: 'Given a code comment and an R programming language code snippet, determine if the comment accurately represents the function of the code. Respond with True if the code matches the comment and False if it does not. The input format is defined as: comment [SEP] code',
-				input_prefix: '',
+                instruction_zero_shot: 'Given a code comment and a Java programming language code snippet, determine if the comment accurately represents the function of the code. Respond with True if the code matches the comment and False if it does not. The input format is defined as: comment [SEP] code',
+                input_prefix: '',
                 output_prefix: '',
                 choices_prefix: '',
                 append_choices_to_input: false,

diff --git a/src/genbench/tasks/nl_codesearch_clf/codesearchnet_javascript/config.jsonnet b/src/genbench/tasks/nl_codesearch_clf/codesearchnet_javascript/config.jsonnet
@@ -1,16 +1,14 @@
 {
     name: 'Natural Language Codesearch Classification (codesearchnet_javascript)',
 
-    // @TODO: Add a description of the task
     description: 'Natural Language Codesearch Classification (codesearchnet_javascript) aims to measure the generalization capabilites of language models in code understanding. This subtasks measures cross-lingual generalization',
 
-    // @TODO: Add a list of keywords that describe the task
     keywords: [
         'codesearch',
         'natural language query',
-		'binary classification',
-		'javascript',
-		'cross-lingual'
+        'binary classification',
+        'javascript',
+        'cross-lingual'
     ],
 
     authors: [
@@ -23,7 +21,7 @@
     data_source: {
         type: 'manual',
         test: 'https://raw.githubusercontent.com/drndr/genbench_ds/master/sample_data/clf/codesearchnet_javascript/test_sample_cbt.jsonl',
-		train:'https://raw.githubusercontent.com/drndr/genbench_ds/master/sample_data/clf/codesearchnet_adv/train_sample_cbt.jsonl',
+        train:'https://raw.githubusercontent.com/drndr/genbench_ds/master/sample_data/clf/codesearchnet_adv/train_sample_cbt.jsonl',
     },
 
     has_validation_set: false,
@@ -36,22 +34,17 @@
             hf_id: 'accuracy',
 			git_commit_sha: '34d6add55811828baef83e0d7c6826e2193f7b6a',
 			best_score: 1.0,
-		},
+        },
     ],
 
     preparation_strategies: {
-
-	    finetuning: {
+        finetuning: {
             objective: 'maximum_likelihood',
         },
-        // A recipe for preparing the model to perform the task by configuring its prompt.
-        // This recipe is suitable for generative LMs such as GPT-3, OPT, T5, etc.
-        // We provide a few options for configuring the prompt. But, the task creator can
-        // also provide a custom prompt preparation in the task's Python class.
         prompt_based_testing: {
             prompt_builder: {
-                instruction_zero_shot: 'Given a code comment and an R programming language code snippet, determine if the comment accurately represents the function of the code. Respond with True if the code matches the comment and False if it does not. The input format is defined as: comment [SEP] code',
-				input_prefix: '',
+                instruction_zero_shot: 'Given a code comment and an Javascript programming language code snippet, determine if the comment accurately represents the function of the code. Respond with True if the code matches the comment and False if it does not. The input format is defined as: comment [SEP] code',
+                input_prefix: '',
                 output_prefix: '',
                 choices_prefix: '',
                 append_choices_to_input: false,

diff --git a/src/genbench/tasks/nl_codesearch_clf/codesearchnet_php/config.jsonnet b/src/genbench/tasks/nl_codesearch_clf/codesearchnet_php/config.jsonnet
@@ -1,16 +1,14 @@
 {
     name: 'Natural Language Codesearch Classification (codesearchnet_php)',
-
-    // @TODO: Add a description of the task
+
     description: 'Natural Language Codesearch Classification (codesearchnet_php) aims to measure the generalization capabilites of language models in code understanding. This subtasks measures cross-lingual generalization',
 
-    // @TODO: Add a list of keywords that describe the task
     keywords: [
         'codesearch',
         'natural language query',
-		'binary classification',
-		'php',
-		'cross-lingual'
+        'binary classification',
+        'php',
+        'cross-lingual'
     ],
 
     authors: [
@@ -23,7 +21,7 @@
     data_source: {
         type: 'manual',
         test: 'https://raw.githubusercontent.com/drndr/genbench_ds/master/sample_data/clf/codesearchnet_php/test_sample_cbt.jsonl',
-		train:'https://raw.githubusercontent.com/drndr/genbench_ds/master/sample_data/clf/codesearchnet_adv/train_sample_cbt.jsonl',
+        train:'https://raw.githubusercontent.com/drndr/genbench_ds/master/sample_data/clf/codesearchnet_adv/train_sample_cbt.jsonl',
     },
 
     has_validation_set: false,
@@ -34,24 +32,19 @@
     evaluation_metrics: [
         {
             hf_id: 'accuracy',
-			git_commit_sha: '34d6add55811828baef83e0d7c6826e2193f7b6a',
-			best_score: 1.0,
-		},
+            git_commit_sha: '34d6add55811828baef83e0d7c6826e2193f7b6a',
+            best_score: 1.0,
+        },
     ],
 
     preparation_strategies: {
-
-	    finetuning: {
+        finetuning: {
             objective: 'maximum_likelihood',
         },
-        // A recipe for preparing the model to perform the task by configuring its prompt.
-        // This recipe is suitable for generative LMs such as GPT-3, OPT, T5, etc.
-        // We provide a few options for configuring the prompt. But, the task creator can
-        // also provide a custom prompt preparation in the task's Python class.
         prompt_based_testing: {
             prompt_builder: {
-                instruction_zero_shot: 'Given a code comment and an R programming language code snippet, determine if the comment accurately represents the function of the code. Respond with True if the code matches the comment and False if it does not. The input format is defined as: comment [SEP] code',
-				input_prefix: '',
+                instruction_zero_shot: 'Given a code comment and a PHP programming language code snippet, determine if the comment accurately represents the function of the code. Respond with True if the code matches the comment and False if it does not. The input format is defined as: comment [SEP] code',
+                input_prefix: '',
                 output_prefix: '',
                 choices_prefix: '',
                 append_choices_to_input: false,

diff --git a/src/genbench/tasks/nl_codesearch_clf/codesearchnet_ruby/config.jsonnet b/src/genbench/tasks/nl_codesearch_clf/codesearchnet_ruby/config.jsonnet
@@ -1,16 +1,14 @@
 {
     name: 'Natural Language Codesearch Classification (codesearchnet_ruby)',
 
-    // @TODO: Add a description of the task
     description: 'Natural Language Codesearch Classification (codesearchnet_ruby) aims to measure the generalization capabilites of language models in code understanding. This subtasks measures cross-lingual generalization',
 
-    // @TODO: Add a list of keywords that describe the task
     keywords: [
         'codesearch',
         'natural language query',
-		'binary classification',
-		'ruby',
-		'cross-lingual'
+        'binary classification',
+        'ruby',
+        'cross-lingual'
     ],
 
     authors: [
@@ -23,7 +21,7 @@
     data_source: {
         type: 'manual',
         test: 'https://raw.githubusercontent.com/drndr/genbench_ds/master/sample_data/clf/codesearchnet_ruby/test_sample_cbt.jsonl',
-		train:'https://raw.githubusercontent.com/drndr/genbench_ds/master/sample_data/clf/codesearchnet_adv/train_sample_cbt.jsonl',
+        train:'https://raw.githubusercontent.com/drndr/genbench_ds/master/sample_data/clf/codesearchnet_adv/train_sample_cbt.jsonl',
     },
 
     has_validation_set: false,
@@ -34,24 +32,19 @@
     evaluation_metrics: [
         {
             hf_id: 'accuracy',
-			git_commit_sha: '34d6add55811828baef83e0d7c6826e2193f7b6a',
-			best_score: 1.0,
-		},
+            git_commit_sha: '34d6add55811828baef83e0d7c6826e2193f7b6a',
+            best_score: 1.0,
+        },
     ],
 
     preparation_strategies: {
-
-	    finetuning: {
+        finetuning: {
             objective: 'maximum_likelihood',
         },
-        // A recipe for preparing the model to perform the task by configuring its prompt.
-        // This recipe is suitable for generative LMs such as GPT-3, OPT, T5, etc.
-        // We provide a few options for configuring the prompt. But, the task creator can
-        // also provide a custom prompt preparation in the task's Python class.
         prompt_based_testing: {
             prompt_builder: {
-                instruction_zero_shot: 'Given a code comment and an R programming language code snippet, determine if the comment accurately represents the function of the code. Respond with True if the code matches the comment and False if it does not. The input format is defined as: comment [SEP] code',
-				input_prefix: '',
+                instruction_zero_shot: 'Given a code comment and a Ruby programming language code snippet, determine if the comment accurately represents the function of the code. Respond with True if the code matches the comment and False if it does not. The input format is defined as: comment [SEP] code',
+                input_prefix: '',
                 output_prefix: '',
                 choices_prefix: '',
                 append_choices_to_input: false,