GitHub datagen

confluentinc · Jul 11, 2022 · a15e630 · a15e630
1 parent a4e775c
commit a15e630
Show file tree

Hide file tree

Showing 12 changed files with 5,166 additions and 15 deletions.
diff --git a/config/connector_github_assignees.config b/config/connector_github_assignees.config
@@ -0,0 +1,14 @@
+{
+  "name": "datagen-github-assignees",
+  "config": {
+    "connector.class": "io.confluent.kafka.connect.datagen.DatagenConnector",
+    "kafka.topic": "github_assignees",
+    "quickstart": "github_assignees",
+    "key.converter": "org.apache.kafka.connect.storage.StringConverter",
+    "value.converter": "org.apache.kafka.connect.json.JsonConverter",
+    "value.converter.schemas.enable": "false",
+    "max.interval": 100,
+    "iterations": 100,
+    "tasks.max": "1"
+  }
+}
diff --git a/config/connector_github_collaborators.config b/config/connector_github_collaborators.config
@@ -0,0 +1,14 @@
+{
+  "name": "datagen-github-collaborators",
+  "config": {
+    "connector.class": "io.confluent.kafka.connect.datagen.DatagenConnector",
+    "kafka.topic": "github_collaborators",
+    "quickstart": "github_collaborators",
+    "key.converter": "org.apache.kafka.connect.storage.StringConverter",
+    "value.converter": "org.apache.kafka.connect.json.JsonConverter",
+    "value.converter.schemas.enable": "false",
+    "max.interval": 100,
+    "iterations": 100,
+    "tasks.max": "1"
+  }
+}
diff --git a/config/connector_github_commits.config b/config/connector_github_commits.config
@@ -0,0 +1,14 @@
+{
+  "name": "datagen-github-commits",
+  "config": {
+    "connector.class": "io.confluent.kafka.connect.datagen.DatagenConnector",
+    "kafka.topic": "github_commits",
+    "quickstart": "github_commits",
+    "key.converter": "org.apache.kafka.connect.storage.StringConverter",
+    "value.converter": "org.apache.kafka.connect.json.JsonConverter",
+    "value.converter.schemas.enable": "false",
+    "max.interval": 100,
+    "iterations": 100,
+    "tasks.max": "1"
+  }
+}
diff --git a/config/connector_github_pull_requests.config b/config/connector_github_pull_requests.config
@@ -0,0 +1,14 @@
+{
+  "name": "datagen-github-pull-requests",
+  "config": {
+    "connector.class": "io.confluent.kafka.connect.datagen.DatagenConnector",
+    "kafka.topic": "github_pull_requests",
+    "quickstart": "github_pull_requests",
+    "key.converter": "org.apache.kafka.connect.storage.StringConverter",
+    "value.converter": "org.apache.kafka.connect.json.JsonConverter",
+    "value.converter.schemas.enable": "false",
+    "max.interval": 100,
+    "iterations": 100,
+    "tasks.max": "1"
+  }
+}
diff --git a/config/connector_github_stargazers.config b/config/connector_github_stargazers.config
@@ -0,0 +1,14 @@
+{
+  "name": "datagen-github-stargazers",
+  "config": {
+    "connector.class": "io.confluent.kafka.connect.datagen.DatagenConnector",
+    "kafka.topic": "github_stargazers",
+    "quickstart": "github_stargazers",
+    "key.converter": "org.apache.kafka.connect.storage.StringConverter",
+    "value.converter": "org.apache.kafka.connect.json.JsonConverter",
+    "value.converter.schemas.enable": "false",
+    "max.interval": 100,
+    "iterations": 100,
+    "tasks.max": "1"
+  }
+}
diff --git a/src/main/java/io/confluent/kafka/connect/datagen/DatagenTask.java b/src/main/java/io/confluent/kafka/connect/datagen/DatagenTask.java
@@ -68,21 +68,26 @@ public class DatagenTask extends SourceTask {
   private Random random;
 
   protected enum Quickstart {
-    CLICKSTREAM_CODES("clickstream_codes_schema.avro", "code"),
-    CLICKSTREAM("clickstream_schema.avro", "ip"),
-    CLICKSTREAM_USERS("clickstream_users_schema.avro", "user_id"),
-    ORDERS("orders_schema.avro", "orderid"),
-    RATINGS("ratings_schema.avro", "rating_id"),
-    USERS("users_schema.avro", "userid"),
-    USERS_("users_array_map_schema.avro", "userid"),
-    PAGEVIEWS("pageviews_schema.avro", "viewtime"),
-    STOCK_TRADES("stock_trades_schema.avro", "symbol"),
-    INVENTORY("inventory.avro", "id"),
-    PRODUCT("product.avro", "id"),
-    PURCHASES("purchase.avro", "id"),
-    TRANSACTIONS("transactions.avro", "transaction_id"),
-    STORES("stores.avro", "store_id"),
-    CREDIT_CARDS("credit_cards.avro", "card_id");
+    CLICKSTREAM_CODES("clickstream_codes_schema.avsc", "code"),
+    CLICKSTREAM("clickstream_schema.avsc", "ip"),
+    CLICKSTREAM_USERS("clickstream_users_schema.avsc", "user_id"),
+    ORDERS("orders_schema.avsc", "orderid"),
+    RATINGS("ratings_schema.avsc", "rating_id"),
+    USERS("users_schema.avsc", "userid"),
+    USERS_("users_array_map_schema.avsc", "userid"),
+    PAGEVIEWS("pageviews_schema.avsc", "viewtime"),
+    STOCK_TRADES("stock_trades_schema.avsc", "symbol"),
+    INVENTORY("inventory.avsc", "id"),
+    PRODUCT("product.avsc", "id"),
+    PURCHASES("purchase.avsc", "id"),
+    TRANSACTIONS("transactions.avsc", "transaction_id"),
+    STORES("stores.avsc", "store_id"),
+    CREDIT_CARDS("credit_cards.avsc", "card_id"),
+    GITHUB_ASSIGNEES("github/assignees.avsc", "id"),
+    GITHUB_COLLABORATORS("github/collaborators.avsc", "id"),
+    GITHUB_COMMITS("github/commits.avsc", "id"),
+    GITHUB_PULL_REQUESTS("github/pull_requests.avsc", "id"),
+    GITHUB_STARGAZERS("github/stargazers.avsc", "id");
 
     static final Set<String> configValues = new HashSet<>();
 

diff --git a/src/main/resources/github/assignees.avsc b/src/main/resources/github/assignees.avsc
@@ -0,0 +1,215 @@
+{
+  "namespace": "datagen.github",
+  "name": "github_assignees",
+  "type": "record",
+  "fields": [
+    {
+      "name": "type",
+      "type": {
+        "type": "string",
+        "arg.properties": {
+          "options": [
+            "ASSIGNEES"
+          ]
+        }
+      }
+    },
+    {
+      "name": "createdAt",
+      "type": {
+        "type": "long",
+        "arg.properties": {
+          "iteration": {
+            "start": 1657053439000,
+            "step": 30000
+          }
+        }
+      }
+    },
+    {
+      "name": "data",
+      "type": {
+        "type": "record",
+        "name": "data_record",
+        "fields": [
+          {
+            "name": "login",
+            "type":{
+              "type":"string",
+              "arg.properties":{
+                "regex":"user[1-9]{2}"
+              }
+            }
+          },
+          {
+            "name": "id",
+            "type": {
+              "type": "long",
+              "arg.properties": {
+                "iteration": {
+                  "start": 829002,
+                  "step": 20
+                }
+              }
+            }
+          },
+          {
+            "name": "node_id",
+            "type":{
+              "type":"string",
+              "arg.properties":{
+                "regex":"[a-zA-Z0-9]{40}="
+              }
+            }
+          },
+          {
+            "name": "avatar_url",
+            "type":{
+              "type":"string",
+              "arg.properties":{
+                "regex":"https://avatars\\.githubusercontent\\.com/u/[a-z]{8}"
+              }
+            }
+          },
+          {
+            "name": "gravatar_id",
+            "type":{
+              "type":"string",
+              "arg.properties":{
+                "regex":"[a-z]{8}"
+              }
+            }
+          },
+          {
+            "name": "url",
+            "type":{
+              "type":"string",
+              "arg.properties":{
+                "regex":"https://api\\.github\\.com/users/org[1-9]{2}"
+              }
+            }
+          },
+          {
+            "name": "html_url",
+            "type":{
+              "type":"string",
+              "arg.properties":{
+                "regex":"https://github\\.com/org[1-9]{2}"
+              }
+            }
+          },
+          {
+            "name": "followers_url",
+            "type":{
+              "type":"string",
+              "arg.properties":{
+                "regex":"https://api\\.github\\.com/users/org[1-9]{2}/followers"
+              }
+            }
+          },
+          {
+            "name": "following_url",
+            "type":{
+              "type":"string",
+              "arg.properties":{
+                "regex":"https://api\\.github\\.com/users/org[1-9]{2}/following\\{/other_user\\}"
+              }
+            }
+          },
+          {
+            "name": "gists_url",
+            "type":{
+              "type":"string",
+              "arg.properties":{
+                "regex":"https://api\\.github\\.com/users/org[1-9]{2}/gists\\{/gist_id\\}"
+              }
+            }
+          },
+          {
+            "name": "starred_url",
+            "type":{
+              "type":"string",
+              "arg.properties":{
+                "regex":"https://api\\.github\\.com/users/org[1-9]{2}/starred\\{/owner\\}\\{/repo\\}"
+              }
+            }
+          },
+          {
+            "name": "subscriptions_url",
+            "type":{
+              "type":"string",
+              "arg.properties":{
+                "regex":"https://api\\.github\\.com/users/org[1-9]{2}/subscriptions"
+              }
+            }
+          },
+          {
+            "name": "organizations_url",
+            "type":{
+              "type":"string",
+              "arg.properties":{
+                "regex":"https://api\\.github\\.com/users/org[1-9]{2}/orgs"
+              }
+            }
+          },
+          {
+            "name": "repos_url",
+            "type":{
+              "type":"string",
+              "arg.properties":{
+                "regex":"https://api\\.github\\.com/users/org[1-9]{2}/repos"
+              }
+            }
+          },
+          {
+            "name": "events_url",
+            "type":{
+              "type":"string",
+              "arg.properties":{
+                "regex":"https://api\\.github\\.com/users/org[1-9]{2}/events\\{/privacy\\}"
+              }
+            }
+          },
+          {
+            "name": "received_events_url",
+            "type":{
+              "type":"string",
+              "arg.properties":{
+                "regex":"https://api\\.github\\.com/users/org[1-9]{2}/received_events"
+              }
+            }
+          },
+          {
+            "name": "type",
+            "type":{
+              "type":"string",
+              "arg.properties":{
+                "regex":"User"
+              }
+            }
+          },
+          {
+            "name": "site_admin",
+            "type":{
+              "type":"boolean",
+              "arg.properties": {
+                "options": [
+                  false
+                ]
+              }
+            }
+          }
+        ]
+      }
+    },
+    {
+      "name": "id",
+      "type":{
+        "type":"string",
+        "arg.properties":{
+          "regex":"[0-9]{20}"
+        }
+      }
+    }
+  ]
+}