GitHub datagen

README.md

@@ -45,18 +45,18 @@ Here we use `v0.4.0` to reference the git tag for the `0.4.0` version, but the s
 ```bash
 git checkout v0.4.0
 mvn clean package
-confluent-hub install target/components/packages/confluentinc-kafka-connect-datagen-0.4.0.zip
+confluent-hub install ./target/components/packages/confluentinc-kafka-connect-datagen-0.4.0.zip
 ```
 
 ### Run connector in local install
 
 Here is an example of how to run the `kafka-connect-datagen` on a local Confluent Platform after it's been installed.  [Configuration](#configuration) details are provided below.
 
 ```bash
-confluent local start connect
-confluent local config datagen-pageviews -- -d config/connector_pageviews.config
-confluent local status connectors
-confluent local consume test1 --value-format avro --max-messages 5 --property print.key=true --property key.deserializer=org.apache.kafka.common.serialization.StringDeserializer --from-beginning
+confluent local services connect start
+confluent local services connect connector config datagen-pageviews -c config/connector_pageviews.config
+confluent local services connect status
+confluent local services kafka consume pageviews --max-messages 5 --property print.key=true --property key.deserializer=org.apache.kafka.common.serialization.StringDeserializer --from-beginning
 ```
 
 ## Install the connector from Confluent Hub into a [Kafka Connect](https://docs.confluent.io/current/connect/index.html) based Docker image

config/connector_custom.config

@@ -3,7 +3,7 @@
   "config": {
     "connector.class": "io.confluent.kafka.connect.datagen.DatagenConnector",
     "kafka.topic": "test1",
-    "schema.filename": "users_schema.avro",
+    "schema.filename": "users_schema.avsc",
     "schema.keyfield": "userid",
     "key.converter": "org.apache.kafka.connect.storage.StringConverter",
     "value.converter": "org.apache.kafka.connect.json.JsonConverter",

config/connector_github_assignees.config

@@ -0,0 +1,14 @@
+{
+  "name": "datagen-github-assignees",
+  "config": {
+    "connector.class": "io.confluent.kafka.connect.datagen.DatagenConnector",
+    "kafka.topic": "github_assignees",
+    "quickstart": "github_assignees",
+    "key.converter": "org.apache.kafka.connect.storage.StringConverter",
+    "value.converter": "org.apache.kafka.connect.json.JsonConverter",
+    "value.converter.schemas.enable": "false",
+    "max.interval": 100,
+    "iterations": 100,
+    "tasks.max": "1"
+  }
+}

config/connector_github_collaborators.config

@@ -0,0 +1,14 @@
+{
+  "name": "datagen-github-collaborators",
+  "config": {
+    "connector.class": "io.confluent.kafka.connect.datagen.DatagenConnector",
+    "kafka.topic": "github_collaborators",
+    "quickstart": "github_collaborators",
+    "key.converter": "org.apache.kafka.connect.storage.StringConverter",
+    "value.converter": "org.apache.kafka.connect.json.JsonConverter",
+    "value.converter.schemas.enable": "false",
+    "max.interval": 100,
+    "iterations": 100,
+    "tasks.max": "1"
+  }
+}

config/connector_github_comments.config

@@ -0,0 +1,14 @@
+{
+  "name": "datagen-github-comments",
+  "config": {
+    "connector.class": "io.confluent.kafka.connect.datagen.DatagenConnector",
+    "kafka.topic": "github_comments",
+    "quickstart": "github_comments",
+    "key.converter": "org.apache.kafka.connect.storage.StringConverter",
+    "value.converter": "org.apache.kafka.connect.json.JsonConverter",
+    "value.converter.schemas.enable": "false",
+    "max.interval": 100,
+    "iterations": 100,
+    "tasks.max": "1"
+  }
+}

config/connector_github_commits.config

@@ -0,0 +1,14 @@
+{
+  "name": "datagen-github-commits",
+  "config": {
+    "connector.class": "io.confluent.kafka.connect.datagen.DatagenConnector",
+    "kafka.topic": "github_commits",
+    "quickstart": "github_commits",
+    "key.converter": "org.apache.kafka.connect.storage.StringConverter",
+    "value.converter": "org.apache.kafka.connect.json.JsonConverter",
+    "value.converter.schemas.enable": "false",
+    "max.interval": 100,
+    "iterations": 100,
+    "tasks.max": "1"
+  }
+}

config/connector_github_issues.config

@@ -0,0 +1,14 @@
+{
+  "name": "datagen-github-issues",
+  "config": {
+    "connector.class": "io.confluent.kafka.connect.datagen.DatagenConnector",
+    "kafka.topic": "github_issues",
+    "quickstart": "github_issues",
+    "key.converter": "org.apache.kafka.connect.storage.StringConverter",
+    "value.converter": "org.apache.kafka.connect.json.JsonConverter",
+    "value.converter.schemas.enable": "false",
+    "max.interval": 100,
+    "iterations": 100,
+    "tasks.max": "1"
+  }
+}

config/connector_github_pull_requests.config

@@ -0,0 +1,14 @@
+{
+  "name": "datagen-github-pull-requests",
+  "config": {
+    "connector.class": "io.confluent.kafka.connect.datagen.DatagenConnector",
+    "kafka.topic": "github_pull_requests",
+    "quickstart": "github_pull_requests",
+    "key.converter": "org.apache.kafka.connect.storage.StringConverter",
+    "value.converter": "org.apache.kafka.connect.json.JsonConverter",
+    "value.converter.schemas.enable": "false",
+    "max.interval": 100,
+    "iterations": 100,
+    "tasks.max": "1"
+  }
+}

config/connector_github_releases.config

@@ -0,0 +1,14 @@
+{
+  "name": "datagen-github-releases",
+  "config": {
+    "connector.class": "io.confluent.kafka.connect.datagen.DatagenConnector",
+    "kafka.topic": "github_releases",
+    "quickstart": "github_releases",
+    "key.converter": "org.apache.kafka.connect.storage.StringConverter",
+    "value.converter": "org.apache.kafka.connect.json.JsonConverter",
+    "value.converter.schemas.enable": "false",
+    "max.interval": 100,
+    "iterations": 100,
+    "tasks.max": "1"
+  }
+}

config/connector_github_review_comments.config

@@ -0,0 +1,14 @@
+{
+  "name": "datagen-github-review-comments",
+  "config": {
+    "connector.class": "io.confluent.kafka.connect.datagen.DatagenConnector",
+    "kafka.topic": "github_review_comments",
+    "quickstart": "github_review_comments",
+    "key.converter": "org.apache.kafka.connect.storage.StringConverter",
+    "value.converter": "org.apache.kafka.connect.json.JsonConverter",
+    "value.converter.schemas.enable": "false",
+    "max.interval": 100,
+    "iterations": 100,
+    "tasks.max": "1"
+  }
+}

config/connector_github_reviews.config

@@ -0,0 +1,14 @@
+{
+  "name": "datagen-github-reviews",
+  "config": {
+    "connector.class": "io.confluent.kafka.connect.datagen.DatagenConnector",
+    "kafka.topic": "github_reviews",
+    "quickstart": "github_reviews",
+    "key.converter": "org.apache.kafka.connect.storage.StringConverter",
+    "value.converter": "org.apache.kafka.connect.json.JsonConverter",
+    "value.converter.schemas.enable": "false",
+    "max.interval": 100,
+    "iterations": 100,
+    "tasks.max": "1"
+  }
+}

config/connector_github_stargazers.config

@@ -0,0 +1,14 @@
+{
+  "name": "datagen-github-stargazers",
+  "config": {
+    "connector.class": "io.confluent.kafka.connect.datagen.DatagenConnector",
+    "kafka.topic": "github_stargazers",
+    "quickstart": "github_stargazers",
+    "key.converter": "org.apache.kafka.connect.storage.StringConverter",
+    "value.converter": "org.apache.kafka.connect.json.JsonConverter",
+    "value.converter.schemas.enable": "false",
+    "max.interval": 100,
+    "iterations": 100,
+    "tasks.max": "1"
+  }
+}

src/main/java/io/confluent/kafka/connect/datagen/DatagenTask.java

@@ -68,21 +68,31 @@ public class DatagenTask extends SourceTask {
   private Random random;
 
   protected enum Quickstart {
-    CLICKSTREAM_CODES("clickstream_codes_schema.avro", "code"),
-    CLICKSTREAM("clickstream_schema.avro", "ip"),
-    CLICKSTREAM_USERS("clickstream_users_schema.avro", "user_id"),
-    ORDERS("orders_schema.avro", "orderid"),
-    RATINGS("ratings_schema.avro", "rating_id"),
-    USERS("users_schema.avro", "userid"),
-    USERS_("users_array_map_schema.avro", "userid"),
-    PAGEVIEWS("pageviews_schema.avro", "viewtime"),
-    STOCK_TRADES("stock_trades_schema.avro", "symbol"),
-    INVENTORY("inventory.avro", "id"),
-    PRODUCT("product.avro", "id"),
-    PURCHASES("purchase.avro", "id"),
-    TRANSACTIONS("transactions.avro", "transaction_id"),
-    STORES("stores.avro", "store_id"),
-    CREDIT_CARDS("credit_cards.avro", "card_id");
+    CLICKSTREAM_CODES("clickstream_codes_schema.avsc", "code"),
+    CLICKSTREAM("clickstream_schema.avsc", "ip"),
+    CLICKSTREAM_USERS("clickstream_users_schema.avsc", "user_id"),
+    ORDERS("orders_schema.avsc", "orderid"),
+    RATINGS("ratings_schema.avsc", "rating_id"),
+    USERS("users_schema.avsc", "userid"),
+    USERS_("users_array_map_schema.avsc", "userid"),
+    PAGEVIEWS("pageviews_schema.avsc", "viewtime"),
+    STOCK_TRADES("stock_trades_schema.avsc", "symbol"),
+    INVENTORY("inventory.avsc", "id"),
+    PRODUCT("product.avsc", "id"),
+    PURCHASES("purchase.avsc", "id"),
+    TRANSACTIONS("transactions.avsc", "transaction_id"),
+    STORES("stores.avsc", "store_id"),
+    CREDIT_CARDS("credit_cards.avsc", "card_id"),
+    GITHUB_ASSIGNEES("github/assignees.avsc", "id"),
+    GITHUB_COLLABORATORS("github/collaborators.avsc", "id"),
+    GITHUB_COMMENTS("github/comments.avsc", "id"),
+    GITHUB_COMMITS("github/commits.avsc", "id"),
+    GITHUB_ISSUES("github/issues.avsc", "id"),
+    GITHUB_PULL_REQUESTS("github/pull_requests.avsc", "id"),
+    GITHUB_RELEASES("github/releases.avsc", "id"),
+    GITHUB_REVIEW_COMMENTS("github/review_comments.avsc", "id"),
+    GITHUB_REVIEWS("github/reviews.avsc", "id"),
+    GITHUB_STARGAZERS("github/stargazers.avsc", "id");
 
     static final Set<String> configValues = new HashSet<>();