From 4a8117e0c5c46bfd8726b85c12871e1a6235ae34 Mon Sep 17 00:00:00 2001 From: yaojie Date: Wed, 10 Apr 2019 15:59:05 -0700 Subject: [PATCH] fix build and add checkstyle --- checkstyle.xml | 387 ++++++++++++++++++ docs/Elasticsearch-realtimesink.md | 48 --- pom.xml | 116 +++++- .../hydrator/plugin/batch/ESProperties.java | 16 + .../plugin/batch/RecordWritableConverter.java | 16 + .../batch/sink/BatchElasticsearchSink.java | 18 +- .../batch/source/ElasticsearchSource.java | 20 +- .../batch/RecordWritableConverterTest.java | 4 +- suppressions.xml | 30 ++ widgets/Elasticsearch-realtimesink.json | 49 --- 10 files changed, 600 insertions(+), 104 deletions(-) create mode 100644 checkstyle.xml delete mode 100644 docs/Elasticsearch-realtimesink.md create mode 100644 suppressions.xml delete mode 100644 widgets/Elasticsearch-realtimesink.json diff --git a/checkstyle.xml b/checkstyle.xml new file mode 100644 index 0000000..410b9ca --- /dev/null +++ b/checkstyle.xml @@ -0,0 +1,387 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/Elasticsearch-realtimesink.md b/docs/Elasticsearch-realtimesink.md deleted file mode 100644 index 2bd102c..0000000 --- a/docs/Elasticsearch-realtimesink.md +++ /dev/null @@ -1,48 +0,0 @@ -# Elasticsearch Real-time Sink - - -Description ------------ -Takes the Structured Record from the input source and converts it to a JSON string, then indexes it in -Elasticsearch using the index, type, and idField specified by the user. The Elasticsearch server should -be running prior to creating the application. - -This sink is used whenever you need to write data into Elasticsearch. -For example, you may want to read Kafka logs and store them in Elasticsearch -to be able to search on them. - - -Configuration -------------- -**referenceName:** This will be used to uniquely identify this sink for lineage, annotating metadata, etc. - -**es.cluster:** The name of the cluster to connect to; defaults to ``'elasticsearch'``. - -**es.transportAddresses:** The addresses for nodes; specify the address for at least one node, -and separate others by commas; other nodes will be sniffed out. - -**es.index:** The name of the index where the data will be stored; if the index does not already exist, -it will be created using Elasticsearch's default properties. - -**es.type:** The name of the type where the data will be stored; if it does not already exist, it will be created. - -**es.idField:** The field that will determine the id for the document; it should match a fieldname in the -Structured Record of the input; if left blank, Elasticsearch will create a unique id for each document. - - -Example --------- -This example connects to Elasticsearch, which is running locally, and writes the data to -the specified index (*logs*) and type (*cdap*). The data is indexed using the timestamp (*ts*) field -in the record: - - { - "name": "Elasticsearch", - "type": "batchsink", - "properties": { - "es.transportAddresses": "localhost:9300", - "es.index": "logs", - "es.type": "cdap", - "es.idField": "ts" - } - } diff --git a/pom.xml b/pom.xml index 9e72f50..c69cb3c 100644 --- a/pom.xml +++ b/pom.xml @@ -1,4 +1,20 @@ + + @@ -313,7 +329,7 @@ <_exportcontents>co.cask.hydrator.plugin.batch.*;org.elasticsearch.hadoop.mr.*; org.apache.commons.lang;org.apache.commons.*;org.apache.commons.logging.*; - org.codehaus.jackson.*;co.cask.hydrator.plugin.realtime.*;io.netty.* + org.codehaus.jackson.*;io.netty.* *;inline=false;scope=compile true lib @@ -348,6 +364,91 @@ + + org.apache.rat + apache-rat-plugin + 0.10 + + + org.apache.maven.doxia + doxia-core + 1.6 + + + xerces + xercesImpl + + + + + + + rat-check + validate + + check + + + + LICENSE*.txt + + *.rst + *.md + **/*.cdap + **/*.yaml + **/*.md + logs/** + .git/** + .idea/** + **/grok/patterns/** + conf/** + data/** + plugins/** + **/*.patch + **/logrotate.d/** + **/limits.d/** + **/*.json + **/*.json.template + **/MANIFEST.MF + + **/org/apache/hadoop/** + + **/resources/** + + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + 2.17 + + + validate + process-test-classes + + checkstyle.xml + suppressions.xml + UTF-8 + true + true + true + **/org/apache/cassandra/**,**/org/apache/hadoop/** + + + check + + + + + + com.puppycrawl.tools + checkstyle + 6.19 + + + @@ -365,7 +466,18 @@ org.apache.maven.plugins maven-antrun-plugin + + co.cask + cdap-maven-plugin + + + org.apache.maven.plugins + maven-checkstyle-plugin + + + org.apache.rat + apache-rat-plugin + - diff --git a/src/main/java/co/cask/hydrator/plugin/batch/ESProperties.java b/src/main/java/co/cask/hydrator/plugin/batch/ESProperties.java index e30588e..5abc8c3 100644 --- a/src/main/java/co/cask/hydrator/plugin/batch/ESProperties.java +++ b/src/main/java/co/cask/hydrator/plugin/batch/ESProperties.java @@ -1,3 +1,19 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package co.cask.hydrator.plugin.batch; /** diff --git a/src/main/java/co/cask/hydrator/plugin/batch/RecordWritableConverter.java b/src/main/java/co/cask/hydrator/plugin/batch/RecordWritableConverter.java index d010230..7313304 100644 --- a/src/main/java/co/cask/hydrator/plugin/batch/RecordWritableConverter.java +++ b/src/main/java/co/cask/hydrator/plugin/batch/RecordWritableConverter.java @@ -1,3 +1,19 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package co.cask.hydrator.plugin.batch; import co.cask.cdap.api.data.format.StructuredRecord; diff --git a/src/main/java/co/cask/hydrator/plugin/batch/sink/BatchElasticsearchSink.java b/src/main/java/co/cask/hydrator/plugin/batch/sink/BatchElasticsearchSink.java index 58844b3..a9f0a26 100644 --- a/src/main/java/co/cask/hydrator/plugin/batch/sink/BatchElasticsearchSink.java +++ b/src/main/java/co/cask/hydrator/plugin/batch/sink/BatchElasticsearchSink.java @@ -1,6 +1,21 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package co.cask.hydrator.plugin.batch.sink; -import co.cask.hydrator.plugin.batch.ESProperties; import co.cask.cdap.api.annotation.Description; import co.cask.cdap.api.annotation.Macro; import co.cask.cdap.api.annotation.Name; @@ -16,6 +31,7 @@ import co.cask.hydrator.common.ReferencePluginConfig; import co.cask.hydrator.common.batch.JobUtils; import co.cask.hydrator.common.batch.sink.SinkOutputFormatProvider; +import co.cask.hydrator.plugin.batch.ESProperties; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; diff --git a/src/main/java/co/cask/hydrator/plugin/batch/source/ElasticsearchSource.java b/src/main/java/co/cask/hydrator/plugin/batch/source/ElasticsearchSource.java index 577204a..4d65e66 100644 --- a/src/main/java/co/cask/hydrator/plugin/batch/source/ElasticsearchSource.java +++ b/src/main/java/co/cask/hydrator/plugin/batch/source/ElasticsearchSource.java @@ -1,7 +1,21 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package co.cask.hydrator.plugin.batch.source; -import co.cask.hydrator.plugin.batch.ESProperties; -import co.cask.hydrator.plugin.batch.RecordWritableConverter; import co.cask.cdap.api.annotation.Description; import co.cask.cdap.api.annotation.Macro; import co.cask.cdap.api.annotation.Name; @@ -19,6 +33,8 @@ import co.cask.hydrator.common.ReferencePluginConfig; import co.cask.hydrator.common.SourceInputFormatProvider; import co.cask.hydrator.common.batch.JobUtils; +import co.cask.hydrator.plugin.batch.ESProperties; +import co.cask.hydrator.plugin.batch.RecordWritableConverter; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.MapWritable; import org.apache.hadoop.io.Text; diff --git a/src/test/java/plugin/batch/RecordWritableConverterTest.java b/src/test/java/plugin/batch/RecordWritableConverterTest.java index 3c92343..fe551d5 100644 --- a/src/test/java/plugin/batch/RecordWritableConverterTest.java +++ b/src/test/java/plugin/batch/RecordWritableConverterTest.java @@ -1,5 +1,5 @@ /* - * Copyright © 2015 Cask Data, Inc. + * Copyright © 2015-2019 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -16,9 +16,9 @@ package plugin.batch; -import co.cask.hydrator.plugin.batch.RecordWritableConverter; import co.cask.cdap.api.data.format.StructuredRecord; import co.cask.cdap.api.data.schema.Schema; +import co.cask.hydrator.plugin.batch.RecordWritableConverter; import com.google.common.base.Charsets; import com.google.common.collect.ImmutableMap; import org.apache.hadoop.io.MapWritable; diff --git a/suppressions.xml b/suppressions.xml new file mode 100644 index 0000000..c79736f --- /dev/null +++ b/suppressions.xml @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + + + + diff --git a/widgets/Elasticsearch-realtimesink.json b/widgets/Elasticsearch-realtimesink.json deleted file mode 100644 index a82ff51..0000000 --- a/widgets/Elasticsearch-realtimesink.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "metadata": { - "spec-version": "1.5" - }, - "display-name": "Elasticsearch", - "configuration-groups": [ - { - "label": "Elasticsearch Configuration", - "properties": [ - { - "widget-type": "textbox", - "label": "Reference Name", - "name": "referenceName" - }, - { - "widget-type": "textbox", - "label": "Index", - "name": "es.index" - }, - { - "widget-type": "textbox", - "label": "Type", - "name": "es.type" - }, - { - "widget-type": "textbox", - "label": "Query", - "name": "query" - }, - { - "widget-type": "textbox", - "label": "ID Field", - "name": "es.idField" - }, - { - "widget-type": "textbox", - "label": "Transport Addresses", - "name": "es.transportAddresses" - }, - { - "widget-type": "textbox", - "label": "Cluster Name", - "name": "es.cluster" - } - ] - } - ], - "outputs": [ ] -}