Skip to content

Commit

Permalink
unable to emit metric should not fail the pipeline
Browse files Browse the repository at this point in the history
  • Loading branch information
itsankit-google committed Oct 27, 2023
1 parent 0c1f8e9 commit 1c2fae6
Show file tree
Hide file tree
Showing 2 changed files with 12 additions and 2 deletions.
2 changes: 1 addition & 1 deletion pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@

<groupId>io.cdap.plugin</groupId>
<artifactId>google-cloud</artifactId>
<version>0.22.4</version>
<version>0.22.5</version>
<name>Google Cloud Plugins</name>
<packaging>jar</packaging>
<description>Plugins for Google Big Query</description>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -169,14 +169,24 @@ public void run(ActionContext context) throws Exception {
}
}

context.getMetrics().gauge(RECORDS_PROCESSED, rows);
try {
recordBytesProcessedMetric(context, queryJob);
} catch (Exception exception) {
// log the exception but not fail the pipeline
LOG.warn("Exception while trying to emit bytes processed metric.",
exception);
}
}

private void recordBytesProcessedMetric(ActionContext context, Job queryJob) {
long processedBytes =
((JobStatistics.QueryStatistics) queryJob.getStatistics()).getTotalBytesProcessed();
LOG.info("Job {} processed {} bytes", queryJob.getJobId(), processedBytes);
Map<String, String> tags = new ImmutableMap.Builder<String, String>()
.put(Constants.Metrics.Tag.APP_ENTITY_TYPE, Action.PLUGIN_TYPE)
.put(Constants.Metrics.Tag.APP_ENTITY_TYPE_NAME, BigQueryExecute.NAME)
.build();
context.getMetrics().gauge(RECORDS_PROCESSED, rows);
context.getMetrics().child(tags).countLong(BigQuerySinkUtils.BYTES_PROCESSED_METRIC,
processedBytes);
}
Expand Down

0 comments on commit 1c2fae6

Please sign in to comment.