Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

CloudWatch custom dimensions and boto3 #679

Open
wants to merge 5 commits into
base: master
Choose a base branch
from
Open

CloudWatch custom dimensions and boto3 #679

Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
e05eef8
clarify help for what collect_by_instance and collect_without_dimensi…
jantman Oct 1, 2017
4ce10b2
cloudwatch handler - add support for user-specified dimensions
jantman Oct 1, 2017
5ac5c43
cloudwatch handler - set timeout on boto.utils.get_instance_metadata …
jantman Oct 1, 2017
19d136f
cloudwatch - additional dimensions should be a separate metric send
jantman Oct 1, 2017
4d99403
issue #678 - convert cloudwatch handler to boto3
jantman Oct 6, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
85 changes: 61 additions & 24 deletions src/diamond/handler/cloudwatch.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

#### Dependencies

* [boto](http://boto.readthedocs.org/en/latest/index.html)
* [boto3](http://boto3.readthedocs.org/en/latest/)

#### Configuration

Expand Down Expand Up @@ -36,6 +36,18 @@
name = Avg05
namespace = MachineLoad
unit = None

[[[SnmpUptime]]]
collect_by_instance = False
collect_without_dimension = True
collector = snmpraw
metric = sysUpTimeInstance
name = Uptime
namespace = Diamond
unit = None
[[[[collect_with_dimensions]]]]
Hostname = foo

"""

import sys
Expand All @@ -45,11 +57,11 @@
from configobj import Section

try:
import boto
import boto.ec2.cloudwatch
import boto.utils
import boto3
from botocore.utils import InstanceMetadataFetcher
except ImportError:
boto = None
boto3 = None
InstanceMetadataFetcher = None


class cloudwatchHandler(Handler):
Expand All @@ -66,7 +78,7 @@ def __init__(self, config=None):
# Initialize Handler
Handler.__init__(self, config)

if not boto:
if not boto3:
self.log.error(
"CloudWatch: Boto is not installed, please install boto.")
return
Expand All @@ -77,17 +89,22 @@ def __init__(self, config=None):
# Initialize Options
self.region = self.config['region']

instance_metadata = boto.utils.get_instance_metadata()
if 'instance-id' in instance_metadata:
self.instance_id = instance_metadata['instance-id']
try:
self.instance_id = InstanceMetadataFetcher(
timeout=1, num_attempts=5
)._get_request(
'http://169.254.169.254/latest/meta-data/instance-id',
1, num_attempts=5
).text.strip()
self.log.debug("Setting InstanceId: " + self.instance_id)
else:
except Exception:
self.instance_id = None
self.log.error('CloudWatch: Failed to load instance metadata')

self.valid_config = ('region', 'collector', 'metric', 'namespace',
'name', 'unit', 'collect_by_instance',
'collect_without_dimension')
'collect_without_dimension',
'collect_with_dimensions')

self.rules = []
for key_name, section in self.config.items():
Expand Down Expand Up @@ -118,7 +135,8 @@ def get_default_rule_config(self):
'name': '',
'unit': 'None',
'collect_by_instance': True,
'collect_without_dimension': False
'collect_without_dimension': False,
'collect_with_dimensions': {}
})
return config

Expand All @@ -135,8 +153,10 @@ def get_default_config_help(self):
'name': 'CloudWatch metric name',
'unit': 'CloudWatch metric unit',
'collector': 'Diamond collector name',
'collect_by_instance': 'Collect metrics for instances separately',
'collect_without_dimension': 'Collect metrics without dimension'
'collect_by_instance': 'Send metric with InstanceId dimension',
'collect_without_dimension': 'Send metric with no dimension',
'collect_with_dimensions': 'Name/Value additional dimensions to '
'send metric with'
})

return config
Expand All @@ -155,7 +175,8 @@ def get_default_config(self):
'name': 'Avg01',
'unit': 'None',
'collect_by_instance': True,
'collect_without_dimension': False
'collect_without_dimension': False,
'collect_with_dimensions': {}
})

return config
Expand All @@ -169,12 +190,13 @@ def _bind(self):
"CloudWatch: Attempting to connect to CloudWatch at Region: %s",
self.region)
try:
self.connection = boto.ec2.cloudwatch.connect_to_region(
self.region)
self.connection = boto3.client(
'cloudwatch', region_name=self.region
)
self.log.debug(
"CloudWatch: Succesfully Connected to CloudWatch at Region: %s",
self.region)
except boto.exception.EC2ResponseError:
except boto3.exceptions.Boto3Error:
self.log.error('CloudWatch: CloudWatch Exception Handler: ')

def __del__(self):
Expand All @@ -190,7 +212,7 @@ def process(self, metric):
"""
Process a metric and send it to CloudWatch
"""
if not boto:
if not boto3:
return

collector = str(metric.getCollectorPath())
Expand Down Expand Up @@ -223,6 +245,12 @@ def process(self, metric):
metric,
{})

if len(rule['collect_with_dimensions']) > 0:
self.send_metrics_to_cloudwatch(
rule,
metric,
rule['collect_with_dimensions'])

def send_metrics_to_cloudwatch(self, rule, metric, dimensions):
"""
Send metrics to CloudWatch for the given dimensions
Expand All @@ -242,11 +270,20 @@ def send_metrics_to_cloudwatch(self, rule, metric, dimensions):

try:
self.connection.put_metric_data(
str(rule['namespace']),
str(rule['name']),
str(metric.value),
timestamp, str(rule['unit']),
dimensions)
Namespace=str(rule['namespace']),
MetricData=[
{
'MetricName': str(rule['name']),
'Dimensions': [
{'Name': x, 'Value': dimensions[x]}
for x in dimensions.keys()
],
'Timestamp': timestamp,
'Value': metric.value,
'Unit': str(rule['unit'])
}
]
)
self.log.debug(
"CloudWatch: Successfully published metric: %s to"
" %s with value (%s) for dimensions %s",
Expand Down