From b32206ca2d685cd86b5d4934c64a2acd2bff40d8 Mon Sep 17 00:00:00 2001 From: Jason Bond Pratt Date: Mon, 8 Jun 2015 16:12:14 -0700 Subject: [PATCH 01/15] added support for all carbon config params; made UTC the default timezone --- defaults/main.yml | 97 ++++++++++++++++++++--- templates/carbon.conf.j2 | 162 ++++++++++++++++++++------------------- 2 files changed, 170 insertions(+), 89 deletions(-) diff --git a/defaults/main.yml b/defaults/main.yml index 451421a..68898b0 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -1,17 +1,96 @@ --- - -graphite_user: graphite +graphite_user: graphite graphite_secret_key: UNSAFE_DEFAULT -graphite_time_zone: "America/Los_Angeles" +graphite_time_zone: "UTC" graphite_admin_date_joined: "2014-07-21T10:11:17.464" -graphite_admin_email: "root@localhost.localhdomain" -graphite_admin_first_name: "" -graphite_admin_last_name: "" -graphite_admin_last_login: "2014-07-21T10:11:17.464" -graphite_admin_username: "admin" -graphite_admin_password: "admin" +graphite_admin_email: "root@localhost.localhdomain" +graphite_admin_first_name: "" +graphite_admin_last_name: "" +graphite_admin_last_login: "2014-07-21T10:11:17.464" +graphite_admin_username: "admin" +graphite_admin_password: "admin" # The default is "60s:1d" (1 day data), this will keep data for 5 years. # If you log a lot of data, you may need to restrict this to a shorter time. graphite_storage_schemas_default_retentions: "10s:14d,1m:90d,30m:1y,1h:5y" + +graphite_cache_amqp_exchange: graphite +graphite_cache_amqp_host: localhost +graphite_cache_amqp_metric_name_in_body: false +graphite_cache_amqp_password: guest +graphite_cache_amqp_port: 5672 +graphite_cache_amqp_user: guest +graphite_cache_amqp_verbose: false +graphite_cache_amqp_vhost: / +graphite_cache_bind_patterns: "#" +graphite_cache_cache_query_interface: 0.0.0.0 +graphite_cache_cache_query_port: 7002 +graphite_cache_cache_write_strategy: sorted +graphite_cache_carbon_metric_interval: 60 +graphite_cache_carbon_metric_prefix: carbon +graphite_cache_enable_amqp: false +graphite_cache_enable_log_rotation: true +graphite_cache_enable_manhole: false +graphite_cache_enable_udp_listener: false +graphite_cache_line_receiver_interface: 0.0.0.0 +graphite_cache_line_receiver_port: 2003 +graphite_cache_local_data_dir: /opt/graphite/storage/whisper +graphite_cache_log_cache_hits: false +graphite_cache_log_cache_queue_sorts: true +graphite_cache_log_listener_connections: true +graphite_cache_log_updates: false +graphite_cache_manhole_interface: 127.0.0.1 +graphite_cache_manhole_port: 7222 +graphite_cache_manhole_public_key: ssh-rsa AAAAB3NzaC1yc2EAAAABiwAaAIEAoxN0sv/e4eZCPpi3N3KYvyzRaBaMeS2RsOQ/cDuKv11dlNzVeiyc3RFmCv5Rjwn/lQ79y0zyHxw67qLyhQ/kDzINc4cY41ivuQXm2tPmgvexdrBv5nsfEpjs3gLZfJnyvlcVyWK/lId8WUvEWSWHTzsbtmXAF2raJMdgLTbQ8wE= +graphite_cache_manhole_user: admin +graphite_cache_max_cache_size: inf +graphite_cache_max_creates_per_minute: 50 +graphite_cache_max_updates_per_second: 500 +graphite_cache_max_updates_per_second_on_shutdown: 1000 +graphite_cache_pickle_receiver_interface: 0.0.0.0 +graphite_cache_pickle_receiver_port: 2004 +graphite_cache_udp_receiver_interface: 0.0.0.0 +graphite_cache_udp_receiver_port: 2003 +graphite_cache_use_flow_control: true +graphite_cache_use_insecure_unpickler: false +graphite_cache_use_whitelist: false +graphite_cache_user: +graphite_cache_whisper_autoflush: false +graphite_cache_whisper_fallocate_create: true +graphite_cache_whisper_lock_writes: false +graphite_cache_whisper_sparse_create: false + +graphite_relay_carbon_metric_interval: 60 +graphite_relay_carbon_metric_prefix: carbon +graphite_relay_destinations: 127.0.0.1:2004 +graphite_relay_line_receiver_interface: 0.0.0.0 +graphite_relay_line_receiver_port: 2013 +graphite_relay_log_listener_connections: true +graphite_relay_max_datapoints_per_message: 500 +graphite_relay_max_queue_size: 10000 +graphite_relay_pickle_receiver_interface: 0.0.0.0 +graphite_relay_pickle_receiver_port: 2014 +graphite_relay_relay_method: rules +graphite_relay_replication_factor: 1 +graphite_relay_use_flow_control: true +graphite_relay_use_whitelist: false +graphite_relay_user: + +graphite_aggregator_carbon_metric_interval: 60 +graphite_aggregator_carbon_metric_prefix: carbon +graphite_aggregator_destinations: 127.0.0.1:2004 +graphite_aggregator_forward_all: true +graphite_aggregator_line_receiver_interface: 0.0.0.0 +graphite_aggregator_line_receiver_port: 2023 +graphite_aggregator_log_listener_connections: true +graphite_aggregator_max_aggregation_intervals: 5 +graphite_aggregator_max_datapoints_per_message: 500 +graphite_aggregator_max_queue_size: 10000 +graphite_aggregator_pickle_receiver_interface: 0.0.0.0 +graphite_aggregator_pickle_receiver_port: 2024 +graphite_aggregator_replication_factor: 1 +graphite_aggregator_use_flow_control: true +graphite_aggregator_use_whitelist: false +graphite_aggregator_user: +graphite_aggregator_write_back_frequency: 0 diff --git a/templates/carbon.conf.j2 b/templates/carbon.conf.j2 index 94d9185..81d28c4 100644 --- a/templates/carbon.conf.j2 +++ b/templates/carbon.conf.j2 @@ -28,15 +28,15 @@ # LOG_DIR = /var/log/carbon/ # PID_DIR = /var/run/ # -#LOCAL_DATA_DIR = /opt/graphite/storage/whisper/ +LOCAL_DATA_DIR = {{ graphite_cache_local_data_dir }} # Enable daily log rotation. If disabled, a kill -HUP can be used after a manual rotate -ENABLE_LOGROTATION = True +ENABLE_LOGROTATION = {{ graphite_cache_enable_log_rotation }} # Specify the user to drop privileges to # If this is blank carbon runs as the user that invokes it # This user must have write access to the local data directory -USER = +USER = {{ graphite_cache_user }} # # NOTE: The above settings must be set under [relay] and [aggregator] # to take effect for those daemons as well @@ -44,20 +44,20 @@ USER = # Limit the size of the cache to avoid swapping or becoming CPU bound. # Sorts and serving cache queries gets more expensive as the cache grows. # Use the value "inf" (infinity) for an unlimited cache size. -MAX_CACHE_SIZE = inf +MAX_CACHE_SIZE = {{ graphite_cache_max_cache_size }} # Limits the number of whisper update_many() calls per second, which effectively # means the number of write requests sent to the disk. This is intended to # prevent over-utilizing the disk and thus starving the rest of the system. # When the rate of required updates exceeds this, then carbon's caching will # take effect and increase the overall throughput accordingly. -MAX_UPDATES_PER_SECOND = 500 +MAX_UPDATES_PER_SECOND = {{ graphite_cache_max_updates_per_second }} # If defined, this changes the MAX_UPDATES_PER_SECOND in Carbon when a # stop/shutdown is initiated. This helps when MAX_UPDATES_PER_SECOND is # relatively low and carbon has cached a lot of updates; it enables the carbon # daemon to shutdown more quickly. -# MAX_UPDATES_PER_SECOND_ON_SHUTDOWN = 1000 +MAX_UPDATES_PER_SECOND_ON_SHUTDOWN = {{ graphite_cache_max_updates_per_second }} # Softly limits the number of whisper files that get created each minute. # Setting this value low (like at 50) is a good way to ensure your graphite @@ -66,43 +66,43 @@ MAX_UPDATES_PER_SECOND = 500 # database files to all get created and thus longer until the data becomes usable. # Setting this value high (like "inf" for infinity) will cause graphite to create # the files quickly but at the risk of slowing I/O down considerably for a while. -MAX_CREATES_PER_MINUTE = 50 +MAX_CREATES_PER_MINUTE = {{ graphite_cache_max_creates_per_minute }} -LINE_RECEIVER_INTERFACE = 0.0.0.0 -LINE_RECEIVER_PORT = 2003 +LINE_RECEIVER_INTERFACE = {{ graphite_cache_line_receiver_interface }} +LINE_RECEIVER_PORT = {{ graphite_cache_line_receiver_port }} # Set this to True to enable the UDP listener. By default this is off # because it is very common to run multiple carbon daemons and managing # another (rarely used) port for every carbon instance is not fun. -ENABLE_UDP_LISTENER = False -UDP_RECEIVER_INTERFACE = 0.0.0.0 -UDP_RECEIVER_PORT = 2003 +ENABLE_UDP_LISTENER = {{ graphite_cache_enable_udp_listener }} +UDP_RECEIVER_INTERFACE = {{ graphite_cache_udp_receiver_interface }} +UDP_RECEIVER_PORT = {{ graphite_cache_udp_receiver_port }} -PICKLE_RECEIVER_INTERFACE = 0.0.0.0 -PICKLE_RECEIVER_PORT = 2004 +PICKLE_RECEIVER_INTERFACE = {{ graphite_cache_pickle_receiver_interface }} +PICKLE_RECEIVER_PORT = {{ graphite_cache_pickle_receiver_port }} # Set to false to disable logging of successful connections -LOG_LISTENER_CONNECTIONS = True +LOG_LISTENER_CONNECTIONS = {{ graphite_cache_log_listener_connections }} -# Per security concerns outlined in Bug #817247 the pickle receiver +# per security concerns outlined in Bug #817247 the pickle receiver # will use a more secure and slightly less efficient unpickler. # Set this to True to revert to the old-fashioned insecure unpickler. -USE_INSECURE_UNPICKLER = False +USE_INSECURE_UNPICKLER = {{ graphite_cache_use_insecure_unpickler }} -CACHE_QUERY_INTERFACE = 0.0.0.0 -CACHE_QUERY_PORT = 7002 +CACHE_QUERY_INTERFACE = {{ graphite_cache_cache_query_interface }} +CACHE_QUERY_PORT = {{ graphite_cache_cache_query_port }} # Set this to False to drop datapoints received after the cache # reaches MAX_CACHE_SIZE. If this is True (the default) then sockets # over which metrics are received will temporarily stop accepting # data until the cache size falls below 95% MAX_CACHE_SIZE. -USE_FLOW_CONTROL = True +USE_FLOW_CONTROL = {{ graphite_cache_use_flow_control }} # By default, carbon-cache will log every whisper update and cache hit. This can be excessive and # degrade performance if logging on the same volume as the whisper data is stored. -LOG_UPDATES = False -LOG_CACHE_HITS = False -LOG_CACHE_QUEUE_SORTS = True +LOG_UPDATES = {{ graphite_cache_log_updates }} +LOG_CACHE_HITS = {{ graphite_cache_log_cache_hits }} +LOG_CACHE_QUEUE_SORTS = {{ graphite_cache_log_cache_queue_sorts }} # The thread that writes metrics to disk can use on of the following strategies # determining the order in which metrics are removed from cache and flushed to @@ -127,12 +127,12 @@ LOG_CACHE_QUEUE_SORTS = True # the OS's i/o scheduler is expected to compensate for the random write # pattern. # -CACHE_WRITE_STRATEGY = sorted +CACHE_WRITE_STRATEGY = {{ graphite_cache_cache_write_strategy }} # On some systems it is desirable for whisper to write synchronously. # Set this option to True if you'd like to try this. Basically it will # shift the onus of buffering writes from the kernel into carbon's cache. -WHISPER_AUTOFLUSH = False +WHISPER_AUTOFLUSH = {{ graphite_cache_whisper_autoflush }} # By default new Whisper files are created pre-allocated with the data region # filled with zeros to prevent fragmentation and speed up contiguous reads and @@ -140,7 +140,7 @@ WHISPER_AUTOFLUSH = False # the file sparsely instead. Enabling this option may allow a large increase of # MAX_CREATES_PER_MINUTE but may have longer term performance implications # depending on the underlying storage configuration. -# WHISPER_SPARSE_CREATE = False +WHISPER_SPARSE_CREATE = {{ graphite_cache_whisper_sparse_create }} # Only beneficial on linux filesystems that support the fallocate system call. # It maintains the benefits of contiguous reads/writes, but with a potentially @@ -148,38 +148,38 @@ WHISPER_AUTOFLUSH = False # allocation and zero-ing. Enabling this option may allow a large increase of # MAX_CREATES_PER_MINUTE. If enabled on an OS or filesystem that is unsupported # this option will gracefully fallback to standard POSIX file access methods. -WHISPER_FALLOCATE_CREATE = True +WHISPER_FALLOCATE_CREATE = {{ graphite_cache_whisper_fallocate_create }} # Enabling this option will cause Whisper to lock each Whisper file it writes # to with an exclusive lock (LOCK_EX, see: man 2 flock). This is useful when # multiple carbon-cache daemons are writing to the same files -# WHISPER_LOCK_WRITES = False +WHISPER_LOCK_WRITES = {{ graphite_cache_whisper_lock_writes }} # Set this to True to enable whitelisting and blacklisting of metrics in # CONF_DIR/whitelist and CONF_DIR/blacklist. If the whitelist is missing or # empty, all metrics will pass through -# USE_WHITELIST = False +USE_WHITELIST = {{ graphite_cache_use_whitelist }} # By default, carbon itself will log statistics (such as a count, # metricsReceived) with the top level prefix of 'carbon' at an interval of 60 # seconds. Set CARBON_METRIC_INTERVAL to 0 to disable instrumentation -# CARBON_METRIC_PREFIX = carbon -# CARBON_METRIC_INTERVAL = 60 +CARBON_METRIC_PREFIX = {{ graphite_cache_carbon_metric_prefix }} +CARBON_METRIC_INTERVAL = {{ graphite_cache_carbon_metric_interval }} # Enable AMQP if you want to receve metrics using an amqp broker -# ENABLE_AMQP = False +ENABLE_AMQP = {{ graphite_cache_enable_amqp }} # Verbose means a line will be logged for every metric received # useful for testing -# AMQP_VERBOSE = False +AMQP_VERBOSE = {{ graphite_cache_amqp_verbose }} -# AMQP_HOST = localhost -# AMQP_PORT = 5672 -# AMQP_VHOST = / -# AMQP_USER = guest -# AMQP_PASSWORD = guest -# AMQP_EXCHANGE = graphite -# AMQP_METRIC_NAME_IN_BODY = False +AMQP_HOST = {{ graphite_cache_amqp_host }} +AMQP_PORT = {{ graphite_cache_amqp_port }} +AMQP_VHOST = {{ graphite_cache_amqp_vhost }} +AMQP_USER = {{ graphite_cache_amqp_user }} +AMQP_PASSWORD = {{ graphite_cache_amqp_password }} +AMQP_EXCHANGE = {{ graphite_cache_amqp_exchange }} +AMQP_METRIC_NAME_IN_BODY = {{ graphite_cache_amqp_metric_name_in_body }} # The manhole interface allows you to SSH into the carbon daemon # and get a python interpreter. BE CAREFUL WITH THIS! If you do @@ -188,11 +188,11 @@ WHISPER_FALLOCATE_CREATE = True # you are familiar with the code. If you are not, please don't # mess with this, you are asking for trouble :) # -# ENABLE_MANHOLE = False -# MANHOLE_INTERFACE = 127.0.0.1 -# MANHOLE_PORT = 7222 -# MANHOLE_USER = admin -# MANHOLE_PUBLIC_KEY = ssh-rsa AAAAB3NzaC1yc2EAAAABiwAaAIEAoxN0sv/e4eZCPpi3N3KYvyzRaBaMeS2RsOQ/cDuKv11dlNzVeiyc3RFmCv5Rjwn/lQ79y0zyHxw67qLyhQ/kDzINc4cY41ivuQXm2tPmgvexdrBv5nsfEpjs3gLZfJnyvlcVyWK/lId8WUvEWSWHTzsbtmXAF2raJMdgLTbQ8wE= +ENABLE_MANHOLE = {{ graphite_cache_enable_manhole }} +MANHOLE_INTERFACE = {{ graphite_cache_manhole_interface }} +MANHOLE_PORT = {{ graphite_cache_manhole_port }} +MANHOLE_USER = {{ graphite_cache_manhole_user }} +MANHOLE_PUBLIC_KEY = {{ graphite_cache_manhole_public_key }} # Patterns for all of the metrics this machine will store. Read more at # http://en.wikipedia.org/wiki/Advanced_Message_Queuing_Protocol#Bindings @@ -201,7 +201,7 @@ WHISPER_FALLOCATE_CREATE = True # BIND_PATTERNS = sales.#, servers.linux.#, #.utilization # # Example: store everything -# BIND_PATTERNS = # +BIND_PATTERNS = {{ graphite_cache_bind_patterns }} # To configure special settings for the carbon-cache instance 'b', uncomment this: #[cache:b] @@ -215,15 +215,16 @@ WHISPER_FALLOCATE_CREATE = True [relay] -LINE_RECEIVER_INTERFACE = 0.0.0.0 -LINE_RECEIVER_PORT = 2013 -PICKLE_RECEIVER_INTERFACE = 0.0.0.0 -PICKLE_RECEIVER_PORT = 2014 +USER = {{ graphite_relay_user }} +LINE_RECEIVER_INTERFACE = {{ graphite_relay_line_receiver_interface }} +LINE_RECEIVER_PORT = {{ graphite_relay_line_receiver_port }} +PICKLE_RECEIVER_INTERFACE = {{ graphite_relay_pickle_receiver_interface }} +PICKLE_RECEIVER_PORT = {{ graphite_relay_pickle_receiver_port }} # Set to false to disable logging of successful connections -LOG_LISTENER_CONNECTIONS = True +LOG_LISTENER_CONNECTIONS = {{ graphite_relay_log_listener_connections }} -# Carbon-relay has several options for metric routing controlled by RELAY_METHOD +# carbon-relay has several options for metric routing controlled by RELAY_METHOD # # Use relay-rules.conf to route metrics to destinations based on pattern rules #RELAY_METHOD = rules @@ -237,11 +238,11 @@ LOG_LISTENER_CONNECTIONS = True # instance. # Enable this for carbon-relays that send to a group of carbon-aggregators #RELAY_METHOD = aggregated-consistent-hashing -RELAY_METHOD = rules +RELAY_METHOD = {{ graphite_relay_relay_method }} # If you use consistent-hashing you can add redundancy by replicating every # datapoint to more than one machine. -REPLICATION_FACTOR = 1 +REPLICATION_FACTOR = {{ graphite_relay_replication_factor }} # This is a list of carbon daemons we will send any relayed or # generated metrics to. The default provided would send to a single @@ -259,47 +260,48 @@ REPLICATION_FACTOR = 1 # # If using RELAY_METHOD = rules, all destinations used in relay-rules.conf # must be defined in this list -DESTINATIONS = 127.0.0.1:2004 +DESTINATIONS = {{ graphite_relay_destinations }} # This defines the maximum "message size" between carbon daemons. # You shouldn't need to tune this unless you really know what you're doing. -MAX_DATAPOINTS_PER_MESSAGE = 500 -MAX_QUEUE_SIZE = 10000 +MAX_DATAPOINTS_PER_MESSAGE = {{ graphite_relay_max_datapoints_per_message }} +MAX_QUEUE_SIZE = {{ graphite_relay_max_queue_size }} # Set this to False to drop datapoints when any send queue (sending datapoints # to a downstream carbon daemon) hits MAX_QUEUE_SIZE. If this is True (the # default) then sockets over which metrics are received will temporarily stop accepting # data until the send queues fall below 80% MAX_QUEUE_SIZE. -USE_FLOW_CONTROL = True +USE_FLOW_CONTROL = {{ graphite_relay_use_flow_control }} -# Set this to True to enable whitelisting and blacklisting of metrics in +# set this to True to enable whitelisting and blacklisting of metrics in # CONF_DIR/whitelist and CONF_DIR/blacklist. If the whitelist is missing or # empty, all metrics will pass through -# USE_WHITELIST = False +USE_WHITELIST = {{ graphite_relay_use_whitelist }} # By default, carbon itself will log statistics (such as a count, # metricsReceived) with the top level prefix of 'carbon' at an interval of 60 # seconds. Set CARBON_METRIC_INTERVAL to 0 to disable instrumentation -# CARBON_METRIC_PREFIX = carbon -# CARBON_METRIC_INTERVAL = 60 +CARBON_METRIC_PREFIX = {{ graphite_relay_carbon_metric_prefix }} +CARBON_METRIC_INTERVAL = {{ graphite_relay_carbon_metric_interval }} [aggregator] -LINE_RECEIVER_INTERFACE = 0.0.0.0 -LINE_RECEIVER_PORT = 2023 +USER = {{ graphite_aggregator_user }} +LINE_RECEIVER_INTERFACE = {{ graphite_aggregator_line_receiver_interface }} +LINE_RECEIVER_PORT = {{ graphite_aggregator_line_receiver_port }} -PICKLE_RECEIVER_INTERFACE = 0.0.0.0 -PICKLE_RECEIVER_PORT = 2024 +PICKLE_RECEIVER_INTERFACE = {{ graphite_aggregator_pickle_receiver_interface }} +PICKLE_RECEIVER_PORT = {{ graphite_aggregator_pickle_receiver_port }} # Set to false to disable logging of successful connections -LOG_LISTENER_CONNECTIONS = True +LOG_LISTENER_CONNECTIONS = {{ graphite_aggregator_log_listener_connections }} # If set true, metric received will be forwarded to DESTINATIONS in addition to # the output of the aggregation rules. If set false the carbon-aggregator will # only ever send the output of aggregation. -FORWARD_ALL = True +FORWARD_ALL = {{ graphite_aggregator_forward_all }} -# This is a list of carbon daemons we will send any relayed or +# this is a list of carbon daemons we will send any relayed or # generated metrics to. The default provided would send to a single # carbon-cache instance on the default port. However if you # use multiple carbon-cache instances then it would look like this: @@ -312,32 +314,32 @@ FORWARD_ALL = True # Note that if the destinations are all carbon-caches then this should # exactly match the webapp's CARBONLINK_HOSTS setting in terms of # instances listed (order matters!). -DESTINATIONS = 127.0.0.1:2004 +DESTINATIONS = {{ graphite_aggregator_destinations }} -# If you want to add redundancy to your data by replicating every +# if you want to add redundancy to your data by replicating every # datapoint to more than one machine, increase this. -REPLICATION_FACTOR = 1 +REPLICATION_FACTOR = {{ graphite_aggregator_replication_factor }} # This is the maximum number of datapoints that can be queued up # for a single destination. Once this limit is hit, we will # stop accepting new data if USE_FLOW_CONTROL is True, otherwise # we will drop any subsequently received datapoints. -MAX_QUEUE_SIZE = 10000 +MAX_QUEUE_SIZE = {{ graphite_aggregator_max_queue_size }} # Set this to False to drop datapoints when any send queue (sending datapoints # to a downstream carbon daemon) hits MAX_QUEUE_SIZE. If this is True (the # default) then sockets over which metrics are received will temporarily stop accepting # data until the send queues fall below 80% MAX_QUEUE_SIZE. -USE_FLOW_CONTROL = True +USE_FLOW_CONTROL = {{ graphite_aggregator_use_flow_control }} # This defines the maximum "message size" between carbon daemons. # You shouldn't need to tune this unless you really know what you're doing. -MAX_DATAPOINTS_PER_MESSAGE = 500 +MAX_DATAPOINTS_PER_MESSAGE = {{ graphite_aggregator_max_datapoints_per_message }} # This defines how many datapoints the aggregator remembers for # each metric. Aggregation only happens for datapoints that fall in # the past MAX_AGGREGATION_INTERVALS * intervalSize seconds. -MAX_AGGREGATION_INTERVALS = 5 +MAX_AGGREGATION_INTERVALS = {{ graphite_aggregator_max_aggregation_intervals }} # By default (WRITE_BACK_FREQUENCY = 0), carbon-aggregator will write back # aggregated data points once every rule.frequency seconds, on a per-rule basis. @@ -345,15 +347,15 @@ MAX_AGGREGATION_INTERVALS = 5 # every N seconds, independent of rule frequency. This is useful, for example, # to be able to query partially aggregated metrics from carbon-cache without # having to first wait rule.frequency seconds. -# WRITE_BACK_FREQUENCY = 0 +WRITE_BACK_FREQUENCY = {{ graphite_aggregator_write_back_frequency }} # Set this to True to enable whitelisting and blacklisting of metrics in # CONF_DIR/whitelist and CONF_DIR/blacklist. If the whitelist is missing or # empty, all metrics will pass through -# USE_WHITELIST = False +USE_WHITELIST = {{ graphite_aggregator_use_whitelist }} # By default, carbon itself will log statistics (such as a count, # metricsReceived) with the top level prefix of 'carbon' at an interval of 60 # seconds. Set CARBON_METRIC_INTERVAL to 0 to disable instrumentation -# CARBON_METRIC_PREFIX = carbon -# CARBON_METRIC_INTERVAL = 60 +CARBON_METRIC_PREFIX = {{ graphite_aggregator_carbon_metric_prefix }} +CARBON_METRIC_INTERVAL = {{ graphite_aggregator_carbon_metric_interval }} From 9115aedbf4d9737556f39c3dea42ef0d6236c875 Mon Sep 17 00:00:00 2001 From: Jason Bond Pratt Date: Mon, 8 Jun 2015 16:36:39 -0700 Subject: [PATCH 02/15] parameterized carbon stats retention schema --- defaults/main.yml | 1 + templates/storage-schemas.conf.j2 | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/defaults/main.yml b/defaults/main.yml index 68898b0..d87796f 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -14,6 +14,7 @@ graphite_admin_password: "admin" # The default is "60s:1d" (1 day data), this will keep data for 5 years. # If you log a lot of data, you may need to restrict this to a shorter time. graphite_storage_schemas_default_retentions: "10s:14d,1m:90d,30m:1y,1h:5y" +graphite_storage_schemas_carbon_retentions: 60:90d graphite_cache_amqp_exchange: graphite graphite_cache_amqp_host: localhost diff --git a/templates/storage-schemas.conf.j2 b/templates/storage-schemas.conf.j2 index ffc5db3..2585dfc 100644 --- a/templates/storage-schemas.conf.j2 +++ b/templates/storage-schemas.conf.j2 @@ -9,7 +9,7 @@ # CARBON_METRIC_PREFIX and CARBON_METRIC_INTERVAL settings [carbon] pattern = ^carbon\. -retentions = 60:90d +retentions = {{ graphite_storage_schemas_carbon_retentions }} [default] pattern = .* From 49debae0b8403b689400acfc459ab3ab2407756a Mon Sep 17 00:00:00 2001 From: Jason Bond Pratt Date: Mon, 8 Jun 2015 17:18:50 -0700 Subject: [PATCH 03/15] parameterized storage schemas --- templates/storage-schemas.conf.j2 | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/templates/storage-schemas.conf.j2 b/templates/storage-schemas.conf.j2 index 2585dfc..ee82c07 100644 --- a/templates/storage-schemas.conf.j2 +++ b/templates/storage-schemas.conf.j2 @@ -5,6 +5,12 @@ # pattern = regex # retentions = timePerPoint:timeToStore, timePerPoint:timeToStore, ... +{% for schema in graphite_storage_schemas %} +[{{ schema.name }}] +pattern = {{ schema.pattern }} +retentions = {{ schema.retentions }} +{% endfor %} + # Carbon's internal metrics. This entry should match what is specified in # CARBON_METRIC_PREFIX and CARBON_METRIC_INTERVAL settings [carbon] @@ -14,3 +20,4 @@ retentions = {{ graphite_storage_schemas_carbon_retentions }} [default] pattern = .* retentions = {{ graphite_storage_schemas_default_retentions }} + From 06eeb2dac4ab59308cd4d967f96a755c2c895dbd Mon Sep 17 00:00:00 2001 From: Jason Bond Pratt Date: Mon, 8 Jun 2015 17:46:06 -0700 Subject: [PATCH 04/15] re-added cache refresh for smooth install --- tasks/Debian.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/Debian.yml b/tasks/Debian.yml index ec089ad..7768a84 100644 --- a/tasks/Debian.yml +++ b/tasks/Debian.yml @@ -1,7 +1,7 @@ --- - name: Install packages with apt - apt: name={{ item }} state=present + apt: name={{ item }} state=present update_cache=true with_items: - python-pip - python-dev From a648f21571e41b62d471e4bf93bc6b58c8b51462 Mon Sep 17 00:00:00 2001 From: Stefan Berggren Date: Mon, 15 Jun 2015 23:40:50 +0200 Subject: [PATCH 05/15] django-tagging 4.0 has dropped support django<1.7 --- tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/main.yml b/tasks/main.yml index 636871c..27cb93a 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -19,7 +19,7 @@ - carbon - graphite-web - "django==1.5" # Current version of graphite do not support 1.6 - - django-tagging + - "django-tagging==0.3.6" # Never versions require Django 1.7+ environment: PYTHONPATH: "{{ graphite_install_path }}/lib:{{ graphite_install_path }}/webapp" notify: restart carbon-cache From 16741df1bd6340bfdb643fdec770b0ade25d088c Mon Sep 17 00:00:00 2001 From: Stefan Berggren Date: Tue, 16 Jun 2015 00:06:00 +0200 Subject: [PATCH 06/15] Add missing variable. --- defaults/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/defaults/main.yml b/defaults/main.yml index d87796f..93c1b46 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -15,6 +15,7 @@ graphite_admin_password: "admin" # If you log a lot of data, you may need to restrict this to a shorter time. graphite_storage_schemas_default_retentions: "10s:14d,1m:90d,30m:1y,1h:5y" graphite_storage_schemas_carbon_retentions: 60:90d +graphite_storage_schemas: [] graphite_cache_amqp_exchange: graphite graphite_cache_amqp_host: localhost From 158f38cc56dedf7162b1f7916f9579f4c5e01dae Mon Sep 17 00:00:00 2001 From: Stefan Berggren Date: Tue, 16 Jun 2015 00:08:43 +0200 Subject: [PATCH 07/15] We got "USER = None", this fixes that. --- defaults/main.yml | 2 +- templates/carbon.conf.j2 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/defaults/main.yml b/defaults/main.yml index 93c1b46..1c25455 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -57,7 +57,7 @@ graphite_cache_udp_receiver_port: 2003 graphite_cache_use_flow_control: true graphite_cache_use_insecure_unpickler: false graphite_cache_use_whitelist: false -graphite_cache_user: +#graphite_cache_user: graphite_cache_whisper_autoflush: false graphite_cache_whisper_fallocate_create: true graphite_cache_whisper_lock_writes: false diff --git a/templates/carbon.conf.j2 b/templates/carbon.conf.j2 index 81d28c4..05e3b89 100644 --- a/templates/carbon.conf.j2 +++ b/templates/carbon.conf.j2 @@ -36,7 +36,7 @@ ENABLE_LOGROTATION = {{ graphite_cache_enable_log_rotation }} # Specify the user to drop privileges to # If this is blank carbon runs as the user that invokes it # This user must have write access to the local data directory -USER = {{ graphite_cache_user }} +USER = {{ graphite_cache_user|default("") }} # # NOTE: The above settings must be set under [relay] and [aggregator] # to take effect for those daemons as well From abd8b398ec685791c987143a4dc29c3b8b8b6e4a Mon Sep 17 00:00:00 2001 From: Stefan Berggren Date: Tue, 16 Jun 2015 00:13:53 +0200 Subject: [PATCH 08/15] Fixed typo, incorrect variable was used. --- templates/carbon.conf.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/carbon.conf.j2 b/templates/carbon.conf.j2 index 05e3b89..bbc4b0c 100644 --- a/templates/carbon.conf.j2 +++ b/templates/carbon.conf.j2 @@ -57,7 +57,7 @@ MAX_UPDATES_PER_SECOND = {{ graphite_cache_max_updates_per_second }} # stop/shutdown is initiated. This helps when MAX_UPDATES_PER_SECOND is # relatively low and carbon has cached a lot of updates; it enables the carbon # daemon to shutdown more quickly. -MAX_UPDATES_PER_SECOND_ON_SHUTDOWN = {{ graphite_cache_max_updates_per_second }} +MAX_UPDATES_PER_SECOND_ON_SHUTDOWN = {{ graphite_cache_max_updates_per_second_on_shutdown }} # Softly limits the number of whisper files that get created each minute. # Setting this value low (like at 50) is a good way to ensure your graphite From c451ab03e5f6f057cecb14298847240954d783cb Mon Sep 17 00:00:00 2001 From: Stefan Berggren Date: Tue, 16 Jun 2015 00:15:06 +0200 Subject: [PATCH 09/15] Remove extra space. --- templates/carbon.conf.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/carbon.conf.j2 b/templates/carbon.conf.j2 index bbc4b0c..eac5754 100644 --- a/templates/carbon.conf.j2 +++ b/templates/carbon.conf.j2 @@ -76,7 +76,7 @@ LINE_RECEIVER_PORT = {{ graphite_cache_line_receiver_port }} # another (rarely used) port for every carbon instance is not fun. ENABLE_UDP_LISTENER = {{ graphite_cache_enable_udp_listener }} UDP_RECEIVER_INTERFACE = {{ graphite_cache_udp_receiver_interface }} -UDP_RECEIVER_PORT = {{ graphite_cache_udp_receiver_port }} +UDP_RECEIVER_PORT = {{ graphite_cache_udp_receiver_port }} PICKLE_RECEIVER_INTERFACE = {{ graphite_cache_pickle_receiver_interface }} PICKLE_RECEIVER_PORT = {{ graphite_cache_pickle_receiver_port }} From 1d005a092e503ee3d89127718250ec4e7aa9b6a8 Mon Sep 17 00:00:00 2001 From: Stefan Berggren Date: Tue, 16 Jun 2015 00:16:08 +0200 Subject: [PATCH 10/15] Revert to upper case P. --- templates/carbon.conf.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/carbon.conf.j2 b/templates/carbon.conf.j2 index eac5754..c0a0a27 100644 --- a/templates/carbon.conf.j2 +++ b/templates/carbon.conf.j2 @@ -84,7 +84,7 @@ PICKLE_RECEIVER_PORT = {{ graphite_cache_pickle_receiver_port }} # Set to false to disable logging of successful connections LOG_LISTENER_CONNECTIONS = {{ graphite_cache_log_listener_connections }} -# per security concerns outlined in Bug #817247 the pickle receiver +# Per security concerns outlined in Bug #817247 the pickle receiver # will use a more secure and slightly less efficient unpickler. # Set this to True to revert to the old-fashioned insecure unpickler. USE_INSECURE_UNPICKLER = {{ graphite_cache_use_insecure_unpickler }} From 13898a66b551e7f99704b7e7eb2fc3914a0781c6 Mon Sep 17 00:00:00 2001 From: Stefan Berggren Date: Tue, 16 Jun 2015 00:23:26 +0200 Subject: [PATCH 11/15] Only show USER, if it is defined. --- defaults/main.yml | 2 +- templates/carbon.conf.j2 | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/defaults/main.yml b/defaults/main.yml index 1c25455..65a4868 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -77,7 +77,7 @@ graphite_relay_relay_method: rules graphite_relay_replication_factor: 1 graphite_relay_use_flow_control: true graphite_relay_use_whitelist: false -graphite_relay_user: +#graphite_relay_user: graphite_aggregator_carbon_metric_interval: 60 graphite_aggregator_carbon_metric_prefix: carbon diff --git a/templates/carbon.conf.j2 b/templates/carbon.conf.j2 index c0a0a27..74db5b4 100644 --- a/templates/carbon.conf.j2 +++ b/templates/carbon.conf.j2 @@ -215,7 +215,9 @@ BIND_PATTERNS = {{ graphite_cache_bind_patterns }} [relay] +{% if graphite_relay_user is defined -%} USER = {{ graphite_relay_user }} +{% endif -%} LINE_RECEIVER_INTERFACE = {{ graphite_relay_line_receiver_interface }} LINE_RECEIVER_PORT = {{ graphite_relay_line_receiver_port }} PICKLE_RECEIVER_INTERFACE = {{ graphite_relay_pickle_receiver_interface }} From 480ab125abbed072185b9a759034e65af3361934 Mon Sep 17 00:00:00 2001 From: Stefan Berggren Date: Tue, 16 Jun 2015 00:25:07 +0200 Subject: [PATCH 12/15] Reverted lover case C. --- templates/carbon.conf.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/carbon.conf.j2 b/templates/carbon.conf.j2 index 74db5b4..57ea0b9 100644 --- a/templates/carbon.conf.j2 +++ b/templates/carbon.conf.j2 @@ -226,7 +226,7 @@ PICKLE_RECEIVER_PORT = {{ graphite_relay_pickle_receiver_port }} # Set to false to disable logging of successful connections LOG_LISTENER_CONNECTIONS = {{ graphite_relay_log_listener_connections }} -# carbon-relay has several options for metric routing controlled by RELAY_METHOD +# Carbon-relay has several options for metric routing controlled by RELAY_METHOD # # Use relay-rules.conf to route metrics to destinations based on pattern rules #RELAY_METHOD = rules From ab7871ee4103bad467329ce5c84752f15218476e Mon Sep 17 00:00:00 2001 From: Stefan Berggren Date: Tue, 16 Jun 2015 00:26:25 +0200 Subject: [PATCH 13/15] revert: Change set to Set. --- templates/carbon.conf.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/carbon.conf.j2 b/templates/carbon.conf.j2 index 57ea0b9..66ee93a 100644 --- a/templates/carbon.conf.j2 +++ b/templates/carbon.conf.j2 @@ -275,7 +275,7 @@ MAX_QUEUE_SIZE = {{ graphite_relay_max_queue_size }} # data until the send queues fall below 80% MAX_QUEUE_SIZE. USE_FLOW_CONTROL = {{ graphite_relay_use_flow_control }} -# set this to True to enable whitelisting and blacklisting of metrics in +# Set this to True to enable whitelisting and blacklisting of metrics in # CONF_DIR/whitelist and CONF_DIR/blacklist. If the whitelist is missing or # empty, all metrics will pass through USE_WHITELIST = {{ graphite_relay_use_whitelist }} From 027d2071002dc94b80b41b3460810fd251cd7d31 Mon Sep 17 00:00:00 2001 From: Stefan Berggren Date: Tue, 16 Jun 2015 00:27:52 +0200 Subject: [PATCH 14/15] Hide USER, if not defined. --- defaults/main.yml | 2 +- templates/carbon.conf.j2 | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/defaults/main.yml b/defaults/main.yml index 65a4868..f58ebf9 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -94,5 +94,5 @@ graphite_aggregator_pickle_receiver_port: 2024 graphite_aggregator_replication_factor: 1 graphite_aggregator_use_flow_control: true graphite_aggregator_use_whitelist: false -graphite_aggregator_user: +#graphite_aggregator_user: graphite_aggregator_write_back_frequency: 0 diff --git a/templates/carbon.conf.j2 b/templates/carbon.conf.j2 index 66ee93a..1d46edf 100644 --- a/templates/carbon.conf.j2 +++ b/templates/carbon.conf.j2 @@ -288,7 +288,9 @@ CARBON_METRIC_INTERVAL = {{ graphite_relay_carbon_metric_interval }} [aggregator] +{% if graphite_aggregator_user is defined -%} USER = {{ graphite_aggregator_user }} +{% endif -%} LINE_RECEIVER_INTERFACE = {{ graphite_aggregator_line_receiver_interface }} LINE_RECEIVER_PORT = {{ graphite_aggregator_line_receiver_port }} From b49a899520f491555c5eef5fe91b38feecc67621 Mon Sep 17 00:00:00 2001 From: Stefan Berggren Date: Tue, 16 Jun 2015 00:29:18 +0200 Subject: [PATCH 15/15] More lowercase... --- templates/carbon.conf.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/carbon.conf.j2 b/templates/carbon.conf.j2 index 1d46edf..38ebbc6 100644 --- a/templates/carbon.conf.j2 +++ b/templates/carbon.conf.j2 @@ -305,7 +305,7 @@ LOG_LISTENER_CONNECTIONS = {{ graphite_aggregator_log_listener_connections }} # only ever send the output of aggregation. FORWARD_ALL = {{ graphite_aggregator_forward_all }} -# this is a list of carbon daemons we will send any relayed or +# This is a list of carbon daemons we will send any relayed or # generated metrics to. The default provided would send to a single # carbon-cache instance on the default port. However if you # use multiple carbon-cache instances then it would look like this: @@ -320,7 +320,7 @@ FORWARD_ALL = {{ graphite_aggregator_forward_all }} # instances listed (order matters!). DESTINATIONS = {{ graphite_aggregator_destinations }} -# if you want to add redundancy to your data by replicating every +# If you want to add redundancy to your data by replicating every # datapoint to more than one machine, increase this. REPLICATION_FACTOR = {{ graphite_aggregator_replication_factor }}