sync fb_syslog with upstream

cookbooks/fb_syslog/README.md

@@ -9,6 +9,7 @@ Attributes
 ----------
 * node['fb_syslog']['syslog-entries']
 * node['fb_syslog']['rsyslog_server']
+* node['fb_syslog']['rsyslog_server_address']
 * node['fb_syslog']['rsyslog_upstream']
 * node['fb_syslog']['rsyslog_port']
 * node['fb_syslog']['rsyslog_early_lines']
@@ -19,6 +20,7 @@ Attributes
 * node['fb_syslog']['rsyslog_additional_sockets']
 * node['fb_syslog']['rsyslog_facilities_sent_to_remote']
 * node['fb_syslog']['rsyslog_omprog_binary']
+* node['fb_syslog']['rsyslog_omprog_binary_args']
 * node['fb_syslog']['rsyslog_use_omprog']
 * node['fb_syslog']['rsyslog_use_omprog_force']
 * node['fb_syslog']['rsyslog_stats_logging']
@@ -142,6 +144,10 @@ $InputUDPServerRun 514
 
 These don't take effect unless `rsyslog_server` is set.
 
+By default, rsyslog listens on the wildcard address. If you want to listen on
+another address (e.g. localhost), you can set the
+`node['fb_syslog']['rsyslog_server_address']` attribute.
+
 ### Escaping control characters in messages
 If messages entering the syslog system contain control characters and it's
 causing you problems, you can enable escaping of non-printable characters by
@@ -192,6 +198,7 @@ that binary. For example:
 node.default['fb_syslog']['rsyslog_facilities_sent_to_remote'] << 'auth.*'
 node.default['fb_syslog']['rsyslog_use_omprog'] = true
 node.default['fb_syslog']['rsyslog_omprog_binary'] = '/usr/bin/myprogram'
+node.default['fb_syslog']['rsyslog_omprog_binary_args'] << '-myProgramArg=42'
 ```
 
 By default, program forwarding (omprog) will only be enabled if
@@ -203,6 +210,12 @@ and a rsyslog server simultaneously. For example:
 node.default['fb_syslog']['rsyslog_use_omprog_force'] = true
 ```
 
+Strings added to `node.default['fb_syslog']['rsyslog_omprog_binary_args']` will
+become command line arguments separated by spaces and appended to
+`node.default['fb_syslog']['rsyslog_omprog_binary']`. Avoid strings
+containing spaces or double quotes, rsyslog/omprog does not support
+proper escaping.
+
 ### Suspension reporting
 Setting `node['fb_syslog']['rsyslog_report_suspension']` controls suspension
 reporting, which defaults to `off`. If the attriubte is set to `nil` suspension

cookbooks/fb_syslog/attributes/default.rb

@@ -30,6 +30,11 @@
   }
 end
 
+syslog_file = value_for_platform_family(
+  ['rhel', 'fedora'] => '/var/log/messages',
+  'debian' => '/var/log/syslog',
+)
+
 # Add in some reasonable defaults for all syslog.confs
 default['fb_syslog'] = {
   'syslog-entries' => {
@@ -38,8 +43,8 @@
       'comment' => 'Log anything info level or higher. A lot ' +
                    'of things go into their own file.',
       'selector' => '*.info;mail,authpriv,cron,' +
-                    'local2,local3,local5,local6.none',
-      'action' => '-/var/log/messages',
+        'local0,local1,local2,local3,local5,local6,local7.none',
+      'action' => "-#{syslog_file}",
     },
     'mail' => {
       'comment' => 'Log all the mail messages in one place.',
@@ -54,7 +59,7 @@
     'emergency' => {
       'comment' => 'Everybody gets emergency messages',
       'selector' => '*.emerg',
-      'action' => '*',
+      'action' => ':omusrmsg:*',
     },
     'news' => {
       'comment' => 'Save news errors of level crit and higher ' +
@@ -69,12 +74,20 @@
     },
   },
   'rsyslog_server' => false,
+  'rsyslog_server_address' => nil,
   'rsyslog_rulesets' => {},
   'rsyslog_nonruleset_ports' => {
     'tcp' => [],
     'udp' => [],
   },
-  'rsyslog_early_lines' => [],
+  'rsyslog_early_lines' => [
+    # Set the default permissions for all log files.
+    '$FileOwner root',
+    '$FileGroup root',
+    '$FileCreateMode 0644',
+    '$DirCreateMode 0755',
+    '$Umask 0002',
+  ],
   'rsyslog_late_lines' => [],
   'rsyslog_additional_sockets' => [],
   'rsyslog_facilities_sent_to_remote' => [],
@@ -83,5 +96,7 @@
   'rsyslog_report_suspension' => false,
   'rsyslog_stats_logging' => false,
   'rsyslog_use_omprog_force' => false,
+  'rsyslog_omprog_binary_args' => [],
   'sysconfig' => sysconfig,
+  '_enable_syslog_socket_override' => true,
 }

cookbooks/fb_syslog/metadata.rb

@@ -5,7 +5,6 @@
 license 'Apache-2.0'
 description 'Installs/Configures syslog'
 source_url 'https://github.com/facebook/chef-cookbooks/'
-long_description IO.read(File.join(File.dirname(__FILE__), 'README.md'))
 version '0.0.1'
 supports 'centos'
 supports 'mac_os_x'

cookbooks/fb_syslog/recipes/default.rb

@@ -37,8 +37,8 @@
   template sysconfig_path do
     not_if { node.systemd? }
     source 'rsyslog-sysconf.erb'
-    owner 'root'
-    group 'root'
+    owner node.root_user
+    group node.root_group
     mode '0644'
     notifies :restart, 'service[rsyslog]'
   end
@@ -55,26 +55,28 @@
     'files' => ['/var/log/rsyslog-stats.log'],
     'overrides' => {
       'missingok' => true,
-      'notifempty' => true,
     },
   }
   directory '/var/spool/rsyslog' do
-    owner 'root'
-    group 'root'
+    owner node.root_user
+    group node.root_group
     mode '0700'
   end
 end
 
 include_recipe 'fb_syslog::packages'
 
 template config_file do
-  owner 'root'
-  group 'root'
+  owner node.root_user
+  group node.root_group
   mode '0644'
   notifies :restart, "service[#{service_name}]"
 end
 
 service service_name do
   action :start
   subscribes :restart, 'package[rsyslog]'
+  # within vagrant, sometimes rsyslog fails to restart the first time
+  retries 5
+  retry_delay 5
 end

cookbooks/fb_syslog/recipes/enable.rb

@@ -1,7 +1,29 @@
+#
+# Cookbook Name:: fb_syslog
+# Recipe:: enable
+#
+# Copyright (c) 2016-present, Facebook, Inc.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 # this is almost identical to running 'systemctl enable rsyslog', except that it
 # has no run-time requirements and can be run while setting up a container.
-link '/etc/systemd/system/syslog.service' do
-  to '/usr/lib/systemd/system/rsyslog.service'
-  owner 'root'
-  group 'root'
+if node.systemd?
+  link '/etc/systemd/system/syslog.service' do
+    to '/lib/systemd/system/rsyslog.service'
+    owner 'root'
+    group 'root'
+    notifies :run, 'fb_systemd_reload[system instance]', :immediately
+  end
 end

cookbooks/fb_syslog/recipes/packages.rb

@@ -23,12 +23,16 @@
   action :upgrade
 end
 
-fb_systemd_override 'override' do
-  unit_name 'rsyslog.service'
-  content({
-            'Unit' => { 'Requires' => 'syslog.socket' },
-            'Install' => { 'Alias' => 'syslog.service' },
-          })
+# TODO(davide125): Document this
+if node.systemd?
+  fb_systemd_override 'override' do
+    only_if { node['fb_syslog']['_enable_syslog_socket_override'] }
+    unit_name 'rsyslog.service'
+    content({
+              'Unit' => { 'Requires' => 'syslog.socket' },
+              'Install' => { 'Alias' => 'syslog.service' },
+            })
+  end
 end
 
 include_recipe 'fb_syslog::enable'

cookbooks/fb_syslog/spec/default_spec.rb

@@ -0,0 +1,72 @@
+# vim: syntax=ruby:expandtab:shiftwidth=2:softtabstop=2:tabstop=2
+#
+# Copyright (c) 2016-present, Facebook, Inc.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+require './spec/spec_helper'
+
+recipe 'fb_syslog::default', :unsupported => [:mac_os_x] do |tc|
+  let(:chef_run) do
+    tc.chef_run do |node|
+      node.default['shard_seed'] = 12345
+    end
+  end
+
+  context 'render /etc/rsyslog.conf' do
+    def reset_attributes(node)
+      node.default['fb_syslog']['syslog-entries'] = {}
+      node.default['fb_syslog']['rsyslog_facilities_sent_to_remote'] = []
+      node.default['fb_syslog']['rsyslog_upstream'] = ''
+    end
+
+    it 'with empty attributes' do
+      chef_run.converge('fb_systemd::reload', described_recipe) do |node|
+        reset_attributes(node)
+      end
+
+      expect(chef_run).to render_file('/etc/rsyslog.conf').
+        with_content(tc.fixture('rsyslog.conf_empty'))
+    end
+
+    it 'with syslog entries' do
+      chef_run.converge('fb_systemd::reload', described_recipe) do |node|
+        reset_attributes(node)
+        node.default['fb_syslog']['syslog-entries'] = {
+          'test' => {
+            'comment' => 'this is a test entry',
+            'selector' => 'local1.info',
+            'action' => '-/var/log/test.log',
+          },
+        }
+      end
+
+      expect(chef_run).to render_file('/etc/rsyslog.conf').
+        with_content(tc.fixture('rsyslog.conf'))
+    end
+
+    it 'with custom facilities' do
+      chef_run.converge('fb_systemd::reload', described_recipe) do |node|
+        reset_attributes(node)
+        node.default['fb_syslog'][
+          'rsyslog_facilities_sent_to_remote'] << 'kern.*'
+        node.default['fb_syslog'][
+          'rsyslog_upstream'] = 'syslog.vip.facebook.com'
+      end
+
+      expect(chef_run).to render_file('/etc/rsyslog.conf').
+        with_content(tc.fixture('rsyslog-kern.conf'))
+    end
+  end
+end

cookbooks/fb_syslog/spec/fixtures/centos6/rsyslog-kern.conf

@@ -0,0 +1,45 @@
+# rsyslog v5 configuration file
+
+# This rsyslog.conf was generated by Chef. To make changes, see
+# fb_syslog/README.md
+
+# For more information see /usr/share/doc/rsyslog-*/rsyslog_conf.html
+# If you experience problems, see http://www.rsyslog.com/doc/troubleshoot.html
+
+# Use FQDN for hostname field
+$PreserveFQDN on
+
+#### MODULES ####
+# Provides support for local system logging (e.g. via logger command)
+$ModLoad imuxsock
+# Provides kernel logging support (previously done by rklogd)
+$ModLoad imklog
+
+#### GLOBAL DIRECTIVES ####
+
+# Use default timestamp format
+$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat
+
+# File syncing capability is disabled by default. This feature is usually not required,
+# not useful and an extreme performance hit
+#$ActionFileEnableSync on
+
+$FileOwner root
+$FileGroup root
+$FileCreateMode 0644
+$DirCreateMode 0755
+$Umask 0002
+
+
+
+# Remote Logging
+kern.* {
+  action(type="omfwd"
+    name="Remote_Logging"
+    target="syslog.vip.facebook.com"
+    port="514"
+    protocol="tcp"
+    action.reportSuspension="off"
+    action.reportSuspensionContinuation="off"
+  )
+}

cookbooks/fb_syslog/spec/fixtures/centos6/rsyslog.conf

@@ -0,0 +1,35 @@
+# rsyslog v5 configuration file
+
+# This rsyslog.conf was generated by Chef. To make changes, see
+# fb_syslog/README.md
+
+# For more information see /usr/share/doc/rsyslog-*/rsyslog_conf.html
+# If you experience problems, see http://www.rsyslog.com/doc/troubleshoot.html
+
+# Use FQDN for hostname field
+$PreserveFQDN on
+
+#### MODULES ####
+# Provides support for local system logging (e.g. via logger command)
+$ModLoad imuxsock
+# Provides kernel logging support (previously done by rklogd)
+$ModLoad imklog
+
+#### GLOBAL DIRECTIVES ####
+
+# Use default timestamp format
+$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat
+
+# File syncing capability is disabled by default. This feature is usually not required,
+# not useful and an extreme performance hit
+#$ActionFileEnableSync on
+
+$FileOwner root
+$FileGroup root
+$FileCreateMode 0644
+$DirCreateMode 0755
+$Umask 0002
+
+
+# this is a test entry
+local1.info  -/var/log/test.log