Merge pull request #132 from airbnb/custom_watchers

Allow pluggable watchers

README.md

@@ -342,29 +342,5 @@ Non-HTTP backends such as MySQL or RabbitMQ will obviously continue to need thei
 
 ### Creating a Service Watcher ###
 
-If you'd like to create a new service watcher:
-
-1. Create a file for your watcher in `service_watcher` dir
-2. Use the following template:
-```ruby
-require 'synapse/service_watcher/base'
-
-module Synapse
-  class NewWatcher < BaseWatcher
-    def start
-      # write code which begins running service discovery
-    end
-
-    private
-    def validate_discovery_opts
-      # here, validate any required options in @discovery
-    end
-  end
-end
-```
-
-3. Implement the `start` and `validate_discovery_opts` methods
-4. Implement whatever additional methods your discovery requires
-
-When your watcher detects a list of new backends, you should call `set_backends` to
-store the new backends and update the HAProxy config.
+See the Service Watcher [README](lib/synapse/service_watcher/README.md) for
+how to add new Service Watchers.

lib/synapse.rb

@@ -1,18 +1,18 @@
+require 'logger'
+require 'json'
+
 require "synapse/version"
-require "synapse/service_watcher/base"
+require "synapse/log"
 require "synapse/haproxy"
 require "synapse/file_output"
 require "synapse/service_watcher"
-require "synapse/log"
 
-require 'logger'
-require 'json'
-
-include Synapse
 
 module Synapse
   class Synapse
+
     include Logging
+
     def initialize(opts={})
       # create the service watchers for all our services
       raise "specify a list of services to connect in the config" unless opts.has_key?('services')

lib/synapse/file_output.rb

@@ -1,4 +1,3 @@
-require 'synapse/log'
 require 'fileutils'
 require 'tempfile'
 

lib/synapse/haproxy.rb

@@ -1,6 +1,5 @@
 require 'fileutils'
 require 'json'
-require 'synapse/log'
 require 'socket'
 
 module Synapse

lib/synapse/service_watcher.rb

@@ -1,22 +1,8 @@
+require "synapse/log"
 require "synapse/service_watcher/base"
-require "synapse/service_watcher/zookeeper"
-require "synapse/service_watcher/ec2tag"
-require "synapse/service_watcher/dns"
-require "synapse/service_watcher/docker"
-require "synapse/service_watcher/zookeeper_dns"
 
 module Synapse
   class ServiceWatcher
-
-    @watchers = {
-      'base' => BaseWatcher,
-      'zookeeper' => ZookeeperWatcher,
-      'ec2tag' => EC2Watcher,
-      'dns' => DnsWatcher,
-      'docker' => DockerWatcher,
-      'zookeeper_dns' => ZookeeperDnsWatcher,
-    }
-
     # the method which actually dispatches watcher creation requests
     def self.create(name, opts, synapse)
       opts['name'] = name
@@ -25,10 +11,16 @@ def self.create(name, opts, synapse)
         unless opts.has_key?('discovery') && opts['discovery'].has_key?('method')
 
       discovery_method = opts['discovery']['method']
-      raise ArgumentError, "Invalid discovery method #{discovery_method}" \
-        unless @watchers.has_key?(discovery_method)
-
-      return @watchers[discovery_method].new(opts, synapse)
+      watcher = begin
+        method = discovery_method.downcase
+        require "synapse/service_watcher/#{method}"
+        # zookeeper_dns => ZookeeperDnsWatcher, ec2tag => Ec2tagWatcher, etc ...
+        method_class  = method.split('_').map{|x| x.capitalize}.join.concat('Watcher')
+        self.const_get("#{method_class}")
+      rescue Exception => e
+        raise ArgumentError, "Specified a discovery method of #{discovery_method}, which could not be found: #{e}"
+      end
+      return watcher.new(opts, synapse)
     end
   end
 end

lib/synapse/service_watcher/README.md

@@ -0,0 +1,74 @@
+## Watcher Classes
+
+Watchers are the piece of Synapse that watch an external service registry
+and reflect those changes in the local HAProxy state. Watchers should look
+like:
+
+```ruby
+require "synapse/service\_watcher/base"
+
+module Synapse::ServiceWatcher
+  class MyWatcher < BaseWatcher
+    def start
+      # write code which begins running service discovery
+    end
+
+    def stop
+      # write code which tears down the service discovery
+    end
+
+    def ping?
+      # write code to check in on the health of the watcher
+    end
+
+    private
+    def validate_discovery_opts
+      # here, validate any required options in @discovery
+    end
+
+    ...
+
+  end
+end
+```
+
+### Watcher Plugin Inteface
+Synapse deduces both the class path and class name from the `method` key within
+the watcher configuration.  Every watcher is passed configuration with the
+`method` key, e.g. `zookeeper` or `ec2tag`.
+
+#### Class Location
+Synapse expects to find your class at `synapse/service\_watcher/#{method}`. You
+must make your watcher available at that path, and Synapse can "just work" and
+find it.
+
+#### Class Name
+These method strings are then transformed into class names via the following
+function:
+
+```
+method_class  = method.split('_').map{|x| x.capitalize}.join.concat('Watcher')
+```
+
+This has the effect of taking the method, splitting on '_', capitalizing each
+part and recombining with an added 'Watcher' on the end. So `zookeeper\_dns`
+becomes `ZookeeperDnsWatcher`, and `zookeeper` becomes `Zookeeper`. Make sure
+your class name is correct.
+
+### Watcher Class Interface
+ServiceWatchers should conform to the interface provided by `BaseWatcher`:
+
+```
+start: start the watcher on a service registry
+
+stop: stop the watcher on a service registry
+
+ping?: healthcheck the watcher's connection to the service registry
+
+private
+validate_discovery_opts: check if the configuration has the right options
+```
+
+When your watcher has received an update from the service registry you should
+call `set\_backends(new\_backends)` to trigger a sync of your watcher state
+with local HAProxy state.

lib/synapse/service_watcher/base.rb

@@ -1,9 +1,9 @@
-require 'set'
 require 'synapse/log'
+require 'set'
 
-module Synapse
+class Synapse::ServiceWatcher
   class BaseWatcher
-    include Logging
+    include Synapse::Logging
 
     LEADER_WARN_INTERVAL = 30
 

lib/synapse/service_watcher/dns.rb

@@ -3,7 +3,7 @@
 require 'thread'
 require 'resolv'
 
-module Synapse
+class Synapse::ServiceWatcher
   class DnsWatcher < BaseWatcher
     def start
       @check_interval = @discovery['check_interval'] || 30.0

lib/synapse/service_watcher/docker.rb

@@ -1,7 +1,7 @@
 require "synapse/service_watcher/base"
 require 'docker'
 
-module Synapse
+class Synapse::ServiceWatcher
   class DockerWatcher < BaseWatcher
     def start
       @check_interval = @discovery['check_interval'] || 15.0

lib/synapse/service_watcher/ec2tag.rb

@@ -1,8 +1,8 @@
 require 'synapse/service_watcher/base'
 require 'aws-sdk'
 
-module Synapse
-  class EC2Watcher < BaseWatcher
+class Synapse::ServiceWatcher
+  class Ec2tagWatcher < BaseWatcher
 
     attr_reader :check_interval
 

lib/synapse/service_watcher/zookeeper.rb

@@ -3,7 +3,7 @@
 require 'thread'
 require 'zk'
 
-module Synapse
+class Synapse::ServiceWatcher
   class ZookeeperWatcher < BaseWatcher
     NUMBERS_RE = /^\d+$/
 

lib/synapse/service_watcher/zookeeper_dns.rb

@@ -19,7 +19,7 @@
 # for messages indicating that new servers are available, the check interval
 # has passed (triggering a re-resolve), or that the watcher should shut down.
 # The DNS watcher is responsible for the actual reconfiguring of backends.
-module Synapse
+class Synapse::ServiceWatcher
   class ZookeeperDnsWatcher < BaseWatcher
 
     # Valid messages that can be passed through the internal message queue
@@ -46,7 +46,7 @@ class StopWatcher; end
       CHECK_INTERVAL_MESSAGE = CheckInterval.new
     end
 
-    class Dns < Synapse::DnsWatcher
+    class Dns < Synapse::ServiceWatcher::DnsWatcher
 
       # Overrides the discovery_servers method on the parent class
       attr_accessor :discovery_servers
@@ -106,7 +106,7 @@ def validate_discovery_opts
       end
     end
 
-    class Zookeeper < Synapse::ZookeeperWatcher
+    class Zookeeper < Synapse::ServiceWatcher::ZookeeperWatcher
       def initialize(opts={}, synapse, message_queue)
         super(opts, synapse)
 

spec/lib/synapse/service_watcher_base_spec.rb

@@ -1,12 +1,12 @@
 require 'spec_helper'
 
-class Synapse::BaseWatcher
+class Synapse::ServiceWatcher::BaseWatcher
   attr_reader :should_exit, :default_servers
 end
 
-describe Synapse::BaseWatcher do
+describe Synapse::ServiceWatcher::BaseWatcher do
   let(:mocksynapse) { double() }
-  subject { Synapse::BaseWatcher.new(args, mocksynapse) }
+  subject { Synapse::ServiceWatcher::BaseWatcher.new(args, mocksynapse) }
   let(:testargs) { { 'name' => 'foo', 'discovery' => { 'method' => 'base' }, 'haproxy' => {} }}
 
   def remove_arg(name)

spec/lib/synapse/service_watcher_docker_spec.rb

@@ -1,13 +1,14 @@
 require 'spec_helper'
+require 'synapse/service_watcher/docker'
 
-class Synapse::DockerWatcher
+class Synapse::ServiceWatcher::DockerWatcher
   attr_reader :check_interval, :watcher, :synapse
   attr_accessor :default_servers
 end
 
-describe Synapse::DockerWatcher do
+describe Synapse::ServiceWatcher::DockerWatcher do
   let(:mocksynapse) { double() }
-  subject { Synapse::DockerWatcher.new(testargs, mocksynapse) }
+  subject { Synapse::ServiceWatcher::DockerWatcher.new(testargs, mocksynapse) }
   let(:testargs) { { 'name' => 'foo', 'discovery' => { 'method' => 'docker', 'servers' => [{'host' => 'server1.local', 'name' => 'mainserver'}], 'image_name' => 'mycool/image', 'container_port' => 6379 }, 'haproxy' => {} }}
   before(:each) do
     allow(subject.log).to receive(:warn)

spec/lib/synapse/service_watcher_ec2tags_spec.rb

@@ -1,7 +1,8 @@
 require 'spec_helper'
+require 'synapse/service_watcher/ec2tag'
 require 'logging'
 
-class Synapse::EC2Watcher
+class Synapse::ServiceWatcher::Ec2tagWatcher
   attr_reader   :synapse
   attr_accessor :default_servers, :ec2
 end
@@ -28,9 +29,9 @@ def fake_address
   end
 end
 
-describe Synapse::EC2Watcher do
+describe Synapse::ServiceWatcher::Ec2tagWatcher do
   let(:mock_synapse) { double }
-  subject { Synapse::EC2Watcher.new(basic_config, mock_synapse) }
+  subject { Synapse::ServiceWatcher::Ec2tagWatcher.new(basic_config, mock_synapse) }
 
   let(:basic_config) do
     { 'name' => 'ec2tagtest',
@@ -88,30 +89,30 @@ def munge_haproxy_arg(name, new_value)
     context 'when missing arguments' do
       it 'complains if aws_region is missing' do
         expect {
-          Synapse::EC2Watcher.new(remove_discovery_arg('aws_region'), mock_synapse)
+          Synapse::ServiceWatcher::Ec2tagWatcher.new(remove_discovery_arg('aws_region'), mock_synapse)
         }.to raise_error(ArgumentError, /Missing aws_region/)
       end
       it 'complains if aws_access_key_id is missing' do
         expect {
-          Synapse::EC2Watcher.new(remove_discovery_arg('aws_access_key_id'), mock_synapse)
+          Synapse::ServiceWatcher::Ec2tagWatcher.new(remove_discovery_arg('aws_access_key_id'), mock_synapse)
         }.to raise_error(ArgumentError, /Missing aws_access_key_id/)
       end
       it 'complains if aws_secret_access_key is missing' do
         expect {
-          Synapse::EC2Watcher.new(remove_discovery_arg('aws_secret_access_key'), mock_synapse)
+          Synapse::ServiceWatcher::Ec2tagWatcher.new(remove_discovery_arg('aws_secret_access_key'), mock_synapse)
         }.to raise_error(ArgumentError, /Missing aws_secret_access_key/)
       end
       it 'complains if server_port_override is missing' do
         expect {
-          Synapse::EC2Watcher.new(remove_haproxy_arg('server_port_override'), mock_synapse)
+          Synapse::ServiceWatcher::Ec2tagWatcher.new(remove_haproxy_arg('server_port_override'), mock_synapse)
         }.to raise_error(ArgumentError, /Missing server_port_override/)
       end
     end
 
     context 'invalid data' do
       it 'complains if the haproxy server_port_override is not a number' do
           expect {
-            Synapse::EC2Watcher.new(munge_haproxy_arg('server_port_override', '80deadbeef'), mock_synapse)
+            Synapse::ServiceWatcher::Ec2tagWatcher.new(munge_haproxy_arg('server_port_override', '80deadbeef'), mock_synapse)
           }.to raise_error(ArgumentError, /Invalid server_port_override/)
       end
     end