Merge pull request #2885 from zendesk/sathish/BRE-865

Datadog APM

.env.example

@@ -78,6 +78,7 @@ PERIODICAL=stop_expired_deploys:60,remove_expired_locks:60,report_system_stats:6
 ## StatsD reporting
 # STATSD_HOST=192.168.1.1
 # STATSD_PORT=8125
+# DATADOG_TRACER=1 # optional, enable datadog APM tracer
 
 # PROJECT_CREATED_NOTIFY_ADDRESS=bobby-the-security-auditor@yourcompany.com
 # PROJECT_DELETED_NOTIFY_ADDRESS=bobby-the-security-auditor@yourcompany.com  # if not set uses PROJECT_CREATED_NOTIFY_ADDRESS

.gitignore

@@ -45,4 +45,4 @@
 
 # .bak files
 *.bak
-/vendor/cache/**/
+/vendor/cache

Gemfile.lock

@@ -21,6 +21,12 @@ PATH
     samson_aws_ecr (0.0.0)
       aws-sdk-ecr
 
+PATH
+  remote: plugins/datadog_tracer
+  specs:
+    samson_datadog_tracer (0.0.1)
+      ddtrace
+
 PATH
   remote: plugins/datadog
   specs:
@@ -230,6 +236,8 @@ GEM
       safe_yaml (~> 1.0.0)
     crass (1.0.4)
     dalli (2.7.6)
+    ddtrace (0.14.2)
+      msgpack
     diffy (3.2.0)
     docker-api (1.33.6)
       excon (>= 0.38.0)
@@ -611,6 +619,7 @@ DEPENDENCIES
   samson_assertible!
   samson_aws_ecr!
   samson_datadog!
+  samson_datadog_tracer!
   samson_deploy_env_vars!
   samson_docker_binary_builder!
   samson_env!

app/models/deploy_service.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 class DeployService
-  include ::NewRelic::Agent::MethodTracer
+  include ::Samson::PerformanceTracer
   attr_reader :user
 
   def initialize(user)
@@ -99,7 +99,7 @@ def send_before_notifications(deploy)
       DeployMailer.bypass_email(deploy, user).deliver_now
     end
   end
-  add_method_tracer :send_before_notifications
+  add_tracer :send_before_notifications
 
   def send_after_notifications(deploy)
     Samson::Hooks.fire(:after_deploy, deploy, deploy.buddy)
@@ -108,7 +108,7 @@ def send_after_notifications(deploy)
     execute_and_log_errors(deploy) { send_failed_deploy_email(deploy) }
     execute_and_log_errors(deploy) { notify_outbound_webhooks(deploy) }
   end
-  add_method_tracer :send_after_notifications
+  add_tracer :send_after_notifications
 
   # basically does the same as the hooks would do
   def execute_and_log_errors(deploy, &block)

app/models/docker_builder_service.rb

@@ -2,7 +2,7 @@
 require 'docker'
 
 class DockerBuilderService
-  include ::NewRelic::Agent::MethodTracer
+  include ::Samson::PerformanceTracer
 
   def initialize(build)
     @build = build
@@ -61,7 +61,7 @@ def before_docker_build(tmp_dir)
     Samson::Hooks.fire(:before_docker_build, tmp_dir, @build, @output)
     execute_build_command(tmp_dir, @build.project.build_command)
   end
-  add_method_tracer :before_docker_build
+  add_tracer :before_docker_build
 
   def build_image(tmp_dir, tag_as_latest:)
     File.write("#{tmp_dir}/REVISION", @build.git_sha)
@@ -83,5 +83,5 @@ def build_image(tmp_dir, tag_as_latest:)
       tmp_dir, @build, @execution.executor, tag_as_latest: tag_as_latest, cache_from: cache
     )
   end
-  add_method_tracer :build_image
+  add_tracer :build_image
 end

app/models/git_repository.rb

@@ -2,7 +2,7 @@
 # Responsible for all git knowledge of a repo
 # Caches a local mirror (not a full checkout) and creates a workspace when deploying
 class GitRepository
-  include ::NewRelic::Agent::MethodTracer
+  include ::Samson::PerformanceTracer
 
   attr_accessor :executor # others set this to listen in on commands being executed
 
@@ -65,7 +65,7 @@ def branches
   def clean!
     FileUtils.rm_rf(repo_cache_dir)
   end
-  add_method_tracer :clean!
+  add_tracer :clean!
 
   def valid_url?
     return false if repository_url.blank?
@@ -127,17 +127,17 @@ def outside_caller
   def clone!
     executor.execute "git -c core.askpass=true clone --mirror #{repository_url} #{repo_cache_dir}"
   end
-  add_method_tracer :clone!
+  add_tracer :clone!
 
   def create_workspace(temp_dir)
     executor.execute "git clone #{repo_cache_dir} #{temp_dir}"
   end
-  add_method_tracer :create_workspace
+  add_tracer :create_workspace!
 
   def update!
     executor.execute("cd #{repo_cache_dir}", 'git fetch -p')
   end
-  add_method_tracer :update!
+  add_tracer :update!
 
   def sha_exist?(sha)
     !!capture_stdout("git", "cat-file", "-t", sha)

app/models/image_builder.rb

@@ -6,7 +6,7 @@ class ImageBuilder
   class << self
     DIGEST_SHA_REGEX = /Digest:.*(sha256:[0-9a-f]{64})/i
 
-    include ::NewRelic::Agent::MethodTracer
+    include ::Samson::PerformanceTracer
 
     def build_image(dir, build, executor, tag_as_latest:, **args)
       if DockerRegistry.all.empty?
@@ -86,7 +86,7 @@ def push_image(image_id, build, executor, tag_as_latest:)
       executor.output.puts("Docker push failed: #{e.message}\n")
       nil
     end
-    add_method_tracer :push_image
+    add_tracer :push_image
 
     def push_image_to_registries(image_id, build, executor, tag:, override_tag:)
       digest = nil

app/models/job_execution.rb

@@ -2,7 +2,7 @@
 require 'shellwords'
 
 class JobExecution
-  include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
+  include ::Samson::PerformanceTracer
 
   cattr_accessor(:cancel_timeout, instance_writer: false) { 15.seconds }
 
@@ -140,7 +140,7 @@ def run
   ensure
     finish unless @cancelled
   end
-  add_transaction_tracer :run,
+  add_asynchronous_tracer :run,
     category: :task,
     params: '{ job_id: id, project: job.project.try(:name), reference: reference }'
 

app/models/multi_lock.rb

@@ -4,7 +4,7 @@ class MultiLock
   cattr_accessor(:locks) { {} }
 
   class << self
-    include ::NewRelic::Agent::MethodTracer
+    include ::Samson::PerformanceTracer
 
     def lock(id, holder, options)
       locked = wait_for_lock(id, holder, options)
@@ -25,7 +25,7 @@ def wait_for_lock(id, holder, options)
       end
       false
     end
-    add_method_tracer :wait_for_lock
+    add_tracer :wait_for_lock
 
     def try_lock(id, holder)
       mutex.synchronize do

docs/plugins.md

@@ -12,6 +12,7 @@ Available plugins:
  - [Airbrake error handling](https://github.com/zendesk/samson/tree/master/plugins/airbrake)
  - [AWS ECR credential refresher](https://github.com/zendesk/samson/tree/master/plugins/aws_ecr)
  - [Datadog monitoring and deploy tracking](https://github.com/zendesk/samson/tree/master/plugins/datadog)
+ - [Datadog APM tracer](https://github.com/zendesk/samson/tree/master/plugins/datadog_tracer)
  - [Deploy Environment Variables](https://github.com/zendesk/samson/tree/master/plugins/deploy_env_vars)
  - [Docker binary builder](https://github.com/zendesk/samson/tree/master/plugins/docker_binary_builder)
  - [ENV var management](https://github.com/zendesk/samson/tree/master/plugins/env)

lib/samson/hooks.rb

@@ -57,7 +57,9 @@ class UserError < StandardError
       :ref_status,
       :release_deploy_conditions,
       :stage_clone,
-      :stage_permitted_params
+      :stage_permitted_params,
+      :performance_tracer,
+      :asynchronous_performance_tracer
     ].freeze
 
     KNOWN = VIEW_HOOKS + EVENT_HOOKS
@@ -164,7 +166,7 @@ def only_callbacks_for_plugin(plugin_name, hook_name)
 
       # use
       def fire(name, *args)
-        NewRelic::Agent::MethodTracerHelpers.trace_execution_scoped("Custom/Hooks/#{name}") do
+        Samson::PerformanceTracer.trace_execution_scoped("Custom/Hooks/#{name}") do
           hooks(name).map { |hook| hook.call(*args) }
         end
       end

lib/samson/performance_tracer.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+module Samson
+  module PerformanceTracer
+    # It's used to trace the hook fire event,
+    # We can't use the samson hook to fetch the available plugins.
+    TRACER_PLUGINS = ['SamsonNewRelic', 'SamsonDatadogTracer::APM'].freeze
+
+    class << self
+      def included(clazz)
+        clazz.extend ClassMethods
+      end
+
+      def trace_execution_scoped(scope_name)
+        # Tracing the scope is restricted to avoid into slow startup
+        # Refer Samson::BootCheck
+        if ['staging', 'production'].include?(Rails.env)
+          plugins = TRACER_PLUGINS.map(&:safe_constantize).compact
+          execution = using_plugins plugins, scope_name do
+            yield
+          end
+          execution.call
+        else
+          yield
+        end
+      end
+
+      def using_plugins(plugins, scope_name, &block)
+        plugins.inject(block) { |inner, plugin| plugin.trace_method_execution_scope(scope_name) { inner } }
+      end
+    end
+
+    # for Newrelic and Datadog -> for tracer plugins.
+    module ClassMethods
+      def add_tracer(method)
+        Samson::Hooks.fire(:performance_tracer, self, method)
+      end
+
+      # TODO: Add asynchronous tracer for Datadog.
+      def add_asynchronous_tracer(method, options)
+        Samson::Hooks.fire(:asynchronous_performance_tracer, self, method, options)
+      end
+    end
+  end
+end

plugins/datadog_tracer/README.md

@@ -0,0 +1,3 @@
+# Datadog Plugin
+
+Plugin that trace requests and notify to Datadog APM

plugins/datadog_tracer/config/initializers/datadog_tracer.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+if SamsonDatadogTracer.enabled?
+
+  SamsonDatadogTracer::IGNORED_URLS = Set[
+    '/ping',
+    '/cable',
+  ].freeze
+
+  require 'ddtrace'
+  Datadog.configure do |c|
+    # Tracer
+    c.tracer(
+      hostname:                ENV['STATSD_HOST'] || '127.0.0.1',
+      tags: {
+        env:                   ENV['RAILS_ENV'],
+        'rails.version':       Rails.version,
+        'ruby.version':        RUBY_VERSION
+      }
+    )
+
+    c.use :rails,
+      service_name: 'samson',
+      controller_service: 'samson-rails-controller',
+      cache_service: 'samson-cache',
+      database_service: 'samson-mysql',
+      distributed_tracing: true
+
+    c.use :faraday, service_name: 'samson-faraday'
+    c.use :dalli, service_name: 'samson-dalli'
+
+    require 'aws-sdk-ecr'
+    c.use :aws, service_name: 'samson-aws'
+  end
+
+  # Span Filters
+  # Filter out the health checks, version checks, and diagnostics
+  uninteresting_controller_filter = Datadog::Pipeline::SpanFilter.new do |span|
+    span.name == 'rack.request' &&
+    SamsonDatadogTracer::IGNORED_URLS.any? { |path| span.get_tag('http.url').include?(path) }
+  end
+
+  Datadog::Pipeline.before_flush(uninteresting_controller_filter)
+end