Merge pull request #223 from Athishpranav2003/add-gzip-support

Add gzip support

README.md

@@ -63,6 +63,7 @@ More configuration parameters:
 - `port`: listen port (default: 24231)
 - `metrics_path`: metrics HTTP endpoint (default: /metrics)
 - `aggregated_metrics_path`: metrics HTTP endpoint (default: /aggregated_metrics)
+- `content_encoding`: encoding format for the exposed metrics (default: identity). Supported formats are {identity, gzip}
 
 When using multiple workers, each worker binds to port + `fluent_worker_id`.
 To scrape metrics from all workers at once, you can access http://localhost:24231/aggregated_metrics.

lib/fluent/plugin/in_prometheus.rb

@@ -3,6 +3,7 @@
 require 'fluent/plugin/prometheus_metrics'
 require 'net/http'
 require 'openssl'
+require 'zlib'
 
 module Fluent::Plugin
   class PrometheusInput < Fluent::Plugin::Input
@@ -32,6 +33,9 @@ class PrometheusInput < Fluent::Plugin::Input
       config_param :extra_conf, :hash, default: nil, symbolize_keys: true, deprecated: 'See http helper config'
     end
 
+    desc 'Content encoding of the exposed metrics, Currently supported encoding is identity, gzip. Ref: https://prometheus.io/docs/instrumenting/exposition_formats/#basic-info'
+    config_param :content_encoding, :enum, list: [:identity, :gzip], default: :identity
+
     def initialize
       super
       @registry = ::Prometheus::Client.registry
@@ -184,7 +188,7 @@ def start_webrick
     end
 
     def all_metrics
-      [200, { 'Content-Type' => ::Prometheus::Client::Formats::Text::CONTENT_TYPE }, ::Prometheus::Client::Formats::Text.marshal(@registry)]
+      response(::Prometheus::Client::Formats::Text.marshal(@registry))
     rescue => e
       [500, { 'Content-Type' => 'text/plain' }, e.to_s]
     end
@@ -197,8 +201,7 @@ def all_workers_metrics
           full_result.add_metrics(resp.body)
         end
       end
-
-      [200, { 'Content-Type' => ::Prometheus::Client::Formats::Text::CONTENT_TYPE }, full_result.get_metrics]
+      response(full_result.get_metrics)
     rescue => e
       [500, { 'Content-Type' => 'text/plain' }, e.to_s]
     end
@@ -226,5 +229,18 @@ def do_request(host:, port:, secure:)
         yield(http)
       end
     end
+
+    def response(metrics)
+      body = nil
+      case @content_encoding
+      when :gzip
+        gzip = Zlib::GzipWriter.new(StringIO.new)
+        gzip << metrics
+        body = gzip.close.string
+      when :identity
+        body = metrics
+      end
+      [200, { 'Content-Type' => ::Prometheus::Client::Formats::Text::CONTENT_TYPE, 'Content-Encoding' => @content_encoding.to_s }, body]
+    end
   end
 end

spec/fluent/plugin/in_prometheus_spec.rb

@@ -3,6 +3,7 @@
 require 'fluent/test/driver/input'
 
 require 'net/http'
+require 'zlib'
 
 describe Fluent::Plugin::PrometheusInput do
   CONFIG = %[
@@ -45,6 +46,24 @@
         expect(driver.instance.metrics_path).to eq('/_test')
       end
     end
+
+    describe 'content_encoding_identity' do
+      let(:config) { CONFIG + %[
+    content_encoding identity
+] }
+      it 'should be configurable' do
+        expect(driver.instance.content_encoding).to eq(:identity)
+      end
+    end
+
+    describe 'content_encoding_gzip' do
+      let(:config) { CONFIG + %[
+    content_encoding gzip
+] }
+      it 'should be configurable' do
+        expect(driver.instance.content_encoding).to eq(:gzip)
+      end
+    end
   end
 
   describe '#start' do
@@ -197,6 +216,43 @@
         end
       end
     end
+
+    context 'response content_encoding identity' do
+      let(:config) { LOCAL_CONFIG + %[
+        content_encoding identity
+  ] }
+      it 'exposes metric' do
+        driver.run(timeout: 1) do
+          registry = driver.instance.instance_variable_get(:@registry)
+          registry.counter(:test,docstring: "Testing metrics") unless registry.exist?(:test)
+          Net::HTTP.start("127.0.0.1", port) do |http|
+            req = Net::HTTP::Get.new("/metrics")
+            req['accept-encoding'] = nil
+            res = http.request(req)
+            expect(res.body).to include("test Testing metrics")
+          end
+        end
+      end
+    end
+
+    context 'response content_encoding gzip' do
+      let(:config) { LOCAL_CONFIG + %[
+        content_encoding gzip
+  ] }
+      it 'exposes metric' do
+        driver.run(timeout: 1) do
+          registry = driver.instance.instance_variable_get(:@registry)
+          registry.counter(:test,docstring: "Testing metrics") unless registry.exist?(:test)
+          Net::HTTP.start("127.0.0.1", port) do |http|
+            req = Net::HTTP::Get.new("/metrics")
+            req['accept-encoding'] = nil
+            res = http.request(req)
+            gzip = Zlib::GzipReader.new(StringIO.new(res.body.to_s))
+            expect(gzip.read).to include("test Testing metrics")
+          end
+        end
+      end
+    end
   end
 
   describe '#run_multi_workers' do