Skip to content

Commit

Permalink
Http compression support (#335)
Browse files Browse the repository at this point in the history
* feat: gzip compression

* fix brotli content-encoding header

* fix brotli encoding-header test

* fix h2 tests
  • Loading branch information
jrauschenbusch authored Jan 12, 2024
1 parent bdcdec0 commit 6b70df1
Show file tree
Hide file tree
Showing 12 changed files with 196 additions and 48 deletions.
3 changes: 2 additions & 1 deletion cmd/flags/grpc_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,10 @@
package flags

import (
"testing"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"testing"
)

func TestGrpc_ToGrpcRequests(t *testing.T) {
Expand Down
10 changes: 6 additions & 4 deletions cmd/flags/http.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,8 @@ var allowedHTTPMethods = map[string]interface{}{

// HTTP stores flags related to HTTP requests.
type HTTP struct {
Requests stringArray
Requests stringArray
Compression string
}

func (h *HTTP) String() string {
Expand All @@ -43,16 +44,17 @@ func (h *HTTP) String() string {

func (h *HTTP) initFlags() {
flag.Var(&h.Requests, "http-requests", `HTTP request to be sent. Request is in '<http-method>:<path>[:body]' format. E.g. post:/ping:{"key":"value"}`)
flag.StringVar(&h.Compression, "http-requests-compression", "", "Compression is disabled by default. Allows compression of Http body either with `gzip`, `deflate` or `brotli`. Using one of the compression algorithms also the according `Content-Encoding` header is added.")
}

func (h *HTTP) getWarmupHTTPRequests() ([]http.Request, error) {
return toHTTPRequests(h.Requests)
return toHTTPRequests(h.Requests, http.CompressionType(h.Compression))
}

func toHTTPRequests(requestsFlag []string) ([]http.Request, error) {
func toHTTPRequests(requestsFlag []string, compression http.CompressionType) ([]http.Request, error) {
var requests []http.Request
for _, requestFlag := range requestsFlag {
request, err := http.ToHTTPRequest(requestFlag)
request, err := http.ToHTTPRequest(requestFlag, compression)
if err != nil {
return nil, err
}
Expand Down
13 changes: 7 additions & 6 deletions cmd/flags/http_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,11 @@
package flags

import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"mittens/internal/pkg/http"
"testing"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

func TestHttp_ToHttpRequests(t *testing.T) {
Expand All @@ -27,7 +28,7 @@ func TestHttp_ToHttpRequests(t *testing.T) {
"get:/ping",
}

requests, err := toHTTPRequests(requestFlags)
requests, err := toHTTPRequests(requestFlags, http.COMPRESSION_NONE)
require.NoError(t, err)

require.Equal(t, 2, len(requests))
Expand All @@ -41,7 +42,7 @@ func TestHttp_ToHttpRequestsInvalidFormat(t *testing.T) {
"get/health",
}

requests, err := toHTTPRequests(requestFlags)
requests, err := toHTTPRequests(requestFlags, http.COMPRESSION_NONE)

var expected []http.Request
require.Error(t, err)
Expand All @@ -55,7 +56,7 @@ func TestHttp_ToHttpRequestsInvalidMethod(t *testing.T) {
"invalidMethod:/health",
}

requests, err := toHTTPRequests(requestFlags)
requests, err := toHTTPRequests(requestFlags, http.COMPRESSION_NONE)

var expected []http.Request
require.Error(t, err)
Expand All @@ -69,7 +70,7 @@ func TestHttp_ToHttpRequestsInvalidBody(t *testing.T) {
"get:/test:file:test",
}

requests, err := toHTTPRequests(requestFlags)
requests, err := toHTTPRequests(requestFlags, http.COMPRESSION_NONE)

var expected []http.Request
require.Error(t, err)
Expand Down
1 change: 1 addition & 0 deletions docs/about/getting-started.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ The application receives a number of command-line flags including the requests t
| -http-headers | strings | N/A | Http headers to be sent with warm up requests. To send multiple headers define this flag for each header |
| -grpc-requests | strings | N/A | gRPC requests to be sent. Request is in '\<service\>\<method\>\[:message\]' format. E.g. health/ping:{"key": "value"}. To send multiple requests, simply repeat this flag for each request. Use the notation `:file/xyz.json` if you want to use an external file for the request body. |
| -http-requests | string | N/A | Http request to be sent. Request is in `<http-method>:<path>[:body]` format. E.g. `post:/ping:{"key": "value"}`. To send multiple requests, simply repeat this flag for each request. Use the notation `:file/xyz.json` if you want to use an external file for the request body. |
| -http-requests-compression | string | N/A | Compression is disabled by default. Allows compression of Http body either with `gzip`, `deflate` or `brotli`. Using one of the compression algorithms also the according `Content-Encoding` header is added. |
| -fail-readiness | bool | false | If set to true readiness will fail if the target did not became ready in time |
| -file-probe-enabled | bool | true | If set to true writes files that can be used as readiness/liveness probes. a file with the name `alive` is created when Mittens starts and a file named `ready` is created when the warmup completes |
| -file-probe-liveness-path | string | alive | File to be used for liveness probe |
Expand Down
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
module mittens

require (
github.com/andybalholm/brotli v1.0.6
github.com/fullstorydev/grpcurl v1.8.9
github.com/golang/protobuf v1.5.3
github.com/jhump/protoreflect v1.15.4
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
github.com/andybalholm/brotli v1.0.6 h1:Yf9fFpf49Zrxb9NlQaluyE92/+X7UVHlhMNJN2sxfOI=
github.com/andybalholm/brotli v1.0.6/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
github.com/bufbuild/protocompile v0.7.1 h1:Kd8fb6EshOHXNNRtYAmLAwy/PotlyFoN0iMbuwGNh0M=
github.com/bufbuild/protocompile v0.7.1/go.mod h1:+Etjg4guZoAqzVk2czwEQP12yaxLJ8DxuqCJ9qHdH94=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
Expand Down
18 changes: 8 additions & 10 deletions internal/pkg/http/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,15 +15,13 @@
package http

import (
"bytes"
"crypto/tls"
"fmt"
"io"
"io/ioutil"
"log"
"mittens/internal/pkg/placeholders"
"mittens/internal/pkg/response"
"mittens/internal/pkg/util"
"net"
"net/http"
"strings"
Expand Down Expand Up @@ -75,12 +73,8 @@ func NewClient(host string, insecure bool, timeoutMilliseconds int, protocol Pro
}

// SendRequest sends a request to the HTTP server and wraps useful information into a Response object.
func (c Client) SendRequest(method, path string, headers []string, requestBody *string) response.Response {
func (c Client) SendRequest(method, path string, headers map[string]string, body io.Reader) response.Response {
const respType = "http"
var body io.Reader
if requestBody != nil {
body = bytes.NewBufferString(*requestBody)
}

url := fmt.Sprintf("%s/%s", c.host, strings.TrimLeft(path, "/"))
req, err := http.NewRequest(method, url, body)
Expand All @@ -90,8 +84,7 @@ func (c Client) SendRequest(method, path string, headers []string, requestBody *
return response.Response{Duration: time.Duration(0), Err: err, Type: respType}
}

headersMap := util.ToHeaders(headers)
for k, v := range headersMap {
for k, v := range headers {
if strings.EqualFold(k, "Host") {
req.Host = v
}
Expand All @@ -101,9 +94,14 @@ func (c Client) SendRequest(method, path string, headers []string, requestBody *

req.Header.Add(k, interpolatedHeaderValue)
}

if err != nil {
defer req.Body.Close()
}
startTime := time.Now()
resp, err := c.httpClient.Do(req)
if resp != nil {
defer resp.Body.Close()
}
endTime := time.Now()
if err != nil {
return response.Response{Duration: endTime.Sub(startTime), Err: err, Type: respType}
Expand Down
19 changes: 13 additions & 6 deletions internal/pkg/http/client_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (
"fmt"
"mittens/fixture"
"net/http"
"strings"
"testing"

"github.com/stretchr/testify/assert"
Expand All @@ -39,44 +40,50 @@ func TestMain(m *testing.M) {
func TestRequestSuccessHTTP1(t *testing.T) {
c := NewClient(serverUrl, false, 10000, HTTP1)
reqBody := ""
resp := c.SendRequest("GET", WorkingPath, []string{}, &reqBody)
reader := strings.NewReader(reqBody)
resp := c.SendRequest("GET", WorkingPath, make(map[string]string), reader)
assert.Nil(t, resp.Err)
}

func TestRequestSuccessH2C(t *testing.T) {
c := NewClient(serverUrl, false, 10000, H2C)
reqBody := ""
resp := c.SendRequest("GET", WorkingPath, []string{}, &reqBody)
reader := strings.NewReader(reqBody)
resp := c.SendRequest("GET", WorkingPath, make(map[string]string), reader)
assert.Nil(t, resp.Err)
}

func TestHttpErrorHTTP1(t *testing.T) {
c := NewClient(serverUrl, false, 10000, HTTP1)
reqBody := ""
resp := c.SendRequest("GET", "/", []string{}, &reqBody)
reader := strings.NewReader(reqBody)
resp := c.SendRequest("GET", "/", make(map[string]string), reader)
assert.Nil(t, resp.Err)
assert.Equal(t, resp.StatusCode, 404)
}

func TestHttpErrorH2C(t *testing.T) {
c := NewClient(serverUrl, false, 10000, H2C)
reqBody := ""
resp := c.SendRequest("GET", "/", []string{}, &reqBody)
reader := strings.NewReader(reqBody)
resp := c.SendRequest("GET", "/", make(map[string]string), reader)
assert.Nil(t, resp.Err)
assert.Equal(t, resp.StatusCode, 404)
}

func TestConnectionErrorHTTP1(t *testing.T) {
c := NewClient("http://localhost:9999", false, 10000, HTTP1)
reqBody := ""
resp := c.SendRequest("GET", "/potato", []string{}, &reqBody)
reader := strings.NewReader(reqBody)
resp := c.SendRequest("GET", "/potato", make(map[string]string), reader)
assert.NotNil(t, resp.Err)
}

func TestConnectionErrorH2C(t *testing.T) {
c := NewClient("http://localhost:9999", false, 10000, H2C)
reqBody := ""
resp := c.SendRequest("GET", "/potato", []string{}, &reqBody)
reader := strings.NewReader(reqBody)
resp := c.SendRequest("GET", "/potato", make(map[string]string), reader)
assert.NotNil(t, resp.Err)
}

Expand Down
85 changes: 77 additions & 8 deletions internal/pkg/http/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,18 +15,34 @@
package http

import (
"bytes"
"compress/flate"
"compress/gzip"
"fmt"
"io"
"mittens/internal/pkg/placeholders"
"strings"

"github.com/andybalholm/brotli"
)

// Request represents an HTTP request.
type Request struct {
Method string
Path string
Body *string
Method string
Headers map[string]string
Path string
Body io.Reader
}

type CompressionType string

const (
COMPRESSION_NONE CompressionType = ""
COMPRESSION_GZIP CompressionType = "gzip"
COMPRESSION_BROTLI CompressionType = "brotli"
COMPRESSION_DEFLATE CompressionType = "deflate"
)

var allowedHTTPMethods = map[string]interface{}{
"GET": nil,
"HEAD": nil,
Expand All @@ -39,9 +55,8 @@ var allowedHTTPMethods = map[string]interface{}{
"TRACE": nil,
}

//
// ToHTTPRequest parses an HTTP request which is in a string format and stores it in a struct.
func ToHTTPRequest(requestString string) (Request, error) {
func ToHTTPRequest(requestString string, compression CompressionType) (Request, error) {
parts := strings.SplitN(requestString, ":", 3)
if len(parts) < 2 {
return Request{}, fmt.Errorf("invalid request flag: %s, expected format <http-method>:<path>[:body]", requestString)
Expand Down Expand Up @@ -72,9 +87,63 @@ func ToHTTPRequest(requestString string) (Request, error) {
}
var body = placeholders.InterpolatePlaceholders(*rawBody)

var reader io.Reader
switch compression {
case COMPRESSION_GZIP:
reader = compressGzip([]byte(body))
case COMPRESSION_BROTLI:
reader = compressBrotli([]byte(body))
case COMPRESSION_DEFLATE:
reader = compressFlate([]byte(body))
default:
reader = bytes.NewBufferString(body)
}

headers := make(map[string]string)
if compression != COMPRESSION_NONE {
encoding := ""
switch compression {
case COMPRESSION_GZIP:
encoding = "gzip"
case COMPRESSION_BROTLI:
encoding = "br"
case COMPRESSION_DEFLATE:
encoding = "deflate"
}
headers["Content-Encoding"] = encoding
}

return Request{
Method: method,
Path: path,
Body: &body,
Method: method,
Headers: headers,
Path: path,
Body: reader,
}, nil
}

func compressGzip(data []byte) io.Reader {
pr, pw := io.Pipe()
go func() {
gz := gzip.NewWriter(pw)
_, err := gz.Write(data)
gz.Close()
pw.CloseWithError(err)
}()
return pr
}

func compressFlate(data []byte) *bytes.Buffer {
var b bytes.Buffer
w, _ := flate.NewWriter(&b, 9)
w.Write(data)
w.Close()
return &b
}

func compressBrotli(data []byte) *bytes.Buffer {
var b bytes.Buffer
w := brotli.NewWriterLevel(&b, brotli.BestCompression)
w.Write(data)
w.Close()
return &b
}
Loading

0 comments on commit 6b70df1

Please sign in to comment.