Limit response body in HTTP binding

Signed-off-by: ItalyPaleAle <[email protected]>

bindings/http/http.go

@@ -30,6 +30,8 @@ import (
 	"strings"
 	"time"
 
+	"k8s.io/apimachinery/pkg/api/resource"
+
 	"github.com/dapr/components-contrib/bindings"
 	"github.com/dapr/components-contrib/internal/utils"
 	"github.com/dapr/components-contrib/metadata"
@@ -41,11 +43,12 @@ const (
 	MTLSClientCert = "MTLSClientCert"
 	MTLSClientKey  = "MTLSClientKey"
 
-	TraceparentHeaderKey = "traceparent"
-	TracestateHeaderKey  = "tracestate"
-	TraceMetadataKey     = "traceHeaders"
-	securityToken        = "securityToken"
-	securityTokenHeader  = "securityTokenHeader"
+	TraceparentHeaderKey            = "traceparent"
+	TracestateHeaderKey             = "tracestate"
+	TraceMetadataKey                = "traceHeaders"
+	securityToken                   = "securityToken"
+	securityTokenHeader             = "securityTokenHeader"
+	defaultMaxResponseBodySizeBytes = 100 << 20 // 100 MB
 )
 
 // HTTPSource is a binding for an http url endpoint invocation
@@ -67,17 +70,29 @@ type httpMetadata struct {
 	SecurityToken       string         `mapstructure:"securityToken"`
 	SecurityTokenHeader string         `mapstructure:"securityTokenHeader"`
 	ResponseTimeout     *time.Duration `mapstructure:"responseTimeout"`
+	// Maximum response to read from HTTP response bodies.
+	// This can either be an integer which is interpreted in bytes, or a string with an added unit such as Mi.
+	// A value <= 0 means no limit.
+	// Default: 100MB
+	MaxResponseBodySize *resource.Quantity `mapstructure:"maxResponseBodySize"`
+
+	maxResponseBodySizeBytes int64
 }
 
 // NewHTTP returns a new HTTPSource.
 func NewHTTP(logger logger.Logger) bindings.OutputBinding {
-	return &HTTPSource{logger: logger}
+	return &HTTPSource{
+		logger: logger,
+	}
 }
 
 // Init performs metadata parsing.
 func (h *HTTPSource) Init(_ context.Context, meta bindings.Metadata) error {
-	var err error
-	if err = metadata.DecodeMetadata(meta.Properties, &h.metadata); err != nil {
+	h.metadata = httpMetadata{
+		MaxResponseBodySize: resource.NewQuantity(defaultMaxResponseBodySizeBytes, resource.BinarySI),
+	}
+	err := metadata.DecodeMetadata(meta.Properties, &h.metadata)
+	if err != nil {
 		return err
 	}
 
@@ -98,6 +113,14 @@ func (h *HTTPSource) Init(_ context.Context, meta bindings.Metadata) error {
 		}
 	}
 
+	if h.metadata.MaxResponseBodySize != nil && !h.metadata.MaxResponseBodySize.IsZero() {
+		val, ok := h.metadata.MaxResponseBodySize.AsInt64()
+		if !ok {
+			return fmt.Errorf("value for maxResponseBodySize cannot be converted to integer: %v", h.metadata.MaxResponseBodySize)
+		}
+		h.metadata.maxResponseBodySizeBytes = val
+	}
+
 	// See guidance on proper HTTP client settings here:
 	// https://medium.com/@nate510/don-t-use-go-s-default-http-client-4804cb19f779
 	dialer := &net.Dialer{
@@ -226,7 +249,7 @@ func (h *HTTPSource) Invoke(parentCtx context.Context, req *bindings.InvokeReque
 
 	if req.Metadata == nil {
 		// Prevent things below from failing if req.Metadata is nil.
-		req.Metadata = make(map[string]string)
+		req.Metadata = make(map[string]string, 0)
 	}
 
 	if req.Metadata["path"] != "" {
@@ -306,11 +329,20 @@ func (h *HTTPSource) Invoke(parentCtx context.Context, req *bindings.InvokeReque
 	if err != nil {
 		return nil, err
 	}
-	defer resp.Body.Close()
+	defer func() {
+		// Drain before closing
+		_, _ = io.Copy(io.Discard, resp.Body)
+		resp.Body.Close()
+	}()
+
+	var respBody io.Reader = resp.Body
+	if h.metadata.maxResponseBodySizeBytes > 0 {
+		respBody = io.LimitReader(resp.Body, h.metadata.maxResponseBodySizeBytes)
+	}
 
 	// Read the response body. For empty responses (e.g. 204 No Content)
 	// `b` will be an empty slice.
-	b, err := io.ReadAll(resp.Body)
+	b, err := io.ReadAll(respBody)
 	if err != nil {
 		return nil, err
 	}

bindings/http/http_test.go

@@ -68,6 +68,7 @@ func (tc TestCase) ToInvokeRequest() bindings.InvokeRequest {
 	}
 
 	requestMetadata["X-Status-Code"] = strconv.Itoa(tc.statusCode)
+	requestMetadata["path"] = tc.path
 
 	return bindings.InvokeRequest{
 		Data:      []byte(tc.input),
@@ -83,6 +84,14 @@ type HTTPHandler struct {
 
 func (h *HTTPHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 	h.Path = req.URL.Path
+	if strings.TrimPrefix(h.Path, "/") == "large" {
+		// Write 5KB
+		for i := 0; i < 1<<10; i++ {
+			fmt.Fprint(w, "12345")
+		}
+		return
+	}
+
 	h.Headers = make(map[string]string)
 	for headerKey, headerValue := range req.Header {
 		h.Headers[headerKey] = headerValue[0]
@@ -728,3 +737,27 @@ func verifyTimeoutBehavior(t *testing.T, hs bindings.OutputBinding, handler *HTT
 		})
 	}
 }
+
+func TestMaxBodySizeHonored(t *testing.T) {
+	handler := NewHTTPHandler()
+	s := httptest.NewServer(handler)
+	defer s.Close()
+
+	hs, err := InitBinding(s, map[string]string{"maxResponseBodySize": "1Ki"})
+	require.NoError(t, err)
+
+	tc := TestCase{
+		input:      "GET",
+		operation:  "get",
+		path:       "/large",
+		err:        "context deadline exceeded",
+		statusCode: 200,
+	}
+
+	req := tc.ToInvokeRequest()
+	response, err := hs.Invoke(context.Background(), &req)
+	require.NoError(t, err)
+
+	// Should have only read 1KB
+	assert.Len(t, response.Data, 1<<10)
+}

bindings/http/metadata.yaml

@@ -25,7 +25,7 @@ binding:
     - name: patch
       description: "Sometimes used to update a subset of fields of a record"
     - name: delete
-      description: "Delete a data/record"
+      description: "Delete data/record"
     - name: options
       description: "Requests for information about the communication options available (not commonly used)"
     - name: trace
@@ -37,51 +37,41 @@ metadata:
     description: "The base URL of the HTTP endpoint to invoke"
     example: '"http://host:port/path", "http://myservice:8000/customer"'
     # If omitted, uses the same values as "<root>.binding"
-    binding:
-      output: true
   - name: responseTimeout
     required: false
     description: "The duration after which HTTP requests should be canceled."
     example: '"10s", "5m"'
-    binding:
-      output: true
+  - name: maxResponseBodySize
+    required: false
+    description: "Max amount of data to read from the response body, as a resource quantity. A value <= 0 means no limit."
+    type: resource
+    default: '"100Mi"'
+    example: '"100" (as bytes), "1k", "10Ki", "1M", "1G"'
   - name: MTLSRootCA
     required: false
-    description: "Path to root ca certificate or pem encoded string"
-    example: "ca.pem"
-    binding:
-      output: true
+    description: "CA certificate: either a PEM-encoded string, or a path to a certificate on disk"
+    example: '"/path/to/ca.pem"'
   - name: MTLSClientCert
     required: false
-    description: "Path to client certificate or pem encoded string"
-    example: "client.pem"
-    binding:
-      output: true
+    description: "Client certificate for mTLS: either a PEM-encoded string, or a path to a certificate on disk"
+    example: '"/path/to/client.pem"'
   - name: MTLSClientKey
     required: false
-    description: "Path to client private key or pem encoded string"
-    example: "client.key"
-    binding:
-      output: true
+    description: "Client key for mTLS: either a PEM-encoded string, or a path to a certificate on disk"
+    example: '"/path/to/client.key"'
   - name: MTLSRenegotiation
     required: false
     description: "Set TLS renegotiation setting"
     allowedValues:
       - "RenegotiateNever"
       - "RenegotiateOnceAsClient"
       - "RenegotiateFreelyAsClient"
-    example: "RenegotiateOnceAsClient"
-    binding:
-      output: true
+    example: '"RenegotiateOnceAsClient"'
   - name: securityToken
     required: false
     description: "The security token to include on an outgoing HTTP request as a header"
-    example: "this-value-is-preferably-injected-from-a-secret-store"
-    binding:
-      output: true
+    example: '"this-value-is-preferably-injected-from-a-secret-store"'
   - name: securityTokenHeader
     required: false
     description: "The header name on an outgoing HTTP request for a security token"
-    example: "X-Security-Token"
-    binding:
-      output: true
+    example: '"X-Security-Token"'