diff --git a/docs/semantics.md b/docs/semantics.md index 531545b1fa..8a0291ca66 100644 --- a/docs/semantics.md +++ b/docs/semantics.md @@ -311,6 +311,17 @@ unlinks it. Process A continues to have a consistent view of the file's contents until it closes the file handle, at which point the contents are lost. +### GCS object metadata + +gcsfuse sets the following pieces of GCS object metadata for file objects: + +* `contentType` is set to GCS's best guess as to the MIME type of the file, + based on its file extension. + +* The custom metadata key `gcsfuse_mtime` is set to track mtime, as discussed + above. + + # Directory inodes diff --git a/internal/fs/fs.go b/internal/fs/fs.go index e1e56979dd..db1f6ae42f 100644 --- a/internal/fs/fs.go +++ b/internal/fs/fs.go @@ -121,6 +121,9 @@ func NewServer(cfg *ServerConfig) (server fuse.Server, err error) { return } + // Set up a bucket that infers content types when creating files. + bucket := gcsx.NewContentTypeBucket(cfg.Bucket) + // Create the object syncer. if cfg.TmpObjectPrefix == "" { err = errors.New("You must set TmpObjectPrefix.") @@ -130,13 +133,13 @@ func NewServer(cfg *ServerConfig) (server fuse.Server, err error) { syncer := gcsx.NewSyncer( cfg.AppendThreshold, cfg.TmpObjectPrefix, - cfg.Bucket) + bucket) // Set up the basic struct. fs := &fileSystem{ mtimeClock: timeutil.RealClock(), cacheClock: cfg.CacheClock, - bucket: cfg.Bucket, + bucket: bucket, syncer: syncer, tempDir: cfg.TempDir, implicitDirs: cfg.ImplicitDirectories, @@ -169,7 +172,7 @@ func NewServer(cfg *ServerConfig) (server fuse.Server, err error) { }, fs.implicitDirs, fs.dirTypeCacheTTL, - cfg.Bucket, + fs.bucket, fs.mtimeClock, fs.cacheClock) diff --git a/internal/fs/local_modifications_test.go b/internal/fs/local_modifications_test.go index e1a6548be6..d647ab4b77 100644 --- a/internal/fs/local_modifications_test.go +++ b/internal/fs/local_modifications_test.go @@ -1322,6 +1322,27 @@ func (t *DirectoryTest) RootAtimeCtimeAndMtime() { ExpectThat(mtime, timeutil.TimeNear(mountTime, delta)) } +func (t *DirectoryTest) ContentTypes() { + testCases := []string{ + "foo/", + "foo.jpg/", + "foo.txt/", + } + + for _, name := range testCases { + p := path.Join(t.mfs.Dir(), name) + + // Create the directory. + err := os.Mkdir(p, 0700) + AssertEq(nil, err) + + // There should be no content type set in GCS. + o, err := t.bucket.StatObject(t.ctx, &gcs.StatObjectRequest{Name: name}) + AssertEq(nil, err) + ExpectEq("", o.ContentType, "name: %q", name) + } +} + //////////////////////////////////////////////////////////////////////// // File interaction //////////////////////////////////////////////////////////////////////// @@ -2135,6 +2156,44 @@ func (t *FileTest) AtimeAndCtime() { ExpectThat(ctime, timeutil.TimeNear(createTime, delta)) } +func (t *FileTest) ContentTypes() { + testCases := map[string]string{ + "foo.jpg": "image/jpeg", + "bar.txt": "text/plain; charset=utf-8", + "baz": "", + } + + runOne := func(name string, expected string) { + p := path.Join(t.mfs.Dir(), name) + + // Create a file. + f, err := os.Create(p) + AssertEq(nil, err) + defer f.Close() + + // Check the GCS content type. + o, err := t.bucket.StatObject(t.ctx, &gcs.StatObjectRequest{Name: name}) + AssertEq(nil, err) + ExpectEq(expected, o.ContentType, "name: %q", name) + + // Modify the file and cause a new generation to be written out. + _, err = f.Write([]byte("taco")) + AssertEq(nil, err) + + err = f.Sync() + AssertEq(nil, err) + + // The GCS content type should still be correct. + o, err = t.bucket.StatObject(t.ctx, &gcs.StatObjectRequest{Name: name}) + AssertEq(nil, err) + ExpectEq(expected, o.ContentType, "name: %q", name) + } + + for name, expected := range testCases { + runOne(name, expected) + } +} + //////////////////////////////////////////////////////////////////////// // Symlinks //////////////////////////////////////////////////////////////////////// diff --git a/internal/gcsx/content_type_bucket.go b/internal/gcsx/content_type_bucket.go new file mode 100644 index 0000000000..3ce015597e --- /dev/null +++ b/internal/gcsx/content_type_bucket.go @@ -0,0 +1,59 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcsx + +import ( + "mime" + "path" + + "github.com/jacobsa/gcloud/gcs" + "golang.org/x/net/context" +) + +// NewContentTypeBucket creates a wrapper bucket that guesses MIME types for +// newly created or composed objects when an explicit type is not already set. +func NewContentTypeBucket(b gcs.Bucket) gcs.Bucket { + return contentTypeBucket{b} +} + +type contentTypeBucket struct { + gcs.Bucket +} + +func (b contentTypeBucket) CreateObject( + ctx context.Context, + req *gcs.CreateObjectRequest) (o *gcs.Object, err error) { + // Guess a content type if necessary. + if req.ContentType == "" { + req.ContentType = mime.TypeByExtension(path.Ext(req.Name)) + } + + // Pass on the request. + o, err = b.Bucket.CreateObject(ctx, req) + return +} + +func (b contentTypeBucket) ComposeObjects( + ctx context.Context, + req *gcs.ComposeObjectsRequest) (o *gcs.Object, err error) { + // Guess a content type if necessary. + if req.ContentType == "" { + req.ContentType = mime.TypeByExtension(path.Ext(req.DstName)) + } + + // Pass on the request. + o, err = b.Bucket.ComposeObjects(ctx, req) + return +} diff --git a/internal/gcsx/content_type_bucket_test.go b/internal/gcsx/content_type_bucket_test.go new file mode 100644 index 0000000000..554f6e8a2c --- /dev/null +++ b/internal/gcsx/content_type_bucket_test.go @@ -0,0 +1,144 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcsx_test + +import ( + "strings" + "testing" + + "github.com/googlecloudplatform/gcsfuse/internal/gcsx" + "github.com/jacobsa/gcloud/gcs" + "github.com/jacobsa/gcloud/gcs/gcsfake" + "github.com/jacobsa/timeutil" + "golang.org/x/net/context" +) + +var contentTypeBucketTestCases = []struct { + name string + request string // ContentType in request + expected string // Expected final type +}{ + ///////////////// + // No extension + ///////////////// + + 0: { + name: "foo/bar", + request: "", + expected: "", + }, + + 1: { + name: "foo/bar", + request: "image/jpeg", + expected: "image/jpeg", + }, + + ////////////////////// + // Unknown extension + ////////////////////// + + 2: { + name: "foo/bar.asdf", + request: "", + expected: "", + }, + + 3: { + name: "foo/bar.asdf", + request: "image/jpeg", + expected: "image/jpeg", + }, + + ////////////////////// + // Known extension + ////////////////////// + + 4: { + name: "foo/bar.jpg", + request: "", + expected: "image/jpeg", + }, + + 5: { + name: "foo/bar.jpg", + request: "text/plain", + expected: "text/plain", + }, +} + +func TestContentTypeBucket_CreateObject(t *testing.T) { + for i, tc := range contentTypeBucketTestCases { + // Set up a bucket. + bucket := gcsx.NewContentTypeBucket( + gcsfake.NewFakeBucket(timeutil.RealClock(), "")) + + // Create the object. + req := &gcs.CreateObjectRequest{ + Name: tc.name, + ContentType: tc.request, + Contents: strings.NewReader(""), + } + + o, err := bucket.CreateObject(context.Background(), req) + if err != nil { + t.Fatalf("Test case %d: CreateObject: %v", i, err) + } + + // Check the content type. + if got, want := o.ContentType, tc.expected; got != want { + t.Errorf("Test case %d: o.ContentType is %q, want %q", i, got, want) + } + } +} + +func TestContentTypeBucket_ComposeObjects(t *testing.T) { + var err error + ctx := context.Background() + + for i, tc := range contentTypeBucketTestCases { + // Set up a bucket. + bucket := gcsx.NewContentTypeBucket( + gcsfake.NewFakeBucket(timeutil.RealClock(), "")) + + // Create a source object. + const srcName = "some_src" + _, err = bucket.CreateObject(ctx, &gcs.CreateObjectRequest{ + Name: srcName, + Contents: strings.NewReader(""), + }) + + if err != nil { + t.Fatalf("Test case %d: CreateObject: %v", err) + } + + // Compose. + req := &gcs.ComposeObjectsRequest{ + DstName: tc.name, + ContentType: tc.request, + Sources: []gcs.ComposeSource{{Name: srcName}}, + } + + o, err := bucket.ComposeObjects(ctx, req) + if err != nil { + t.Fatalf("Test case %d: ComposeObject: %v", i, err) + } + + // Check the content type. + if got, want := o.ContentType, tc.expected; got != want { + t.Errorf("Test case %d: o.ContentType is %q, want %q", i, got, want) + } + } +} diff --git a/vendor/github.com/jacobsa/gcloud/LICENSE b/vendor/github.com/jacobsa/gcloud/LICENSE new file mode 100644 index 0000000000..e06d208186 --- /dev/null +++ b/vendor/github.com/jacobsa/gcloud/LICENSE @@ -0,0 +1,202 @@ +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/vendor/github.com/jacobsa/gcloud/gcs/compose_objects.go b/vendor/github.com/jacobsa/gcloud/gcs/compose_objects.go index b8aebe1900..b532259158 100644 --- a/vendor/github.com/jacobsa/gcloud/gcs/compose_objects.go +++ b/vendor/github.com/jacobsa/gcloud/gcs/compose_objects.go @@ -37,12 +37,9 @@ func (b *bucket) makeComposeObjectsBody( // Create a request in the form expected by the API. r := storagev1.ComposeRequest{ Destination: &storagev1.Object{ - Name: req.DstName, - Metadata: req.Metadata, - - // We get an HTTP 400 if we don't set this. - // Cf. Google-internal bug 21588058. - ContentType: "application/octet-stream", + Name: req.DstName, + ContentType: req.ContentType, + Metadata: req.Metadata, }, } diff --git a/vendor/github.com/jacobsa/gcloud/gcs/gcsfake/bucket.go b/vendor/github.com/jacobsa/gcloud/gcs/gcsfake/bucket.go index 00a84e17aa..3d6233b593 100644 --- a/vendor/github.com/jacobsa/gcloud/gcs/gcsfake/bucket.go +++ b/vendor/github.com/jacobsa/gcloud/gcs/gcsfake/bucket.go @@ -220,11 +220,6 @@ func (b *bucket) mintObject( // Set up data. o.data = contents - // Support the same default content type as GCS. - if o.metadata.ContentType == "" { - o.metadata.ContentType = "application/octet-stream" - } - return } @@ -679,6 +674,7 @@ func (b *bucket) ComposeObjects( GenerationPrecondition: req.DstGenerationPrecondition, MetaGenerationPrecondition: req.DstMetaGenerationPrecondition, Contents: io.MultiReader(srcReaders...), + ContentType: req.ContentType, Metadata: req.Metadata, } @@ -733,12 +729,6 @@ func (b *bucket) UpdateObject( b.mu.Lock() defer b.mu.Unlock() - // Match real GCS in not allowing the removal of ContentType. - if req.ContentType != nil && *req.ContentType == "" { - err = errors.New("The ContentType field is required and cannot be removed.") - return - } - // Does the object exist? index := b.objects.find(req.Name) if index == len(b.objects) { @@ -809,8 +799,9 @@ func (b *bucket) UpdateObject( } } - // Bump up the entry generation number. + // Bump up the entry generation number and the update time. obj.MetaGeneration++ + obj.Updated = b.clock.Now() // Make a copy to avoid handing back internal state. var objCopy gcs.Object = *obj diff --git a/vendor/github.com/jacobsa/gcloud/gcs/requests.go b/vendor/github.com/jacobsa/gcloud/gcs/requests.go index 79d03bd764..af54cd507f 100644 --- a/vendor/github.com/jacobsa/gcloud/gcs/requests.go +++ b/vendor/github.com/jacobsa/gcloud/gcs/requests.go @@ -124,7 +124,8 @@ type ComposeObjectsRequest struct { // // https://cloud.google.com/storage/docs/json_api/v1/objects#resource // - Metadata map[string]string + ContentType string + Metadata map[string]string } type ComposeSource struct { @@ -142,8 +143,11 @@ type ComposeSource struct { // // * If Limit is less than or equal to Start, the range is treated as empty. // -// * If Limit is greater than the length of the object, the range is -// implicitly truncated. +// * The effective range is [start, limit) intersected with [0, L), where L +// is the length of the object. +// +// For example, a read for [L-1, L+10) returns the last byte of the object, +// and [L+2, L+10) is legal but returns nothing. // type ByteRange struct { Start uint64 diff --git a/vendor/github.com/jacobsa/gcloud/gcs/retry.go b/vendor/github.com/jacobsa/gcloud/gcs/retry.go index e25c4d16c0..3261da2f5e 100644 --- a/vendor/github.com/jacobsa/gcloud/gcs/retry.go +++ b/vendor/github.com/jacobsa/gcloud/gcs/retry.go @@ -113,7 +113,7 @@ func chooseDelay(prevSleepCount uint) (d time.Duration) { // Choose a a delay in [0, 2^prevSleepCount * baseDelay). d = (1 << prevSleepCount) * baseDelay - d = time.Duration(float64(d) * rand.Float64()) + d = time.Duration(rand.Int63n(int64(d))) return } @@ -429,8 +429,8 @@ func (rb *retryBucket) CreateObject( // attempt might exhaust some of the req.Contents reader, leaving missing // contents for the second attempt. // - // So, copy out all contents and create a modified request that serves from - // memory. + // So, copy out all contents and create a copy of the request that we will + // modify to serve from memory for each call. contents, err := ioutil.ReadAll(req.Contents) if err != nil { err = fmt.Errorf("ioutil.ReadAll: %v", err) @@ -438,7 +438,6 @@ func (rb *retryBucket) CreateObject( } reqCopy := *req - reqCopy.Contents = bytes.NewReader(contents) // Call through with that request. err = oneShotExpBackoff( @@ -446,6 +445,7 @@ func (rb *retryBucket) CreateObject( fmt.Sprintf("CreateObject(%q)", req.Name), rb.maxSleep, func() (err error) { + reqCopy.Contents = bytes.NewReader(contents) o, err = rb.wrapped.CreateObject(ctx, &reqCopy) return }) diff --git a/vendor/github.com/jacobsa/gcloud/httputil/debugging_round_tripper.go b/vendor/github.com/jacobsa/gcloud/httputil/debugging_round_tripper.go index b3bb4cdabb..a236a2576c 100644 --- a/vendor/github.com/jacobsa/gcloud/httputil/debugging_round_tripper.go +++ b/vendor/github.com/jacobsa/gcloud/httputil/debugging_round_tripper.go @@ -16,10 +16,11 @@ package httputil import ( "bytes" - "io" + "fmt" "io/ioutil" "log" "net/http" + "net/http/httputil" ) // An interface for transports that support the signature of @@ -46,26 +47,28 @@ func DebuggingRoundTripper( // Helpers //////////////////////////////////////////////////////////////////////// -func readAllAndClose(rc io.ReadCloser) string { - // Read. - contents, err := ioutil.ReadAll(rc) - if err != nil { - panic(err) +// Ensure that the supplied request has a correct ContentLength field set. +func fillInContentLength(req *http.Request) (err error) { + body := req.Body + if body == nil { + req.ContentLength = 0 + return } - // Close. - if err := rc.Close(); err != nil { - panic(err) + // Make a copy. + contents, err := ioutil.ReadAll(req.Body) + if err != nil { + err = fmt.Errorf("ReadAll: %v", err) + return } - return string(contents) -} + req.Body.Close() -// Read everything from *rc, then replace it with a copy. -func snarfBody(rc *io.ReadCloser) string { - contents := readAllAndClose(*rc) - *rc = ioutil.NopCloser(bytes.NewBufferString(contents)) - return contents + // Fill in the content length and restore the body. + req.ContentLength = int64(len(contents)) + req.Body = ioutil.NopCloser(bytes.NewReader(contents)) + + return } //////////////////////////////////////////////////////////////////////// @@ -78,41 +81,47 @@ type debuggingRoundTripper struct { } func (t *debuggingRoundTripper) RoundTrip( - req *http.Request) (*http.Response, error) { - // Print information about the request. - t.logger.Println("========== REQUEST ===========") - t.logger.Println(req.Method, req.URL, req.Proto) - for k, vs := range req.Header { - for _, v := range vs { - t.logger.Printf("%s: %s\n", k, v) - } + req *http.Request) (resp *http.Response, err error) { + // Ensure that the request has a ContentLength field, so that it doesn't need + // to be transmitted with chunked encoding. This improves debugging output. + err = fillInContentLength(req) + if err != nil { + err = fmt.Errorf("fillInContentLength: %v", err) + return } - if req.Body != nil { - t.logger.Printf("\n%s\n", snarfBody(&req.Body)) + // Dump the request. + { + var dumped []byte + dumped, err = httputil.DumpRequestOut(req, true) + if err != nil { + err = fmt.Errorf("DumpRequestOut: %v", err) + return + } + + t.logger.Printf("========== REQUEST:\n%s", dumped) } // Execute the request. - res, err := t.wrapped.RoundTrip(req) + resp, err = t.wrapped.RoundTrip(req) if err != nil { - return res, err + return } - // Print the response. - t.logger.Println("========== RESPONSE ==========") - t.logger.Println(res.Proto, res.Status) - for k, vs := range res.Header { - for _, v := range vs { - t.logger.Printf("%s: %s\n", k, v) + // Dump the response. + { + var dumped []byte + dumped, err = httputil.DumpResponse(resp, true) + if err != nil { + err = fmt.Errorf("DumpResponse: %v", err) + return } - } - if res.Body != nil { - t.logger.Printf("\n%s\n", snarfBody(&res.Body)) + t.logger.Printf("========== RESPONSE:\n%s", dumped) + t.logger.Println("====================") } - t.logger.Println("==============================") - return res, err + return } func (t *debuggingRoundTripper) CancelRequest(req *http.Request) { diff --git a/vendor/vendor.json b/vendor/vendor.json index 43794841fd..ef88f52b50 100755 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -54,28 +54,28 @@ }, { "path": "github.com/jacobsa/gcloud/gcs", - "revision": "a49fd56fbbeb6288d0ff40fa8b4fd9c8e0df8c1b", - "revisionTime": "2015-08-31T11:02:33+10:00" + "revision": "60dc0c7874d8adf958f7ded1e2edda4d7b3e8318", + "revisionTime": "2016-02-19T11:39:26+11:00" }, { "path": "github.com/jacobsa/gcloud/gcs/gcscaching", - "revision": "a49fd56fbbeb6288d0ff40fa8b4fd9c8e0df8c1b", - "revisionTime": "2015-08-31T11:02:33+10:00" + "revision": "60dc0c7874d8adf958f7ded1e2edda4d7b3e8318", + "revisionTime": "2016-02-19T11:39:26+11:00" }, { "path": "github.com/jacobsa/gcloud/gcs/gcsfake", - "revision": "a49fd56fbbeb6288d0ff40fa8b4fd9c8e0df8c1b", - "revisionTime": "2015-08-31T11:02:33+10:00" + "revision": "60dc0c7874d8adf958f7ded1e2edda4d7b3e8318", + "revisionTime": "2016-02-19T11:39:26+11:00" }, { "path": "github.com/jacobsa/gcloud/gcs/gcsutil", - "revision": "a49fd56fbbeb6288d0ff40fa8b4fd9c8e0df8c1b", - "revisionTime": "2015-08-31T11:02:33+10:00" + "revision": "60dc0c7874d8adf958f7ded1e2edda4d7b3e8318", + "revisionTime": "2016-02-19T11:39:26+11:00" }, { "path": "github.com/jacobsa/gcloud/httputil", - "revision": "a49fd56fbbeb6288d0ff40fa8b4fd9c8e0df8c1b", - "revisionTime": "2015-08-31T11:02:33+10:00" + "revision": "60dc0c7874d8adf958f7ded1e2edda4d7b3e8318", + "revisionTime": "2016-02-19T11:39:26+11:00" }, { "path": "github.com/jacobsa/oglematchers",