From c409f69dc856f5f19147f4f3e84ffc62408984e5 Mon Sep 17 00:00:00 2001 From: Ashmeen Kaur Date: Wed, 20 Sep 2023 07:22:39 +0000 Subject: [PATCH] rebase changes --- .../implicit_dir/local_file_test.go | 16 +-- .../local_file/create_file_test.go | 18 +-- .../local_file/helpers/gcs_helper.go | 103 ------------------ .../local_file/local_file_test.go | 23 ++-- .../util/client/gcs_helper.go | 63 ++++++++--- 5 files changed, 80 insertions(+), 143 deletions(-) delete mode 100644 tools/integration_tests/local_file/helpers/gcs_helper.go diff --git a/tools/integration_tests/implicit_dir/local_file_test.go b/tools/integration_tests/implicit_dir/local_file_test.go index 52c6ba1c32..03a8907a4a 100644 --- a/tools/integration_tests/implicit_dir/local_file_test.go +++ b/tools/integration_tests/implicit_dir/local_file_test.go @@ -45,7 +45,7 @@ func TestNewFileUnderImplicitDirectoryShouldNotGetSyncedToGCSTillClose(t *testin CreateImplicitDir(ctx, storageClient, testDirName, t) fileName := path.Join(ImplicitDirName, FileName1) - fh := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName, t) + _, fh := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName, t) operations.WriteWithoutClose(fh, FileContents, t) ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, fileName, t) @@ -58,8 +58,8 @@ func TestReadDirForImplicitDirWithLocalFile(t *testing.T) { CreateImplicitDir(ctx, storageClient, testDirName, t) fileName1 := path.Join(ImplicitDirName, FileName1) fileName2 := path.Join(ImplicitDirName, FileName2) - fh1 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName1, t) - fh2 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName2, t) + _, fh1 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName1, t) + _, fh2 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName2, t) // Attempt to list implicit directory. entries := operations.ReadDirectory(path.Join(testDirPath, ImplicitDirName), t) @@ -68,7 +68,7 @@ func TestReadDirForImplicitDirWithLocalFile(t *testing.T) { operations.VerifyCountOfDirectoryEntries(3, len(entries), t) operations.VerifyFileEntry(entries[0], FileName1, 0, t) operations.VerifyFileEntry(entries[1], FileName2, 0, t) - operations.VerifyFileEntry(entries[2], ImplicitFileName1, ImplicitFileSize, t) + operations.VerifyFileEntry(entries[2], ImplicitFileName1, GCSFileSize, t) // Close the local files. CloseFileAndValidateContentFromGCS(ctx, storageClient, testDirName, fh1, fileName1, "", t) CloseFileAndValidateContentFromGCS(ctx, storageClient, testDirName, fh2, fileName2, "", t) @@ -88,13 +88,13 @@ func TestRecursiveListingWithLocalFiles(t *testing.T) { fileName2 := path.Join(ExplicitDirName, ExplicitFileName1) fileName3 := path.Join(ImplicitDirName, FileName2) // Create local file in mnt/ dir. - fh1 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, FileName1, t) + _, fh1 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, FileName1, t) // Create explicit dir with 1 local file. operations.CreateDirectory(path.Join(testDirPath, ExplicitDirName), t) - fh2 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName2, t) + _, fh2 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName2, t) // Create implicit dir with 1 local file1 and 1 synced file. CreateImplicitDir(ctx, storageClient, testDirName, t) - fh3 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName3, t) + _, fh3 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName3, t) // Recursively list mntDir/ directory. err := filepath.WalkDir(testDirPath, @@ -130,7 +130,7 @@ func TestRecursiveListingWithLocalFiles(t *testing.T) { // numberOfObjects = 2 operations.VerifyCountOfDirectoryEntries(2, len(objs), t) operations.VerifyFileEntry(objs[0], FileName2, 0, t) - operations.VerifyFileEntry(objs[1], ImplicitFileName1, ImplicitFileSize, t) + operations.VerifyFileEntry(objs[1], ImplicitFileName1, GCSFileSize, t) } return nil }) diff --git a/tools/integration_tests/local_file/create_file_test.go b/tools/integration_tests/local_file/create_file_test.go index dd790ac89b..c6351eaf85 100644 --- a/tools/integration_tests/local_file/create_file_test.go +++ b/tools/integration_tests/local_file/create_file_test.go @@ -19,37 +19,37 @@ import ( "path" "testing" - . "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/local_file/helpers" + . "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/client" "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/operations" "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/setup" ) func TestNewFileShouldNotGetSyncedToGCSTillClose(t *testing.T) { - testDirPath = setup.SetupTestDirectory(LocalFileTestDirInBucket) + testDirPath = setup.SetupTestDirectory(testDirName) // Validate. - NewFileShouldGetSyncedToGCSAtClose(testDirPath, FileName1, t) + NewFileShouldGetSyncedToGCSAtClose(ctx, storageClient, testDirPath, FileName1, t) } func TestNewFileUnderExplicitDirectoryShouldNotGetSyncedToGCSTillClose(t *testing.T) { - testDirPath = setup.SetupTestDirectory(LocalFileTestDirInBucket) + testDirPath = setup.SetupTestDirectory(testDirName) // Make explicit directory. operations.CreateDirectory(path.Join(testDirPath, ExplicitDirName), t) // Validate. - NewFileShouldGetSyncedToGCSAtClose(testDirPath, path.Join(ExplicitDirName, ExplicitFileName1), t) + NewFileShouldGetSyncedToGCSAtClose(ctx, storageClient, testDirPath, path.Join(ExplicitDirName, ExplicitFileName1), t) } func TestCreateNewFileWhenSameFileExistsOnGCS(t *testing.T) { - testDirPath = setup.SetupTestDirectory(LocalFileTestDirInBucket) + testDirPath = setup.SetupTestDirectory(testDirName) // Create a local file. - _, fh := CreateLocalFileInTestDir(testDirPath, FileName1, t) + _, fh := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, FileName1, t) // Create a file on GCS with the same name. - CreateObjectInGCSTestDir(FileName1, GCSFileContent, t) + CreateObjectInGCSTestDir(ctx, storageClient, testDirName, FileName1, GCSFileContent, t) // Write to local file. operations.WriteWithoutClose(fh, FileContents, t) // Close the local file and ensure that the content on GCS is not overwritten. - CloseFileAndValidateObjectContentsFromGCS(fh, FileName1, GCSFileContent, t) + CloseFileAndValidateObjectContentsFromGCS(ctx, storageClient, fh, testDirName, FileName1, GCSFileContent, t) } diff --git a/tools/integration_tests/local_file/helpers/gcs_helper.go b/tools/integration_tests/local_file/helpers/gcs_helper.go deleted file mode 100644 index b1536fd62f..0000000000 --- a/tools/integration_tests/local_file/helpers/gcs_helper.go +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright 2023 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package helpers - -import ( - "os" - "path" - "strings" - "testing" - - "cloud.google.com/go/storage" - "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/client" - "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/operations" - "golang.org/x/net/context" -) - -const ( - FileName1 = "foo1" - ExplicitDirName = "explicit" - ExplicitFileName1 = "explicitFile1" - FilePerms = 0644 - FileContents = "teststring" - GCSFileContent = "gcsContent" - LocalFileTestDirInBucket = "LocalFileTest" - ReadSize = 1024 -) - -var StorageClient *storage.Client -var Ctx context.Context - -func ValidateObjectNotFoundErrOnGCS(fileName string, t *testing.T) { - _, err := client.ReadObjectFromGCS( - StorageClient, - path.Join(LocalFileTestDirInBucket, fileName), - ReadSize, - Ctx) - if err == nil || !strings.Contains(err.Error(), "storage: object doesn't exist") { - t.Fatalf("Incorrect error returned from GCS for file %s: %v", fileName, err) - } -} - -func ValidateObjectContentsFromGCS(fileName string, expectedContent string, t *testing.T) { - gotContent, err := client.ReadObjectFromGCS( - StorageClient, - path.Join(LocalFileTestDirInBucket, fileName), - ReadSize, - Ctx) - if err != nil { - t.Fatalf("Error while reading synced local file from GCS, Err: %v", err) - } - - if expectedContent != gotContent { - t.Fatalf("GCS file %s content mismatch. Got: %s, Expected: %s ", fileName, gotContent, expectedContent) - } -} - -func CloseFileAndValidateObjectContentsFromGCS(f *os.File, fileName string, contents string, t *testing.T) { - operations.CloseFileShouldNotThrowError(f, t) - ValidateObjectContentsFromGCS(fileName, contents, t) -} - -func WritingToLocalFileShouldNotWriteToGCS(fh *os.File, fileName string, t *testing.T) { - operations.WriteWithoutClose(fh, FileContents, t) - ValidateObjectNotFoundErrOnGCS(fileName, t) -} - -func NewFileShouldGetSyncedToGCSAtClose(testDirPath, fileName string, t *testing.T) { - // Create a local file. - _, fh := CreateLocalFileInTestDir(testDirPath, fileName, t) - - // Writing contents to local file shouldn't create file on GCS. - WritingToLocalFileShouldNotWriteToGCS(fh, fileName, t) - - // Close the file and validate if the file is created on GCS. - CloseFileAndValidateObjectContentsFromGCS(fh, fileName, FileContents, t) -} - -func CreateObjectInGCSTestDir(fileName, content string, t *testing.T) { - objectName := path.Join(LocalFileTestDirInBucket, fileName) - err := client.CreateObjectOnGCS(StorageClient, objectName, content, Ctx) - if err != nil { - t.Fatalf("Create Object %s on GCS: %v.", objectName, err) - } -} - -func CreateLocalFileInTestDir(testDirPath, fileName string, t *testing.T) (string, *os.File) { - filePath := path.Join(testDirPath, fileName) - fh := operations.CreateFile(filePath, FilePerms, t) - ValidateObjectNotFoundErrOnGCS(fileName, t) - return filePath, fh -} diff --git a/tools/integration_tests/local_file/local_file_test.go b/tools/integration_tests/local_file/local_file_test.go index 23863f04d6..4f261a8457 100644 --- a/tools/integration_tests/local_file/local_file_test.go +++ b/tools/integration_tests/local_file/local_file_test.go @@ -24,8 +24,8 @@ import ( "testing" "time" + "cloud.google.com/go/storage" "github.com/googlecloudplatform/gcsfuse/internal/config" - "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/local_file/helpers" "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/client" "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/mounting/dynamic_mounting" "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/mounting/only_dir_mounting" @@ -33,13 +33,20 @@ import ( "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/setup" ) -// testDirPath holds the path to the test subdirectory in the mounted bucket. -var testDirPath string +const ( + testDirName = "LocalFileTest" +) + +var ( + testDirPath string + storageClient *storage.Client + ctx context.Context +) func TestMain(m *testing.M) { setup.ParseSetUpFlags() - helpers.Ctx = context.Background() + ctx = context.Background() var cancel context.CancelFunc var err error @@ -51,8 +58,8 @@ func TestMain(m *testing.M) { } // Create storage client before running tests. - helpers.Ctx, cancel = context.WithTimeout(helpers.Ctx, time.Minute*15) - helpers.StorageClient, err = client.CreateStorageClient(helpers.Ctx) + ctx, cancel = context.WithTimeout(ctx, time.Minute*15) + storageClient, err = client.CreateStorageClient(ctx) if err != nil { log.Fatalf("client.CreateStorageClient: %v", err) } @@ -89,10 +96,10 @@ func TestMain(m *testing.M) { } // Close storage client and release resources. - helpers.StorageClient.Close() + storageClient.Close() cancel() // Clean up test directory created. - setup.CleanupDirectoryOnGCS(path.Join(setup.TestBucket(), helpers.LocalFileTestDirInBucket)) + setup.CleanupDirectoryOnGCS(path.Join(setup.TestBucket(), testDirName)) setup.RemoveBinFileCopiedForTesting() os.Exit(successCode) } diff --git a/tools/integration_tests/util/client/gcs_helper.go b/tools/integration_tests/util/client/gcs_helper.go index 5837ac1536..7d8bf102e0 100644 --- a/tools/integration_tests/util/client/gcs_helper.go +++ b/tools/integration_tests/util/client/gcs_helper.go @@ -26,17 +26,17 @@ import ( ) const ( - FileName1 = "foo1" - FileName2 = "foo2" - ExplicitDirName = "explicit" - ExplicitFileName1 = "explicitFile1" - ImplicitDirName = "implicit" - ImplicitFileName1 = "implicitFile1" - FileContents = "testString" - ImplicitFileContents = "GCSteststring" - ImplicitFileSize = 13 - FilePerms = 0644 - ReadSize = 1024 + FileName1 = "foo1" + FileName2 = "foo2" + ExplicitDirName = "explicit" + ExplicitFileName1 = "explicitFile1" + ImplicitDirName = "implicit" + ImplicitFileName1 = "implicitFile1" + FileContents = "testString" + GCSFileContent = "GCSteststring" + GCSFileSize = 13 + FilePerms = 0644 + ReadSize = 1024 ) func CreateImplicitDir(ctx context.Context, storageClient *storage.Client, @@ -45,7 +45,7 @@ func CreateImplicitDir(ctx context.Context, storageClient *storage.Client, ctx, storageClient, path.Join(testDirName, ImplicitDirName, ImplicitFileName1), - ImplicitFileContents) + GCSFileContent) if err != nil { t.Errorf("Error while creating implicit directory, err: %v", err) } @@ -78,15 +78,48 @@ func CloseFileAndValidateContentFromGCS(ctx context.Context, storageClient *stor } func CreateLocalFileInTestDir(ctx context.Context, storageClient *storage.Client, - testDirPath, fileName string, t *testing.T) (fh *os.File) { + testDirPath, fileName string, t *testing.T) (string, *os.File) { filePath := path.Join(testDirPath, fileName) - fh = operations.CreateFile(filePath, FilePerms, t) + fh := operations.CreateFile(filePath, FilePerms, t) testDirName := getDirName(testDirPath) ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, fileName, t) - return + return filePath, fh } func getDirName(testDirPath string) string { dirName := testDirPath[strings.LastIndex(testDirPath, "/")+1:] return dirName } + +func WritingToLocalFileShouldNotWriteToGCS(ctx context.Context, storageClient *storage.Client, fh *os.File, testDirName, fileName string, t *testing.T) { + operations.WriteWithoutClose(fh, FileContents, t) + ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, fileName, t) +} + +func NewFileShouldGetSyncedToGCSAtClose(ctx context.Context, storageClient *storage.Client, + testDirPath, fileName string, t *testing.T) { + // Create a local file. + _, fh := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName, t) + + // Writing contents to local file shouldn't create file on GCS. + testDirName := getDirName(testDirPath) + WritingToLocalFileShouldNotWriteToGCS(ctx, storageClient, fh, testDirName, fileName, t) + + // Close the file and validate if the file is created on GCS. + CloseFileAndValidateObjectContentsFromGCS(ctx, storageClient, fh, testDirName, fileName, FileContents, t) +} + +func CloseFileAndValidateObjectContentsFromGCS(ctx context.Context, storageClient *storage.Client, + f *os.File, testDirName, fileName, contents string, t *testing.T) { + operations.CloseFileShouldNotThrowError(f, t) + validateObjectContentsFromGCS(ctx, storageClient, testDirName, fileName, contents, t) +} + +func CreateObjectInGCSTestDir(ctx context.Context, storageClient *storage.Client, + testDirName, fileName, content string, t *testing.T) { + objectName := path.Join(testDirName, fileName) + err := CreateObjectOnGCS(ctx, storageClient, objectName, content) + if err != nil { + t.Fatalf("Create Object %s on GCS: %v.", objectName, err) + } +}