From bd3183f6b2e405ed4e7630045a9bd72202e87d11 Mon Sep 17 00:00:00 2001 From: Ashmeen Kaur Date: Tue, 19 Sep 2023 11:04:53 +0000 Subject: [PATCH] verify that file creation does not creates file on GCS until synced --- .../local_file/create_file_test.go | 55 ++++++++++ .../local_file/helpers/gcs_helper.go | 103 ++++++++++++++++++ .../local_file/local_file_test.go | 98 +++++++++++++++++ .../run_tests_mounted_directory.sh | 11 ++ .../dynamic_mounting/dynamic_mounting.go | 6 + .../only_dir_mounting/only_dir_mounting.go | 4 + 6 files changed, 277 insertions(+) create mode 100644 tools/integration_tests/local_file/create_file_test.go create mode 100644 tools/integration_tests/local_file/helpers/gcs_helper.go create mode 100644 tools/integration_tests/local_file/local_file_test.go diff --git a/tools/integration_tests/local_file/create_file_test.go b/tools/integration_tests/local_file/create_file_test.go new file mode 100644 index 0000000000..dd790ac89b --- /dev/null +++ b/tools/integration_tests/local_file/create_file_test.go @@ -0,0 +1,55 @@ +// Copyright 2023 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Provides integration tests for create local file. +package local_file_test + +import ( + "path" + "testing" + + . "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/local_file/helpers" + "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/setup" +) + +func TestNewFileShouldNotGetSyncedToGCSTillClose(t *testing.T) { + testDirPath = setup.SetupTestDirectory(LocalFileTestDirInBucket) + + // Validate. + NewFileShouldGetSyncedToGCSAtClose(testDirPath, FileName1, t) +} + +func TestNewFileUnderExplicitDirectoryShouldNotGetSyncedToGCSTillClose(t *testing.T) { + testDirPath = setup.SetupTestDirectory(LocalFileTestDirInBucket) + // Make explicit directory. + operations.CreateDirectory(path.Join(testDirPath, ExplicitDirName), t) + + // Validate. + NewFileShouldGetSyncedToGCSAtClose(testDirPath, path.Join(ExplicitDirName, ExplicitFileName1), t) +} + +func TestCreateNewFileWhenSameFileExistsOnGCS(t *testing.T) { + testDirPath = setup.SetupTestDirectory(LocalFileTestDirInBucket) + // Create a local file. + _, fh := CreateLocalFileInTestDir(testDirPath, FileName1, t) + + // Create a file on GCS with the same name. + CreateObjectInGCSTestDir(FileName1, GCSFileContent, t) + + // Write to local file. + operations.WriteWithoutClose(fh, FileContents, t) + // Close the local file and ensure that the content on GCS is not overwritten. + CloseFileAndValidateObjectContentsFromGCS(fh, FileName1, GCSFileContent, t) +} diff --git a/tools/integration_tests/local_file/helpers/gcs_helper.go b/tools/integration_tests/local_file/helpers/gcs_helper.go new file mode 100644 index 0000000000..b1536fd62f --- /dev/null +++ b/tools/integration_tests/local_file/helpers/gcs_helper.go @@ -0,0 +1,103 @@ +// Copyright 2023 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package helpers + +import ( + "os" + "path" + "strings" + "testing" + + "cloud.google.com/go/storage" + "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/operations" + "golang.org/x/net/context" +) + +const ( + FileName1 = "foo1" + ExplicitDirName = "explicit" + ExplicitFileName1 = "explicitFile1" + FilePerms = 0644 + FileContents = "teststring" + GCSFileContent = "gcsContent" + LocalFileTestDirInBucket = "LocalFileTest" + ReadSize = 1024 +) + +var StorageClient *storage.Client +var Ctx context.Context + +func ValidateObjectNotFoundErrOnGCS(fileName string, t *testing.T) { + _, err := client.ReadObjectFromGCS( + StorageClient, + path.Join(LocalFileTestDirInBucket, fileName), + ReadSize, + Ctx) + if err == nil || !strings.Contains(err.Error(), "storage: object doesn't exist") { + t.Fatalf("Incorrect error returned from GCS for file %s: %v", fileName, err) + } +} + +func ValidateObjectContentsFromGCS(fileName string, expectedContent string, t *testing.T) { + gotContent, err := client.ReadObjectFromGCS( + StorageClient, + path.Join(LocalFileTestDirInBucket, fileName), + ReadSize, + Ctx) + if err != nil { + t.Fatalf("Error while reading synced local file from GCS, Err: %v", err) + } + + if expectedContent != gotContent { + t.Fatalf("GCS file %s content mismatch. Got: %s, Expected: %s ", fileName, gotContent, expectedContent) + } +} + +func CloseFileAndValidateObjectContentsFromGCS(f *os.File, fileName string, contents string, t *testing.T) { + operations.CloseFileShouldNotThrowError(f, t) + ValidateObjectContentsFromGCS(fileName, contents, t) +} + +func WritingToLocalFileShouldNotWriteToGCS(fh *os.File, fileName string, t *testing.T) { + operations.WriteWithoutClose(fh, FileContents, t) + ValidateObjectNotFoundErrOnGCS(fileName, t) +} + +func NewFileShouldGetSyncedToGCSAtClose(testDirPath, fileName string, t *testing.T) { + // Create a local file. + _, fh := CreateLocalFileInTestDir(testDirPath, fileName, t) + + // Writing contents to local file shouldn't create file on GCS. + WritingToLocalFileShouldNotWriteToGCS(fh, fileName, t) + + // Close the file and validate if the file is created on GCS. + CloseFileAndValidateObjectContentsFromGCS(fh, fileName, FileContents, t) +} + +func CreateObjectInGCSTestDir(fileName, content string, t *testing.T) { + objectName := path.Join(LocalFileTestDirInBucket, fileName) + err := client.CreateObjectOnGCS(StorageClient, objectName, content, Ctx) + if err != nil { + t.Fatalf("Create Object %s on GCS: %v.", objectName, err) + } +} + +func CreateLocalFileInTestDir(testDirPath, fileName string, t *testing.T) (string, *os.File) { + filePath := path.Join(testDirPath, fileName) + fh := operations.CreateFile(filePath, FilePerms, t) + ValidateObjectNotFoundErrOnGCS(fileName, t) + return filePath, fh +} diff --git a/tools/integration_tests/local_file/local_file_test.go b/tools/integration_tests/local_file/local_file_test.go new file mode 100644 index 0000000000..23863f04d6 --- /dev/null +++ b/tools/integration_tests/local_file/local_file_test.go @@ -0,0 +1,98 @@ +// Copyright 2023 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Provides integration tests for file and directory operations. + +package local_file_test + +import ( + "context" + "log" + "os" + "path" + "testing" + "time" + + "github.com/googlecloudplatform/gcsfuse/internal/config" + "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/local_file/helpers" + "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/mounting/dynamic_mounting" + "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/mounting/only_dir_mounting" + "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/mounting/static_mounting" + "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/setup" +) + +// testDirPath holds the path to the test subdirectory in the mounted bucket. +var testDirPath string + +func TestMain(m *testing.M) { + setup.ParseSetUpFlags() + + helpers.Ctx = context.Background() + var cancel context.CancelFunc + var err error + + setup.ExitWithFailureIfBothTestBucketAndMountedDirectoryFlagsAreNotSet() + + if setup.MountedDirectory() != "" && setup.TestBucket() == "" { + log.Print("Both --testbucket and --mountedDirectory should be specified to run mounted directory tests.") + os.Exit(1) + } + + // Create storage client before running tests. + helpers.Ctx, cancel = context.WithTimeout(helpers.Ctx, time.Minute*15) + helpers.StorageClient, err = client.CreateStorageClient(helpers.Ctx) + if err != nil { + log.Fatalf("client.CreateStorageClient: %v", err) + } + + // Run tests for mountedDirectory only if --mountedDirectory flag is set. + setup.RunTestsForMountedDirectoryFlag(m) + + // Else run tests for testBucket. + // Set up test directory. + setup.SetUpTestDirForTestBucketFlag() + // Set up config file with create-empty-file: false. (default) + mountConfig := config.MountConfig{ + WriteConfig: config.WriteConfig{ + CreateEmptyFile: false, + }, + LogConfig: config.LogConfig{ + Severity: config.TRACE, + }, + } + configFile := setup.YAMLConfigFile(mountConfig) + // Set up flags to run tests on. + flags := [][]string{ + {"--implicit-dirs=true", "--rename-dir-limit=3", "--config-file=" + configFile}, + {"--implicit-dirs=false", "--rename-dir-limit=3", "--config-file=" + configFile}} + + successCode := static_mounting.RunTests(flags, m) + + if successCode == 0 { + successCode = only_dir_mounting.RunTests(flags, m) + } + + if successCode == 0 { + successCode = dynamic_mounting.RunTests(flags, m) + } + + // Close storage client and release resources. + helpers.StorageClient.Close() + cancel() + // Clean up test directory created. + setup.CleanupDirectoryOnGCS(path.Join(setup.TestBucket(), helpers.LocalFileTestDirInBucket)) + setup.RemoveBinFileCopiedForTesting() + os.Exit(successCode) +} diff --git a/tools/integration_tests/run_tests_mounted_directory.sh b/tools/integration_tests/run_tests_mounted_directory.sh index a3b000c866..62a87c4892 100755 --- a/tools/integration_tests/run_tests_mounted_directory.sh +++ b/tools/integration_tests/run_tests_mounted_directory.sh @@ -231,3 +231,14 @@ sudo umount $MOUNT_DIR gcsfuse --implicit-dirs $TEST_BUCKET_NAME $MOUNT_DIR GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/gzip/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME sudo umount $MOUNT_DIR + +# package local_file +# Run test with static mounting. (flags: --implicit-dirs=true) +gcsfuse --implicit-dirs=true --rename-dir-limit=3 $TEST_BUCKET_NAME $MOUNT_DIR +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/local_file/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME +sudo umount $MOUNT_DIR + +# Run test with static mounting. (flags: --implicit-dirs=false) +gcsfuse --implicit-dirs=false --rename-dir-limit=3 $TEST_BUCKET_NAME $MOUNT_DIR +GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/local_file/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME +sudo umount $MOUNT_DIR diff --git a/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go b/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go index e58de71901..3081aa735d 100644 --- a/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go +++ b/tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go @@ -79,13 +79,19 @@ func executeTestsForDynamicMounting(flags [][]string, m *testing.M) (successCode // mntDir - bucket1, bucket2, bucket3, ... // We will test on passed testBucket and one created bucket. + // SetDynamicBucketMounted to the passed test bucket. + setup.SetDynamicBucketMounted(setup.TestBucket()) // Test on testBucket successCode = runTestsOnGivenMountedTestBucket(setup.TestBucket(), flags, rootMntDir, m) // Test on created bucket. + // SetDynamicBucketMounted to the mounted bucket. + setup.SetDynamicBucketMounted(testBucketForDynamicMounting) if successCode == 0 { successCode = runTestsOnGivenMountedTestBucket(testBucketForDynamicMounting, flags, rootMntDir, m) } + // Reset SetDynamicBucketMounted to empty after tests are done. + setup.SetDynamicBucketMounted("") // Setting back the original mntDir after testing. setup.SetMntDir(rootMntDir) diff --git a/tools/integration_tests/util/mounting/only_dir_mounting/only_dir_mounting.go b/tools/integration_tests/util/mounting/only_dir_mounting/only_dir_mounting.go index 8a219272b8..61f8f1098d 100644 --- a/tools/integration_tests/util/mounting/only_dir_mounting/only_dir_mounting.go +++ b/tools/integration_tests/util/mounting/only_dir_mounting/only_dir_mounting.go @@ -57,6 +57,8 @@ func mountGcsFuseForFlagsAndExecuteTests(flags [][]string, dir string, m *testin } func executeTestsForOnlyDirMounting(flags [][]string, m *testing.M) (successCode int) { + // Set onlyDirMounted value to the directory being mounted. + setup.SetOnlyDirMounted(DirectoryInTestBucket) mountDirInBucket := path.Join(setup.TestBucket(), DirectoryInTestBucket) // Clean the bucket. @@ -77,6 +79,8 @@ func executeTestsForOnlyDirMounting(flags [][]string, m *testing.M) (successCode // Clean the bucket after testing. setup.RunScriptForTestData("../util/mounting/only_dir_mounting/testdata/delete_objects.sh", setup.TestBucket()) + // Reset onlyDirMounted value to empty string after only dir mount tests are done. + setup.SetOnlyDirMounted("") return }