verify that file creation does not creates file on GCS until synced

tools/integration_tests/local_file/create_file_test.go

@@ -0,0 +1,55 @@
+// Copyright 2023 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Provides integration tests for create local file.
+package local_file_test
+
+import (
+	"path"
+	"testing"
+
+	. "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/local_file/helpers"
+	"github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/operations"
+	"github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/setup"
+)
+
+func TestNewFileShouldNotGetSyncedToGCSTillClose(t *testing.T) {
+	testDirPath = setup.SetupTestDirectory(LocalFileTestDirInBucket)
+
+	// Validate.
+	NewFileShouldGetSyncedToGCSAtClose(testDirPath, FileName1, t)
+}
+
+func TestNewFileUnderExplicitDirectoryShouldNotGetSyncedToGCSTillClose(t *testing.T) {
+	testDirPath = setup.SetupTestDirectory(LocalFileTestDirInBucket)
+	// Make explicit directory.
+	operations.CreateDirectory(path.Join(testDirPath, ExplicitDirName), t)
+
+	// Validate.
+	NewFileShouldGetSyncedToGCSAtClose(testDirPath, path.Join(ExplicitDirName, ExplicitFileName1), t)
+}
+
+func TestCreateNewFileWhenSameFileExistsOnGCS(t *testing.T) {
+	testDirPath = setup.SetupTestDirectory(LocalFileTestDirInBucket)
+	// Create a local file.
+	_, fh := CreateLocalFileInTestDir(testDirPath, FileName1, t)
+
+	// Create a file on GCS with the same name.
+	CreateObjectInGCSTestDir(FileName1, GCSFileContent, t)
+
+	// Write to local file.
+	operations.WriteWithoutClose(fh, FileContents, t)
+	// Close the local file and ensure that the content on GCS is not overwritten.
+	CloseFileAndValidateObjectContentsFromGCS(fh, FileName1, GCSFileContent, t)
+}

tools/integration_tests/local_file/helpers/gcs_helper.go

@@ -0,0 +1,103 @@
+// Copyright 2023 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package helpers
+
+import (
+	"os"
+	"path"
+	"strings"
+	"testing"
+
+	"cloud.google.com/go/storage"
+	"github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/client"
+	"github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/operations"
+	"golang.org/x/net/context"
+)
+
+const (
+	FileName1                = "foo1"
+	ExplicitDirName          = "explicit"
+	ExplicitFileName1        = "explicitFile1"
+	FilePerms                = 0644
+	FileContents             = "teststring"
+	GCSFileContent           = "gcsContent"
+	LocalFileTestDirInBucket = "LocalFileTest"
+	ReadSize                 = 1024
+)
+
+var StorageClient *storage.Client
+var Ctx context.Context
+
+func ValidateObjectNotFoundErrOnGCS(fileName string, t *testing.T) {
+	_, err := client.ReadObjectFromGCS(
+		StorageClient,
+		path.Join(LocalFileTestDirInBucket, fileName),
+		ReadSize,
+		Ctx)
+	if err == nil || !strings.Contains(err.Error(), "storage: object doesn't exist") {
+		t.Fatalf("Incorrect error returned from GCS for file %s: %v", fileName, err)
+	}
+}
+
+func ValidateObjectContentsFromGCS(fileName string, expectedContent string, t *testing.T) {
+	gotContent, err := client.ReadObjectFromGCS(
+		StorageClient,
+		path.Join(LocalFileTestDirInBucket, fileName),
+		ReadSize,
+		Ctx)
+	if err != nil {
+		t.Fatalf("Error while reading synced local file from GCS, Err: %v", err)
+	}
+
+	if expectedContent != gotContent {
+		t.Fatalf("GCS file %s content mismatch. Got: %s, Expected: %s ", fileName, gotContent, expectedContent)
+	}
+}
+
+func CloseFileAndValidateObjectContentsFromGCS(f *os.File, fileName string, contents string, t *testing.T) {
+	operations.CloseFileShouldNotThrowError(f, t)
+	ValidateObjectContentsFromGCS(fileName, contents, t)
+}
+
+func WritingToLocalFileShouldNotWriteToGCS(fh *os.File, fileName string, t *testing.T) {
+	operations.WriteWithoutClose(fh, FileContents, t)
+	ValidateObjectNotFoundErrOnGCS(fileName, t)
+}
+
+func NewFileShouldGetSyncedToGCSAtClose(testDirPath, fileName string, t *testing.T) {
+	// Create a local file.
+	_, fh := CreateLocalFileInTestDir(testDirPath, fileName, t)
+
+	// Writing contents to local file shouldn't create file on GCS.
+	WritingToLocalFileShouldNotWriteToGCS(fh, fileName, t)
+
+	// Close the file and validate if the file is created on GCS.
+	CloseFileAndValidateObjectContentsFromGCS(fh, fileName, FileContents, t)
+}
+
+func CreateObjectInGCSTestDir(fileName, content string, t *testing.T) {
+	objectName := path.Join(LocalFileTestDirInBucket, fileName)
+	err := client.CreateObjectOnGCS(StorageClient, objectName, content, Ctx)
+	if err != nil {
+		t.Fatalf("Create Object %s on GCS: %v.", objectName, err)
+	}
+}
+
+func CreateLocalFileInTestDir(testDirPath, fileName string, t *testing.T) (string, *os.File) {
+	filePath := path.Join(testDirPath, fileName)
+	fh := operations.CreateFile(filePath, FilePerms, t)
+	ValidateObjectNotFoundErrOnGCS(fileName, t)
+	return filePath, fh
+}

tools/integration_tests/local_file/local_file_test.go

@@ -0,0 +1,98 @@
+// Copyright 2023 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Provides integration tests for file and directory operations.
+
+package local_file_test
+
+import (
+	"context"
+	"log"
+	"os"
+	"path"
+	"testing"
+	"time"
+
+	"github.com/googlecloudplatform/gcsfuse/internal/config"
+	"github.com/googlecloudplatform/gcsfuse/tools/integration_tests/local_file/helpers"
+	"github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/client"
+	"github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/mounting/dynamic_mounting"
+	"github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/mounting/only_dir_mounting"
+	"github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/mounting/static_mounting"
+	"github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/setup"
+)
+
+// testDirPath holds the path to the test subdirectory in the mounted bucket.
+var testDirPath string
+
+func TestMain(m *testing.M) {
+	setup.ParseSetUpFlags()
+
+	helpers.Ctx = context.Background()
+	var cancel context.CancelFunc
+	var err error
+
+	setup.ExitWithFailureIfBothTestBucketAndMountedDirectoryFlagsAreNotSet()
+
+	if setup.MountedDirectory() != "" && setup.TestBucket() == "" {
+		log.Print("Both --testbucket and --mountedDirectory should be specified to run mounted directory tests.")
+		os.Exit(1)
+	}
+
+	// Create storage client before running tests.
+	helpers.Ctx, cancel = context.WithTimeout(helpers.Ctx, time.Minute*15)
+	helpers.StorageClient, err = client.CreateStorageClient(helpers.Ctx)
+	if err != nil {
+		log.Fatalf("client.CreateStorageClient: %v", err)
+	}
+
+	// Run tests for mountedDirectory only if --mountedDirectory flag is set.
+	setup.RunTestsForMountedDirectoryFlag(m)
+
+	// Else run tests for testBucket.
+	// Set up test directory.
+	setup.SetUpTestDirForTestBucketFlag()
+	// Set up config file with create-empty-file: false. (default)
+	mountConfig := config.MountConfig{
+		WriteConfig: config.WriteConfig{
+			CreateEmptyFile: false,
+		},
+		LogConfig: config.LogConfig{
+			Severity: config.TRACE,
+		},
+	}
+	configFile := setup.YAMLConfigFile(mountConfig)
+	// Set up flags to run tests on.
+	flags := [][]string{
+		{"--implicit-dirs=true", "--rename-dir-limit=3", "--config-file=" + configFile},
+		{"--implicit-dirs=false", "--rename-dir-limit=3", "--config-file=" + configFile}}
+
+	successCode := static_mounting.RunTests(flags, m)
+
+	if successCode == 0 {
+		successCode = only_dir_mounting.RunTests(flags, m)
+	}
+
+	if successCode == 0 {
+		successCode = dynamic_mounting.RunTests(flags, m)
+	}
+
+	// Close storage client and release resources.
+	helpers.StorageClient.Close()
+	cancel()
+	// Clean up test directory created.
+	setup.CleanupDirectoryOnGCS(path.Join(setup.TestBucket(), helpers.LocalFileTestDirInBucket))
+	setup.RemoveBinFileCopiedForTesting()
+	os.Exit(successCode)
+}

tools/integration_tests/run_tests_mounted_directory.sh

@@ -231,3 +231,14 @@ sudo umount $MOUNT_DIR
 gcsfuse --implicit-dirs $TEST_BUCKET_NAME $MOUNT_DIR
 GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/gzip/...  -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME
 sudo umount $MOUNT_DIR
+
+# package local_file
+# Run test with static mounting. (flags: --implicit-dirs=true)
+gcsfuse --implicit-dirs=true --rename-dir-limit=3 $TEST_BUCKET_NAME $MOUNT_DIR
+GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/local_file/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME
+sudo umount $MOUNT_DIR
+
+# Run test with static mounting. (flags: --implicit-dirs=false)
+gcsfuse --implicit-dirs=false --rename-dir-limit=3 $TEST_BUCKET_NAME $MOUNT_DIR
+GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/local_file/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME
+sudo umount $MOUNT_DIR

tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go

@@ -79,13 +79,19 @@ func executeTestsForDynamicMounting(flags [][]string, m *testing.M) (successCode
 	// mntDir - bucket1, bucket2, bucket3, ...
 	// We will test on passed testBucket and one created bucket.
 
+	// SetDynamicBucketMounted to the passed test bucket.
+	setup.SetDynamicBucketMounted(setup.TestBucket())
 	// Test on testBucket
 	successCode = runTestsOnGivenMountedTestBucket(setup.TestBucket(), flags, rootMntDir, m)
 
 	// Test on created bucket.
+	// SetDynamicBucketMounted to the mounted bucket.
+	setup.SetDynamicBucketMounted(testBucketForDynamicMounting)
 	if successCode == 0 {
 		successCode = runTestsOnGivenMountedTestBucket(testBucketForDynamicMounting, flags, rootMntDir, m)
 	}
+	// Reset SetDynamicBucketMounted to empty after tests are done.
+	setup.SetDynamicBucketMounted("")
 
 	// Setting back the original mntDir after testing.
 	setup.SetMntDir(rootMntDir)

tools/integration_tests/util/mounting/only_dir_mounting/only_dir_mounting.go

@@ -57,6 +57,8 @@ func mountGcsFuseForFlagsAndExecuteTests(flags [][]string, dir string, m *testin
 }
 
 func executeTestsForOnlyDirMounting(flags [][]string, m *testing.M) (successCode int) {
+	// Set onlyDirMounted value to the directory being mounted.
+	setup.SetOnlyDirMounted(DirectoryInTestBucket)
 	mountDirInBucket := path.Join(setup.TestBucket(), DirectoryInTestBucket)
 	// Clean the bucket.
 
@@ -77,6 +79,8 @@ func executeTestsForOnlyDirMounting(flags [][]string, m *testing.M) (successCode
 	// Clean the bucket after testing.
 	setup.RunScriptForTestData("../util/mounting/only_dir_mounting/testdata/delete_objects.sh", setup.TestBucket())
 
+	// Reset onlyDirMounted value to empty string after only dir mount tests are done.
+	setup.SetOnlyDirMounted("")
 	return
 }