Integration tests to verify that file creation does not creates file on GCS until synced.

tools/integration_tests/implicit_dir/local_file_test.go

@@ -45,21 +45,21 @@ func TestNewFileUnderImplicitDirectoryShouldNotGetSyncedToGCSTillClose(t *testin
 	CreateImplicitDir(ctx, storageClient, testDirName, t)
 	fileName := path.Join(ImplicitDirName, FileName1)
 
-	fh := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName, t)
+	_, fh := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName, t)
 	operations.WriteWithoutClose(fh, FileContents, t)
 	ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, fileName, t)
 
 	// Validate.
-	CloseFileAndValidateContentFromGCS(ctx, storageClient, testDirName, fh, fileName, FileContents, t)
+	CloseFileAndValidateContentFromGCS(ctx, storageClient, fh, testDirName, fileName, FileContents, t)
 }
 
 func TestReadDirForImplicitDirWithLocalFile(t *testing.T) {
 	testDirPath = setup.SetupTestDirectory(testDirName)
 	CreateImplicitDir(ctx, storageClient, testDirName, t)
 	fileName1 := path.Join(ImplicitDirName, FileName1)
 	fileName2 := path.Join(ImplicitDirName, FileName2)
-	fh1 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName1, t)
-	fh2 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName2, t)
+	_, fh1 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName1, t)
+	_, fh2 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName2, t)
 
 	// Attempt to list implicit directory.
 	entries := operations.ReadDirectory(path.Join(testDirPath, ImplicitDirName), t)
@@ -68,10 +68,10 @@ func TestReadDirForImplicitDirWithLocalFile(t *testing.T) {
 	operations.VerifyCountOfDirectoryEntries(3, len(entries), t)
 	operations.VerifyFileEntry(entries[0], FileName1, 0, t)
 	operations.VerifyFileEntry(entries[1], FileName2, 0, t)
-	operations.VerifyFileEntry(entries[2], ImplicitFileName1, ImplicitFileSize, t)
+	operations.VerifyFileEntry(entries[2], ImplicitFileName1, GCSFileSize, t)
 	// Close the local files.
-	CloseFileAndValidateContentFromGCS(ctx, storageClient, testDirName, fh1, fileName1, "", t)
-	CloseFileAndValidateContentFromGCS(ctx, storageClient, testDirName, fh2, fileName2, "", t)
+	CloseFileAndValidateContentFromGCS(ctx, storageClient, fh1, testDirName, fileName1, "", t)
+	CloseFileAndValidateContentFromGCS(ctx, storageClient, fh2, testDirName, fileName2, "", t)
 }
 
 func TestRecursiveListingWithLocalFiles(t *testing.T) {
@@ -88,13 +88,13 @@ func TestRecursiveListingWithLocalFiles(t *testing.T) {
 	fileName2 := path.Join(ExplicitDirName, ExplicitFileName1)
 	fileName3 := path.Join(ImplicitDirName, FileName2)
 	// Create local file in mnt/ dir.
-	fh1 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, FileName1, t)
+	_, fh1 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, FileName1, t)
 	// Create explicit dir with 1 local file.
 	operations.CreateDirectory(path.Join(testDirPath, ExplicitDirName), t)
-	fh2 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName2, t)
+	_, fh2 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName2, t)
 	// Create implicit dir with 1 local file1 and 1 synced file.
 	CreateImplicitDir(ctx, storageClient, testDirName, t)
-	fh3 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName3, t)
+	_, fh3 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName3, t)
 
 	// Recursively list mntDir/ directory.
 	err := filepath.WalkDir(testDirPath,
@@ -130,7 +130,7 @@ func TestRecursiveListingWithLocalFiles(t *testing.T) {
 				// numberOfObjects = 2
 				operations.VerifyCountOfDirectoryEntries(2, len(objs), t)
 				operations.VerifyFileEntry(objs[0], FileName2, 0, t)
-				operations.VerifyFileEntry(objs[1], ImplicitFileName1, ImplicitFileSize, t)
+				operations.VerifyFileEntry(objs[1], ImplicitFileName1, GCSFileSize, t)
 			}
 			return nil
 		})
@@ -139,7 +139,7 @@ func TestRecursiveListingWithLocalFiles(t *testing.T) {
 	if err != nil {
 		t.Errorf("filepath.WalkDir() err: %v", err)
 	}
-	CloseFileAndValidateContentFromGCS(ctx, storageClient, testDirName, fh1, FileName1, "", t)
-	CloseFileAndValidateContentFromGCS(ctx, storageClient, testDirName, fh2, fileName2, "", t)
-	CloseFileAndValidateContentFromGCS(ctx, storageClient, testDirName, fh3, fileName3, "", t)
+	CloseFileAndValidateContentFromGCS(ctx, storageClient, fh1, testDirName, FileName1, "", t)
+	CloseFileAndValidateContentFromGCS(ctx, storageClient, fh2, testDirName, fileName2, "", t)
+	CloseFileAndValidateContentFromGCS(ctx, storageClient, fh3, testDirName, fileName3, "", t)
 }

tools/integration_tests/local_file/create_file_test.go

@@ -0,0 +1,57 @@
+// Copyright 2023 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Provides integration tests for create local file.
+package local_file_test
+
+import (
+	"path"
+	"testing"
+
+	. "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/client"
+	"github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/operations"
+	"github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/setup"
+)
+
+func TestNewFileShouldNotGetSyncedToGCSTillClose(t *testing.T) {
+	testDirPath = setup.SetupTestDirectory(testDirName)
+
+	// Validate.
+	NewFileShouldGetSyncedToGCSAtClose(ctx, storageClient, testDirPath, FileName1, t)
+}
+
+func TestNewFileUnderExplicitDirectoryShouldNotGetSyncedToGCSTillClose(t *testing.T) {
+	testDirPath = setup.SetupTestDirectory(testDirName)
+	// Make explicit directory.
+	operations.CreateDirectory(path.Join(testDirPath, ExplicitDirName), t)
+
+	// Validate.
+	NewFileShouldGetSyncedToGCSAtClose(ctx, storageClient, testDirPath, path.Join(ExplicitDirName, ExplicitFileName1), t)
+}
+
+func TestCreateNewFileWhenSameFileExistsOnGCS(t *testing.T) {
+	testDirPath = setup.SetupTestDirectory(testDirName)
+	// Create a local file.
+	_, fh := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, FileName1, t)
+
+	// Create a file on GCS with the same name.
+	CreateObjectInGCSTestDir(ctx, storageClient, testDirName, FileName1, GCSFileContent, t)
+
+	// Write to local file.
+	operations.WriteWithoutClose(fh, FileContents, t)
+	// Close the local file.
+	operations.CloseFileShouldNotThrowError(fh, t)
+	//  Ensure that the content on GCS is not overwritten.
+	ValidateObjectContentsFromGCS(ctx, storageClient, testDirName, FileName1, GCSFileContent, t)
+}

tools/integration_tests/local_file/local_file_test.go

@@ -0,0 +1,93 @@
+// Copyright 2023 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Provides integration tests for file and directory operations.
+
+package local_file_test
+
+import (
+	"context"
+	"log"
+	"os"
+	"path"
+	"testing"
+	"time"
+
+	"cloud.google.com/go/storage"
+	"github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/client"
+	"github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/mounting/dynamic_mounting"
+	"github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/mounting/only_dir_mounting"
+	"github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/mounting/static_mounting"
+	"github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/setup"
+)
+
+const (
+	testDirName = "LocalFileTest"
+)
+
+var (
+	testDirPath   string
+	storageClient *storage.Client
+	ctx           context.Context
+)
+
+func TestMain(m *testing.M) {
+	setup.ParseSetUpFlags()
+
+	ctx = context.Background()
+	var cancel context.CancelFunc
+	var err error
+
+	setup.ExitWithFailureIfBothTestBucketAndMountedDirectoryFlagsAreNotSet()
+
+	// Create storage client before running tests.
+	ctx, cancel = context.WithTimeout(ctx, time.Minute*15)
+	storageClient, err = client.CreateStorageClient(ctx)
+	if err != nil {
+		log.Fatalf("client.CreateStorageClient: %v", err)
+	}
+
+	// To run mountedDirectory tests, we need both testBucket and mountedDirectory
+	// flags to be set, as local_file tests validates content from the bucket.
+	if setup.RunMountedDirectoryTestsWithTestBucket() {
+		setup.RunTestsForMountedDirectoryFlag(m)
+	}
+
+	// Else run tests for testBucket.
+	// Set up test directory.
+	setup.SetUpTestDirForTestBucketFlag()
+
+	// Set up flags to run tests on.
+	flags := [][]string{
+		{"--implicit-dirs=true", "--rename-dir-limit=3"},
+		{"--implicit-dirs=false", "--rename-dir-limit=3"}}
+
+	successCode := static_mounting.RunTests(flags, m)
+
+	if successCode == 0 {
+		successCode = only_dir_mounting.RunTests(flags, m)
+	}
+
+	if successCode == 0 {
+		successCode = dynamic_mounting.RunTests(flags, m)
+	}
+
+	// Close storage client and release resources.
+	storageClient.Close()
+	cancel()
+	// Clean up test directory created.
+	setup.CleanupDirectoryOnGCS(path.Join(setup.TestBucket(), testDirName))
+	setup.RemoveBinFileCopiedForTesting()
+	os.Exit(successCode)
+}

tools/integration_tests/run_tests_mounted_directory.sh

@@ -231,3 +231,14 @@ sudo umount $MOUNT_DIR
 gcsfuse --implicit-dirs $TEST_BUCKET_NAME $MOUNT_DIR
 GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/gzip/...  -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME
 sudo umount $MOUNT_DIR
+
+# package local_file
+# Run test with static mounting. (flags: --implicit-dirs=true)
+gcsfuse --implicit-dirs=true --rename-dir-limit=3 $TEST_BUCKET_NAME $MOUNT_DIR
+GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/local_file/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME
+sudo umount $MOUNT_DIR
+
+# Run test with static mounting. (flags: --implicit-dirs=false)
+gcsfuse --implicit-dirs=false --rename-dir-limit=3 $TEST_BUCKET_NAME $MOUNT_DIR
+GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/local_file/... -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR --testbucket=$TEST_BUCKET_NAME
+sudo umount $MOUNT_DIR

tools/integration_tests/util/client/gcs_helper.go

@@ -26,17 +26,17 @@ import (
 )
 
 const (
-	FileName1            = "foo1"
-	FileName2            = "foo2"
-	ExplicitDirName      = "explicit"
-	ExplicitFileName1    = "explicitFile1"
-	ImplicitDirName      = "implicit"
-	ImplicitFileName1    = "implicitFile1"
-	FileContents         = "testString"
-	ImplicitFileContents = "GCSteststring"
-	ImplicitFileSize     = 13
-	FilePerms            = 0644
-	ReadSize             = 1024
+	FileName1         = "foo1"
+	FileName2         = "foo2"
+	ExplicitDirName   = "explicit"
+	ExplicitFileName1 = "explicitFile1"
+	ImplicitDirName   = "implicit"
+	ImplicitFileName1 = "implicitFile1"
+	FileContents      = "testString"
+	GCSFileContent    = "GCSteststring"
+	GCSFileSize       = 13
+	FilePerms         = 0644
+	ReadSize          = 1024
 )
 
 func CreateImplicitDir(ctx context.Context, storageClient *storage.Client,
@@ -45,7 +45,7 @@ func CreateImplicitDir(ctx context.Context, storageClient *storage.Client,
 		ctx,
 		storageClient,
 		path.Join(testDirName, ImplicitDirName, ImplicitFileName1),
-		ImplicitFileContents)
+		GCSFileContent)
 	if err != nil {
 		t.Errorf("Error while creating implicit directory, err: %v", err)
 	}
@@ -59,7 +59,7 @@ func ValidateObjectNotFoundErrOnGCS(ctx context.Context, storageClient *storage.
 	}
 }
 
-func validateObjectContentsFromGCS(ctx context.Context, storageClient *storage.Client,
+func ValidateObjectContentsFromGCS(ctx context.Context, storageClient *storage.Client,
 	testDirName string, fileName string, expectedContent string, t *testing.T) {
 	gotContent, err := ReadObjectFromGCS(ctx, storageClient, path.Join(testDirName, fileName), ReadSize)
 	if err != nil {
@@ -72,21 +72,48 @@ func validateObjectContentsFromGCS(ctx context.Context, storageClient *storage.C
 }
 
 func CloseFileAndValidateContentFromGCS(ctx context.Context, storageClient *storage.Client,
-	testDirName string, fh *os.File, fileName, content string, t *testing.T) {
+	fh *os.File, testDirName, fileName, content string, t *testing.T) {
 	operations.CloseFileShouldNotThrowError(fh, t)
-	validateObjectContentsFromGCS(ctx, storageClient, testDirName, fileName, content, t)
+	ValidateObjectContentsFromGCS(ctx, storageClient, testDirName, fileName, content, t)
 }
 
 func CreateLocalFileInTestDir(ctx context.Context, storageClient *storage.Client,
-	testDirPath, fileName string, t *testing.T) (fh *os.File) {
+	testDirPath, fileName string, t *testing.T) (string, *os.File) {
 	filePath := path.Join(testDirPath, fileName)
-	fh = operations.CreateFile(filePath, FilePerms, t)
+	fh := operations.CreateFile(filePath, FilePerms, t)
 	testDirName := getDirName(testDirPath)
 	ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, fileName, t)
-	return
+	return filePath, fh
 }
 
 func getDirName(testDirPath string) string {
 	dirName := testDirPath[strings.LastIndex(testDirPath, "/")+1:]
 	return dirName
 }
+
+func WritingToLocalFileShouldNotWriteToGCS(ctx context.Context, storageClient *storage.Client, fh *os.File, testDirName, fileName string, t *testing.T) {
+	operations.WriteWithoutClose(fh, FileContents, t)
+	ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, fileName, t)
+}
+
+func NewFileShouldGetSyncedToGCSAtClose(ctx context.Context, storageClient *storage.Client,
+	testDirPath, fileName string, t *testing.T) {
+	// Create a local file.
+	_, fh := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName, t)
+
+	// Writing contents to local file shouldn't create file on GCS.
+	testDirName := getDirName(testDirPath)
+	WritingToLocalFileShouldNotWriteToGCS(ctx, storageClient, fh, testDirName, fileName, t)
+
+	// Close the file and validate if the file is created on GCS.
+	CloseFileAndValidateContentFromGCS(ctx, storageClient, fh, testDirName, fileName, FileContents, t)
+}
+
+func CreateObjectInGCSTestDir(ctx context.Context, storageClient *storage.Client,
+	testDirName, fileName, content string, t *testing.T) {
+	objectName := path.Join(testDirName, fileName)
+	err := CreateObjectOnGCS(ctx, storageClient, objectName, content)
+	if err != nil {
+		t.Fatalf("Create Object %s on GCS: %v.", objectName, err)
+	}
+}

tools/integration_tests/util/mounting/dynamic_mounting/dynamic_mounting.go

@@ -79,13 +79,19 @@ func executeTestsForDynamicMounting(flags [][]string, m *testing.M) (successCode
 	// mntDir - bucket1, bucket2, bucket3, ...
 	// We will test on passed testBucket and one created bucket.
 
+	// SetDynamicBucketMounted to the passed test bucket.
+	setup.SetDynamicBucketMounted(setup.TestBucket())
 	// Test on testBucket
 	successCode = runTestsOnGivenMountedTestBucket(setup.TestBucket(), flags, rootMntDir, m)
 
 	// Test on created bucket.
+	// SetDynamicBucketMounted to the mounted bucket.
+	setup.SetDynamicBucketMounted(testBucketForDynamicMounting)
 	if successCode == 0 {
 		successCode = runTestsOnGivenMountedTestBucket(testBucketForDynamicMounting, flags, rootMntDir, m)
 	}
+	// Reset SetDynamicBucketMounted to empty after tests are done.
+	setup.SetDynamicBucketMounted("")
 
 	// Setting back the original mntDir after testing.
 	setup.SetMntDir(rootMntDir)

tools/integration_tests/util/mounting/only_dir_mounting/only_dir_mounting.go

@@ -57,6 +57,8 @@ func mountGcsFuseForFlagsAndExecuteTests(flags [][]string, dir string, m *testin
 }
 
 func executeTestsForOnlyDirMounting(flags [][]string, m *testing.M) (successCode int) {
+	// Set onlyDirMounted value to the directory being mounted.
+	setup.SetOnlyDirMounted(DirectoryInTestBucket)
 	mountDirInBucket := path.Join(setup.TestBucket(), DirectoryInTestBucket)
 	// Clean the bucket.
 
@@ -77,6 +79,8 @@ func executeTestsForOnlyDirMounting(flags [][]string, m *testing.M) (successCode
 	// Clean the bucket after testing.
 	setup.RunScriptForTestData("../util/mounting/only_dir_mounting/testdata/delete_objects.sh", setup.TestBucket())
 
+	// Reset onlyDirMounted value to empty string after only dir mount tests are done.
+	setup.SetOnlyDirMounted("")
 	return
 }
 

tools/integration_tests/util/setup/setup.go

@@ -327,3 +327,13 @@ func CleanupDirectoryOnGCS(directoryPathOnGCS string) {
 			directoryPathOnGCS, err)
 	}
 }
+
+func RunMountedDirectoryTestsWithTestBucket() bool {
+	if MountedDirectory() != "" {
+		if TestBucket() == "" {
+			log.Fatal("Set both --mountedDirectory and --testBucket to run mounted directory tests.")
+		}
+		return true
+	}
+	return false
+}