Rename and remove dir integration test for local files (#1385)

* integration tests to verify rename and removeDir operations on local file

* rebase changes

* Empty-Commit

* lint

* review comments

* review comments

tools/integration_tests/local_file/local_file_test.go

@@ -29,6 +29,7 @@ import (
 	"github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/mounting/dynamic_mounting"
 	"github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/mounting/only_dir_mounting"
 	"github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/mounting/static_mounting"
+	"github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/operations"
 	"github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/setup"
 )
 
@@ -42,6 +43,33 @@ var (
 	ctx           context.Context
 )
 
+////////////////////////////////////////////////////////////////////////
+// Helpers
+////////////////////////////////////////////////////////////////////////
+
+func WritingToLocalFileShouldNotWriteToGCS(ctx context.Context, storageClient *storage.Client,
+	fh *os.File, testDirName, fileName string, t *testing.T) {
+	operations.WriteWithoutClose(fh, client.FileContents, t)
+	client.ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, fileName, t)
+}
+
+func NewFileShouldGetSyncedToGCSAtClose(ctx context.Context, storageClient *storage.Client,
+	testDirPath, fileName string, t *testing.T) {
+	// Create a local file.
+	_, fh := client.CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName, t)
+
+	// Writing contents to local file shouldn't create file on GCS.
+	testDirName := client.GetDirName(testDirPath)
+	WritingToLocalFileShouldNotWriteToGCS(ctx, storageClient, fh, testDirName, fileName, t)
+
+	// Close the file and validate if the file is created on GCS.
+	client.CloseFileAndValidateContentFromGCS(ctx, storageClient, fh, testDirName, fileName, client.FileContents, t)
+}
+
+////////////////////////////////////////////////////////////////////////
+// TestMain
+////////////////////////////////////////////////////////////////////////
+
 func TestMain(m *testing.M) {
 	setup.ParseSetUpFlags()
 

tools/integration_tests/local_file/remove_dir_test.go

@@ -0,0 +1,73 @@
+// Copyright 2023 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Provides integration tests for removeDir operation on directories containing local files.
+package local_file_test
+
+import (
+	"path"
+	"testing"
+
+	. "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/client"
+	"github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/operations"
+	"github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/setup"
+)
+
+func TestRmDirOfDirectoryContainingGCSAndLocalFiles(t *testing.T) {
+	testDirPath = setup.SetupTestDirectory(testDirName)
+	// Create explicit directory with one synced and one local file.
+	operations.CreateDirectory(path.Join(testDirPath, ExplicitDirName), t)
+	syncedFile := path.Join(ExplicitDirName, FileName1)
+	localFile := path.Join(ExplicitDirName, FileName2)
+	_, fh1 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, syncedFile, t)
+	CloseFileAndValidateContentFromGCS(ctx, storageClient, fh1, testDirName, syncedFile, "", t)
+	_, fh2 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, localFile, t)
+
+	// Attempt to remove explicit directory.
+	operations.RemoveDir(path.Join(testDirPath, ExplicitDirName))
+
+	// Verify that directory is removed.
+	operations.ValidateNoFileOrDirError(path.Join(testDirPath, ExplicitDirName), t)
+	// Validate writing content to unlinked local file does not throw error.
+	operations.WriteWithoutClose(fh2, FileContents, t)
+	// Validate flush file does not throw error and does not create object on GCS.
+	operations.CloseFileShouldNotThrowError(fh2, t)
+	ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, localFile, t)
+	// Validate synced files are also deleted.
+	ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, syncedFile, t)
+	ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, ExplicitDirName, t)
+}
+
+func TestRmDirOfDirectoryContainingOnlyLocalFiles(t *testing.T) {
+	testDirPath = setup.SetupTestDirectory(testDirName)
+	// Create a directory with two local files.
+	operations.CreateDirectory(path.Join(testDirPath, ExplicitDirName), t)
+	localFile1 := path.Join(ExplicitDirName, FileName1)
+	localFile2 := path.Join(ExplicitDirName, FileName2)
+	_, fh1 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, localFile1, t)
+	_, fh2 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, localFile2, t)
+
+	// Attempt to remove explicit directory.
+	operations.RemoveDir(path.Join(testDirPath, ExplicitDirName))
+
+	// Verify rmDir operation succeeds.
+	operations.ValidateNoFileOrDirError(path.Join(testDirPath, ExplicitDirName), t)
+	// Close the local files and validate they are not present on GCS.
+	operations.CloseFileShouldNotThrowError(fh1, t)
+	ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, localFile1, t)
+	operations.CloseFileShouldNotThrowError(fh2, t)
+	ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, localFile2, t)
+	// Validate directory is also deleted.
+	ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, ExplicitDirName, t)
+}

tools/integration_tests/local_file/rename_test.go

@@ -0,0 +1,128 @@
+// Copyright 2023 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Provides integration tests for rename operation on local files.
+package local_file_test
+
+import (
+	"os"
+	"path"
+	"strings"
+	"testing"
+
+	. "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/client"
+	"github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/operations"
+	"github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/setup"
+)
+
+////////////////////////////////////////////////////////////////////////
+// Helpers
+////////////////////////////////////////////////////////////////////////
+
+func verifyRenameOperationNotSupported(err error, t *testing.T) {
+	if err == nil || !strings.Contains(err.Error(), "operation not supported") {
+		t.Fatalf("os.Rename(), expected err: %s, got err: %v",
+			"operation not supported", err)
+	}
+}
+
+////////////////////////////////////////////////////////////////////////
+// Tests
+////////////////////////////////////////////////////////////////////////
+
+func TestRenameOfLocalFileFails(t *testing.T) {
+	testDirPath = setup.SetupTestDirectory(testDirName)
+	// Create local file with some content.
+	_, fh := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, FileName1, t)
+	WritingToLocalFileShouldNotWriteToGCS(ctx, storageClient, fh, testDirName, FileName1, t)
+
+	// Attempt to rename local file.
+	err := os.Rename(
+		path.Join(testDirPath, FileName1),
+		path.Join(testDirPath, NewFileName))
+
+	// Verify rename operation fails.
+	verifyRenameOperationNotSupported(err, t)
+	// write more content to local file.
+	WritingToLocalFileShouldNotWriteToGCS(ctx, storageClient, fh, testDirName, FileName1, t)
+	// Close the local file.
+	CloseFileAndValidateContentFromGCS(ctx, storageClient, fh, testDirName,
+		FileName1, FileContents+FileContents, t)
+}
+
+func TestRenameOfDirectoryWithLocalFileFails(t *testing.T) {
+	testDirPath = setup.SetupTestDirectory(testDirName)
+	//Create directory with 1 synced and 1 local file.
+	operations.CreateDirectory(path.Join(testDirPath, ExplicitDirName), t)
+	// Create synced file.
+	CreateObjectInGCSTestDir(ctx, storageClient, testDirName,
+		path.Join(ExplicitDirName, FileName1), GCSFileContent, t)
+	// Create local file with some content.
+	_, fh := CreateLocalFileInTestDir(ctx, storageClient, testDirPath,
+		path.Join(ExplicitDirName, FileName2), t)
+	WritingToLocalFileShouldNotWriteToGCS(ctx, storageClient, fh, testDirName,
+		path.Join(ExplicitDirName, FileName2), t)
+
+	// Attempt to rename directory containing local file.
+	err := os.Rename(
+		path.Join(testDirPath, ExplicitDirName),
+		path.Join(testDirPath, NewDirName))
+
+	// Verify rename operation fails.
+	verifyRenameOperationNotSupported(err, t)
+	// Write more content to local file.
+	WritingToLocalFileShouldNotWriteToGCS(ctx, storageClient, fh, testDirName, FileName2, t)
+	// Close the local file.
+	CloseFileAndValidateContentFromGCS(ctx, storageClient, fh, testDirName,
+		path.Join(ExplicitDirName, FileName2), FileContents+FileContents, t)
+}
+
+func TestRenameOfLocalFileSucceedsAfterSync(t *testing.T) {
+	TestRenameOfLocalFileFails(t)
+
+	// Attempt to Rename synced file.
+	err := os.Rename(
+		path.Join(testDirPath, FileName1),
+		path.Join(testDirPath, NewFileName))
+
+	// Validate.
+	if err != nil {
+		t.Fatalf("os.Rename() failed on synced file: %v", err)
+	}
+	ValidateObjectContentsFromGCS(ctx, storageClient, testDirName, NewFileName,
+		FileContents+FileContents, t)
+	ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, FileName1, t)
+}
+
+func TestRenameOfDirectoryWithLocalFileSucceedsAfterSync(t *testing.T) {
+	TestRenameOfDirectoryWithLocalFileFails(t)
+
+	// Attempt to rename directory again after sync.
+	err := os.Rename(
+		path.Join(testDirPath, ExplicitDirName),
+		path.Join(testDirPath, NewDirName))
+
+	// Validate.
+	if err != nil {
+		t.Fatalf("os.Rename() failed on directory containing synced files: %v", err)
+	}
+	ValidateObjectContentsFromGCS(ctx, storageClient, testDirName,
+		path.Join(NewDirName, FileName1), GCSFileContent, t)
+	ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName,
+		path.Join(ExplicitDirName, FileName1), t)
+	ValidateObjectContentsFromGCS(ctx, storageClient, testDirName,
+		path.Join(NewDirName, FileName2), FileContents+FileContents, t)
+	ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName,
+		path.Join(ExplicitDirName, FileName2), t)
+}

tools/integration_tests/util/client/gcs_helper.go

@@ -39,6 +39,8 @@ const (
 	FilePerms          = 0644
 	ReadSize           = 1024
 	SizeTruncate       = 5
+	NewFileName        = "newName"
+	NewDirName         = "newDirName"
 )
 
 func CreateImplicitDir(ctx context.Context, storageClient *storage.Client,
@@ -83,35 +85,16 @@ func CreateLocalFileInTestDir(ctx context.Context, storageClient *storage.Client
 	testDirPath, fileName string, t *testing.T) (string, *os.File) {
 	filePath := path.Join(testDirPath, fileName)
 	fh := operations.CreateFile(filePath, FilePerms, t)
-	testDirName := getDirName(testDirPath)
+	testDirName := GetDirName(testDirPath)
 	ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, fileName, t)
 	return filePath, fh
 }
 
-func getDirName(testDirPath string) string {
+func GetDirName(testDirPath string) string {
 	dirName := testDirPath[strings.LastIndex(testDirPath, "/")+1:]
 	return dirName
 }
 
-func WritingToLocalFileShouldNotWriteToGCS(ctx context.Context, storageClient *storage.Client,
-	fh *os.File, testDirName, fileName string, t *testing.T) {
-	operations.WriteWithoutClose(fh, FileContents, t)
-	ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, fileName, t)
-}
-
-func NewFileShouldGetSyncedToGCSAtClose(ctx context.Context, storageClient *storage.Client,
-	testDirPath, fileName string, t *testing.T) {
-	// Create a local file.
-	_, fh := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, fileName, t)
-
-	// Writing contents to local file shouldn't create file on GCS.
-	testDirName := getDirName(testDirPath)
-	WritingToLocalFileShouldNotWriteToGCS(ctx, storageClient, fh, testDirName, fileName, t)
-
-	// Close the file and validate if the file is created on GCS.
-	CloseFileAndValidateContentFromGCS(ctx, storageClient, fh, testDirName, fileName, FileContents, t)
-}
-
 func CreateObjectInGCSTestDir(ctx context.Context, storageClient *storage.Client,
 	testDirName, fileName, content string, t *testing.T) {
 	objectName := path.Join(testDirName, fileName)

tools/integration_tests/util/operations/validation_helper.go

@@ -0,0 +1,15 @@
+package operations
+
+import (
+	"os"
+	"strings"
+	"testing"
+)
+
+func ValidateNoFileOrDirError(path string, t *testing.T) {
+	_, err := os.Stat(path)
+	if err == nil || !strings.Contains(err.Error(), "no such file or directory") {
+		t.Fatalf("os.Stat(%s). Expected: %s, Got: %v", path,
+			"no such file or directory", err)
+	}
+}