Sequential write large file integration tests (#1240)

* adding test for writing file of 100MB

* adding test for writing file of 100MB

* adding commands in script

* adding commands in script

* small fix

* increasing chunk size

* small fix

* using min function

* small fix

* adding comment

* fixing lint

* writing 500MB in more redable way

* fixing comments

* fixing comments

* changing chunksize to 20MB

* testing sync

* testing

* merging

* merging

* testing sync

* small fix

tools/integration_tests/run_tests_mounted_directory.sh

@@ -263,3 +263,12 @@ sudo umount $MOUNT_DIR
 gcsfuse --enable-storage-client-library=false --implicit-dirs $TEST_BUCKET_NAME $MOUNT_DIR
 GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/read_large_files/...  -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR
 sudo umount $MOUNT_DIR
+
+# Run integration tests for write_large_files directory with static mounting
+gcsfuse --implicit-dirs $TEST_BUCKET_NAME $MOUNT_DIR
+GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/write_large_files/...  -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR
+sudo umount $MOUNT_DIR
+
+gcsfuse --implicit-dirs --enable-storage-client-library=false $TEST_BUCKET_NAME $MOUNT_DIR
+GODEBUG=asyncpreemptoff=1 go test ./tools/integration_tests/write_large_files/...  -p 1 --integrationTest -v --mountedDirectory=$MOUNT_DIR
+sudo umount $MOUNT_DIR

tools/integration_tests/util/operations/file_operations.go

@@ -16,6 +16,7 @@
 package operations
 
 import (
+	"crypto/rand"
 	"fmt"
 	"io"
 	"log"
@@ -192,3 +193,46 @@ func ReadFileSequentially(filePath string, chunkSize int64) (content []byte, err
 	}
 	return
 }
+
+func WriteFileSequentially(filePath string, fileSize int64, chunkSize int64) (err error) {
+	file, err := os.OpenFile(filePath, os.O_RDWR|syscall.O_DIRECT|os.O_CREATE, FilePermission_0600)
+	if err != nil {
+		log.Printf("Error in opening file:%v", err)
+	}
+
+	// Closing file at the end.
+	defer CloseFile(file)
+
+	var offset int64 = 0
+
+	for offset < fileSize {
+		// Get random chunkSize or remaining filesize data into chunk.
+		if (fileSize - offset) < chunkSize {
+			chunkSize = (fileSize - offset)
+		}
+		chunk := make([]byte, chunkSize)
+		_, err = rand.Read(chunk)
+		if err != nil {
+			log.Fatalf("error while generating random string: %s", err)
+		}
+
+		var numberOfBytes int
+
+		// Writes random chunkSize or remaining filesize data into file.
+		numberOfBytes, err = file.Write(chunk)
+		err = file.Sync()
+		if err != nil {
+			log.Printf("Error in syncing file:%v", err)
+		}
+
+		if err != nil {
+			return
+		}
+		if int64(numberOfBytes) != chunkSize {
+			log.Fatalf("Incorrect number of bytes written in the file.")
+		}
+
+		offset = offset + chunkSize
+	}
+	return
+}

tools/integration_tests/write_large_files/write_large_files_test.go

@@ -0,0 +1,50 @@
+// Copyright 2023 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Provides integration tests for write large files sequentially and randomly.
+package write_large_files
+
+import (
+	"log"
+	"os"
+	"testing"
+
+	"github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/mounting/static_mounting"
+	"github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/setup"
+)
+
+func TestMain(m *testing.M) {
+	setup.ParseSetUpFlags()
+
+	flags := [][]string{{"--implicit-dirs"}, {"--enable-storage-client-library=false", "--implicit-dirs"}}
+
+	setup.ExitWithFailureIfBothTestBucketAndMountedDirectoryFlagsAreNotSet()
+
+	if setup.TestBucket() != "" && setup.MountedDirectory() != "" {
+		log.Print("Both --testbucket and --mountedDirectory can't be specified at the same time.")
+		os.Exit(1)
+	}
+
+	// Run tests for mountedDirectory only if --mountedDirectory flag is set.
+	setup.RunTestsForMountedDirectoryFlag(m)
+
+	// Run tests for testBucket
+	setup.SetUpTestDirForTestBucketFlag()
+
+	successCode := static_mounting.RunTests(flags, m)
+
+	setup.RemoveBinFileCopiedForTesting()
+
+	os.Exit(successCode)
+}

tools/integration_tests/write_large_files/write_one_large_file_sequentially_test.go

@@ -0,0 +1,51 @@
+// Copyright 2023 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package write_large_files
+
+import (
+	"os"
+	"path"
+	"testing"
+
+	"github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/operations"
+	"github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/setup"
+)
+
+const FiveHundredMB = 500 * 1024 * 1024
+const FiveHundredMBFile = "fiveHundredMBFile.txt"
+const ChunkSize = 20 * 1024 * 1024
+
+func TestWriteLargeFileSequentially(t *testing.T) {
+	// Clean the mountedDirectory before running test.
+	setup.CleanMntDir()
+
+	filePath := path.Join(setup.MntDir(), FiveHundredMBFile)
+
+	// Sequentially read the data from file.
+	err := operations.WriteFileSequentially(filePath, FiveHundredMB, ChunkSize)
+	if err != nil {
+		t.Errorf("Error in writing file: %v", err)
+	}
+
+	// Check if 500MB data written in the file.
+	fStat, err := os.Stat(filePath)
+	if err != nil {
+		t.Errorf("Error in stating file:%v", err)
+	}
+
+	if fStat.Size() != FiveHundredMB {
+		t.Errorf("Expecred file size %v found %d", FiveHundredMB, fStat.Size())
+	}
+}