From 5b35898e3db64932ae570d44e06dae7d08d6a5ba Mon Sep 17 00:00:00 2001 From: Tulsishah <46474643+Tulsishah@users.noreply.github.com> Date: Tue, 1 Aug 2023 12:44:36 +0530 Subject: [PATCH] Random read integration tests (#1247) * adding tests for random read files * fixing comments * creating common function * small fix * fixing comment * creating common function * small fix * merge * fixing comments and fixing typos in naming * renaming function name * renaming function name --- .../random_read_large_file_test.go | 59 +++++++++++++++++++ .../read_large_files/read_large_files_test.go | 18 ++++++ ...ly_test.go => seq_read_large_file_test.go} | 17 +----- .../write_content_of_fix_size_in_file.sh | 1 - .../readonly/readonly_test.go | 1 - .../perisistent_mounting.go | 8 +-- .../static_mounting/static_mounting.go | 4 +- .../util/operations/file_operations.go | 37 ++++++++++++ 8 files changed, 123 insertions(+), 22 deletions(-) create mode 100644 tools/integration_tests/read_large_files/random_read_large_file_test.go rename tools/integration_tests/read_large_files/{read_one_large_file_sequentially_test.go => seq_read_large_file_test.go} (74%) diff --git a/tools/integration_tests/read_large_files/random_read_large_file_test.go b/tools/integration_tests/read_large_files/random_read_large_file_test.go new file mode 100644 index 0000000000..35317e8e0e --- /dev/null +++ b/tools/integration_tests/read_large_files/random_read_large_file_test.go @@ -0,0 +1,59 @@ +// Copyright 2023 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package read_large_files + +import ( + "bytes" + "math/rand" + "os" + "path" + "testing" + + "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/setup" +) + +func TestReadLargeFileRandomly(t *testing.T) { + // Clean the mountedDirectory before running test. + setup.CleanMntDir() + + fileInLocalDisk := path.Join(os.Getenv("HOME"), FiveHundredMBFile) + file := path.Join(setup.MntDir(), FiveHundredMBFile) + // Create and copy the local file in mountedDirectory. + createFileOnDiskAndCopyToMntDir(fileInLocalDisk, file, FiveHundredMB, t) + + for i := 0; i < NumberOfRandomReadCalls; i++ { + offset := rand.Int63n(MaxReadableByteFromFile - MinReadableByteFromFile) + // Randomly read the data from file in mountedDirectory. + content, err := operations.ReadChunkFromFile(file, ChunkSize, offset) + if err != nil { + t.Errorf("Error in reading file: %v", err) + } + + // Read actual content from file located in local disk. + actualContent, err := operations.ReadChunkFromFile(fileInLocalDisk, ChunkSize, offset) + if err != nil { + t.Errorf("Error in reading file: %v", err) + } + + // Compare actual content and expect content. + if bytes.Equal(actualContent, content) == false { + t.Errorf("Error in reading file randomly.") + } + } + + // Removing file after testing. + operations.RemoveFile(fileInLocalDisk) +} diff --git a/tools/integration_tests/read_large_files/read_large_files_test.go b/tools/integration_tests/read_large_files/read_large_files_test.go index 73eb532990..3ebb4d381f 100644 --- a/tools/integration_tests/read_large_files/read_large_files_test.go +++ b/tools/integration_tests/read_large_files/read_large_files_test.go @@ -18,12 +18,22 @@ package read_large_files import ( "log" "os" + "strconv" "testing" "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/mounting/static_mounting" + "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/operations" "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/setup" ) +const OneMB = 1024 * 1024 +const FiveHundredMB = 500 * OneMB +const FiveHundredMBFile = "fiveHundredMBFile.txt" +const ChunkSize = 200 * OneMB +const NumberOfRandomReadCalls = 200 +const MinReadableByteFromFile = 0 +const MaxReadableByteFromFile = 500 * OneMB + func TestMain(m *testing.M) { setup.ParseSetUpFlags() @@ -48,3 +58,11 @@ func TestMain(m *testing.M) { os.Exit(successCode) } + +func createFileOnDiskAndCopyToMntDir(fileInLocalDisk string, fileInMntDir string, fileSize int, t *testing.T) { + setup.RunScriptForTestData("testdata/write_content_of_fix_size_in_file.sh", fileInLocalDisk, strconv.Itoa(fileSize)) + err := operations.CopyFile(fileInLocalDisk, fileInMntDir) + if err != nil { + t.Errorf("Error in copying file:%v", err) + } +} diff --git a/tools/integration_tests/read_large_files/read_one_large_file_sequentially_test.go b/tools/integration_tests/read_large_files/seq_read_large_file_test.go similarity index 74% rename from tools/integration_tests/read_large_files/read_one_large_file_sequentially_test.go rename to tools/integration_tests/read_large_files/seq_read_large_file_test.go index a75080ab9a..c1f7fc117a 100644 --- a/tools/integration_tests/read_large_files/read_one_large_file_sequentially_test.go +++ b/tools/integration_tests/read_large_files/seq_read_large_file_test.go @@ -18,34 +18,23 @@ import ( "bytes" "os" "path" - "strconv" "testing" "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/operations" "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/setup" ) -const FiveHundredMB = 500 * 1024 * 1024 -const FiveHundredMBFile = "fiveHundredMBFile.txt" -const chunkSize = 200 * 1024 * 1024 - func TestReadLargeFileSequentially(t *testing.T) { // Clean the mountedDirectory before running test. setup.CleanMntDir() - // Create file of 500 MB with random data in local disk. fileInLocalDisk := path.Join(os.Getenv("HOME"), FiveHundredMBFile) - setup.RunScriptForTestData("testdata/write_content_of_fix_size_in_file.sh", fileInLocalDisk, strconv.Itoa(FiveHundredMB)) - - // Copy the file in mounted directory. file := path.Join(setup.MntDir(), FiveHundredMBFile) - err := operations.CopyFile(fileInLocalDisk, file) - if err != nil { - t.Errorf("Error in copying file:%v", err) - } + // Create and copy the local file in mountedDirectory. + createFileOnDiskAndCopyToMntDir(fileInLocalDisk, file, FiveHundredMB, t) // Sequentially read the data from file. - content, err := operations.ReadFileSequentially(file, chunkSize) + content, err := operations.ReadFileSequentially(file, ChunkSize) if err != nil { t.Errorf("Error in reading file: %v", err) } diff --git a/tools/integration_tests/read_large_files/testdata/write_content_of_fix_size_in_file.sh b/tools/integration_tests/read_large_files/testdata/write_content_of_fix_size_in_file.sh index 60f66b60d0..c060f00b28 100644 --- a/tools/integration_tests/read_large_files/testdata/write_content_of_fix_size_in_file.sh +++ b/tools/integration_tests/read_large_files/testdata/write_content_of_fix_size_in_file.sh @@ -14,7 +14,6 @@ FILE_PATH=$1 FILE_SIZE=$2 -TEST_BUCKET=$3 # It will write filesize random data in a file. head -c $FILE_SIZE $FILE_PATH diff --git a/tools/integration_tests/readonly/readonly_test.go b/tools/integration_tests/readonly/readonly_test.go index 727e93c15e..49917aa2bf 100644 --- a/tools/integration_tests/readonly/readonly_test.go +++ b/tools/integration_tests/readonly/readonly_test.go @@ -81,7 +81,6 @@ func TestMain(m *testing.M) { successCode := static_mounting.RunTests(flags, m) if successCode == 0 { - // Test for viewer permission on test bucket. successCode = persistent_mounting.RunTests(flags, m) } diff --git a/tools/integration_tests/util/mounting/persistent_mounting/perisistent_mounting.go b/tools/integration_tests/util/mounting/persistent_mounting/perisistent_mounting.go index a664237192..688d85e9bd 100644 --- a/tools/integration_tests/util/mounting/persistent_mounting/perisistent_mounting.go +++ b/tools/integration_tests/util/mounting/persistent_mounting/perisistent_mounting.go @@ -39,7 +39,7 @@ func makePersistentMountingArgs(flags []string) (args []string, err error) { return } -func mountGcsfuseWithStaticMounting(flags []string) (err error) { +func mountGcsfuseWithPersistentMounting(flags []string) (err error) { defaultArg := []string{setup.TestBucket(), setup.MntDir(), "-o", @@ -69,11 +69,11 @@ func mountGcsfuseWithStaticMounting(flags []string) (err error) { return err } -func executeTestsForStatingMounting(flags [][]string, m *testing.M) (successCode int) { +func executeTestsForPersistentMounting(flags [][]string, m *testing.M) (successCode int) { var err error for i := 0; i < len(flags); i++ { - if err = mountGcsfuseWithStaticMounting(flags[i]); err != nil { + if err = mountGcsfuseWithPersistentMounting(flags[i]); err != nil { setup.LogAndExit(fmt.Sprintf("mountGcsfuse: %v\n", err)) } setup.ExecuteTestForFlagsSet(flags[i], m) @@ -82,7 +82,7 @@ func executeTestsForStatingMounting(flags [][]string, m *testing.M) (successCode } func RunTests(flags [][]string, m *testing.M) (successCode int) { - successCode = executeTestsForStatingMounting(flags, m) + successCode = executeTestsForPersistentMounting(flags, m) log.Printf("Test log: %s\n", setup.LogFile()) diff --git a/tools/integration_tests/util/mounting/static_mounting/static_mounting.go b/tools/integration_tests/util/mounting/static_mounting/static_mounting.go index 090b783025..7bfaf72f25 100644 --- a/tools/integration_tests/util/mounting/static_mounting/static_mounting.go +++ b/tools/integration_tests/util/mounting/static_mounting/static_mounting.go @@ -41,7 +41,7 @@ func mountGcsfuseWithStaticMounting(flags []string) (err error) { return err } -func executeTestsForStatingMounting(flags [][]string, m *testing.M) (successCode int) { +func executeTestsForStaticMounting(flags [][]string, m *testing.M) (successCode int) { var err error for i := 0; i < len(flags); i++ { @@ -54,7 +54,7 @@ func executeTestsForStatingMounting(flags [][]string, m *testing.M) (successCode } func RunTests(flags [][]string, m *testing.M) (successCode int) { - successCode = executeTestsForStatingMounting(flags, m) + successCode = executeTestsForStaticMounting(flags, m) log.Printf("Test log: %s\n", setup.LogFile()) diff --git a/tools/integration_tests/util/operations/file_operations.go b/tools/integration_tests/util/operations/file_operations.go index 8f36290c91..d2b20899cb 100644 --- a/tools/integration_tests/util/operations/file_operations.go +++ b/tools/integration_tests/util/operations/file_operations.go @@ -236,3 +236,40 @@ func WriteFileSequentially(filePath string, fileSize int64, chunkSize int64) (er } return } + +func ReadChunkFromFile(filePath string, chunkSize int64, offset int64) (chunk []byte, err error) { + chunk = make([]byte, chunkSize) + + file, err := os.OpenFile(filePath, os.O_RDONLY, FilePermission_0600) + if err != nil { + log.Printf("Error in opening file:%v", err) + return + } + + f, err := os.Stat(filePath) + if err != nil { + log.Printf("Error in stating file:%v", err) + return + } + + // Closing the file at the end. + defer CloseFile(file) + + var numberOfBytes int + + // Reading chunk size randomly from the file. + numberOfBytes, err = file.ReadAt(chunk, offset) + if err == io.EOF { + err = nil + } + if err != nil { + return + } + + // The number of bytes read is not equal to 200MB. + if int64(numberOfBytes) != chunkSize && int64(numberOfBytes) != f.Size()-offset { + log.Printf("Incorrect number of bytes read from file.") + } + + return +}