From 7b830b005a36d24ec2bcaa4fc02347f381270d2d Mon Sep 17 00:00:00 2001 From: yuchanns Date: Sat, 13 Jul 2024 01:48:30 +0800 Subject: [PATCH] feat(bindings/go): Add full native support from C to Go. Signed-off-by: Hanchin Hsieh --- .github/workflows/ci_bindings_go.yml | 29 +- bindings/go/DEPENDENCIES.md | 2 +- bindings/go/README.md | 131 ++++-- bindings/go/copy_test.go | 158 +++++++ .../{build_dynamic.go => create_dir_test.go} | 44 +- bindings/go/delete.go | 70 +++ bindings/go/delete_test.go | 78 ++++ bindings/go/error.go | 116 +++++ bindings/go/ffi.go | 136 ++++++ bindings/go/go.mod | 16 +- bindings/go/go.sum | 17 +- bindings/go/list_test.go | 254 +++++++++++ bindings/go/lister.go | 424 ++++++++++++++++++ bindings/go/metadata.go | 179 ++++++++ bindings/go/opendal.go | 234 +++++++--- bindings/go/opendal_test.go | 223 +++++++-- bindings/go/operator.go | 298 ++++++++++++ bindings/go/operator_info.go | 381 ++++++++++++++++ bindings/go/read_test.go | 98 ++++ bindings/go/reader.go | 299 ++++++++++++ bindings/go/rename_test.go | 156 +++++++ bindings/go/stat.go | 157 +++++++ bindings/go/stat_test.go | 143 ++++++ bindings/go/types.go | 286 ++++++++++++ bindings/go/write.go | 152 +++++++ bindings/go/write_test.go | 101 +++++ 26 files changed, 4020 insertions(+), 162 deletions(-) create mode 100644 bindings/go/copy_test.go rename bindings/go/{build_dynamic.go => create_dir_test.go} (50%) create mode 100644 bindings/go/delete.go create mode 100644 bindings/go/delete_test.go create mode 100644 bindings/go/error.go create mode 100644 bindings/go/ffi.go create mode 100644 bindings/go/list_test.go create mode 100644 bindings/go/lister.go create mode 100644 bindings/go/metadata.go create mode 100644 bindings/go/operator.go create mode 100644 bindings/go/operator_info.go create mode 100644 bindings/go/read_test.go create mode 100644 bindings/go/reader.go create mode 100644 bindings/go/rename_test.go create mode 100644 bindings/go/stat.go create mode 100644 bindings/go/stat_test.go create mode 100644 bindings/go/types.go create mode 100644 bindings/go/write.go create mode 100644 bindings/go/write_test.go diff --git a/.github/workflows/ci_bindings_go.yml b/.github/workflows/ci_bindings_go.yml index 5ec7c3cc43a..edb85392181 100644 --- a/.github/workflows/ci_bindings_go.yml +++ b/.github/workflows/ci_bindings_go.yml @@ -46,32 +46,9 @@ jobs: steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 - with: - go-version: '1.20' - - - name: Setup Rust toolchain - uses: ./.github/actions/setup - - - name: Build c binding - working-directory: bindings/c - run: make build - - - name: Check diff - run: git diff --exit-code - - - name: Generate pkg-config file - run: | - echo "libdir=$(pwd)/bindings/c/target/debug/" >> opendal_c.pc - echo "includedir=$(pwd)/bindings/c/include/" >> opendal_c.pc - echo "Name: opendal_c" >> opendal_c.pc - echo "Description: opendal c binding" >> opendal_c.pc - echo "Version: 0.0.1" >> opendal_c.pc - echo "Libs: -L\${libdir} -lopendal_c" >> opendal_c.pc - echo "Cflags: -I\${includedir}" >> opendal_c.pc - - echo "PKG_CONFIG_PATH=$(pwd)" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=$(pwd)/bindings/c/target/debug" >> $GITHUB_ENV - name: Run tests + env: + OPENDAL_TEST: "memory" working-directory: bindings/go - run: go test -tags dynamic . + run: CGO_ENABLE=0 go test -v -run TestBehavior diff --git a/bindings/go/DEPENDENCIES.md b/bindings/go/DEPENDENCIES.md index 46c744cc286..9e3222eeb3d 100644 --- a/bindings/go/DEPENDENCIES.md +++ b/bindings/go/DEPENDENCIES.md @@ -1,4 +1,4 @@ # Dependencies OpenDAL Go Binding is based on the C Binding. -There are no extra runtime dependencies except those conveyed from C Binding. +Installation of libffi is required. diff --git a/bindings/go/README.md b/bindings/go/README.md index a51567ea5da..12871e8aacf 100644 --- a/bindings/go/README.md +++ b/bindings/go/README.md @@ -2,60 +2,123 @@ ![](https://img.shields.io/badge/status-unreleased-red) -opendal-go requires opendal-c to be installed. +opendal-go is a **Native** support Go binding without CGO enabled and is built on top of opendal-c. -```shell -cd bindings/c -make build +```bash +go get github.com/apache/opendal/bindings/go@latest ``` -You will find `libopendal_c.so` under `{root}/target`. +opendal-go requires **libffi** to be installed. -Then, we need to add a `opendal_c.pc` files +## Basic Usage -```pc -libdir=/path/to/opendal/target/debug/ -includedir=/path/to/opendal/bindings/c/include/ +```go +package main -Name: opendal_c -Description: opendal c binding -Version: +import ( + "fmt" + "os" -Libs: -L${libdir} -lopendal_c -Cflags: -I${includedir} -``` + "github.com/yuchanns/opendal-go-services/memory" + "github.com/apache/opendal/bindings/go" +) -And set the `PKG_CONFIG_PATH` environment variable to the directory where `opendal_c.pc` is located. +func main() { + // Initialize a new in-memory operator + op, err := opendal.NewOperator(memory.Scheme, opendal.OperatorOptions{}) + if err != nil { + panic(err) + } + defer op.Close() -```shell -export PKG_CONFIG_PATH=/dir/of/opendal_c.pc -``` + // Write data to a file named "test" + err = op.Write("test", []byte("Hello opendal go binding!")) + if err != nil { + panic(err) + } -Then, we can build the go binding. + // Read data from the file "test" + data, err := op.Read("test") + if err != nil { + panic(err) + } + fmt.Printf("Read content: %s\n", data) -```shell -cd bindings/go -go build -tags dynamic . -``` + // List all entries under the root directory "/" + lister, err := op.List("/") + if err != nil { + panic(err) + } + defer lister.Close() -To running the go binding tests, we need to tell the linker where to find the `libopendal_c.so` file. + // Iterate through all entries + for lister.Next() { + entry := lister.Entry() -```shell -expose LD_LIBRARY_PATH=/path/to/opendal/bindings/c/target/debug/ -``` + // Get entry name (not used in this example) + _ = entry.Name() + + // Get metadata for the current entry + meta, _ := op.Stat(entry.Path()) + + // Print file size + fmt.Printf("Size: %d bytes\n", meta.ContentLength()) -Then, we can run the tests. + // Print last modified time + fmt.Printf("Last modified: %s\n", meta.LastModified()) -```shell -go test -tags dynamic . + // Check if the entry is a directory or a file + fmt.Printf("Is directory: %v, Is file: %v\n", meta.IsDir(), meta.IsFile()) + } + + // Check for any errors that occurred during iteration + if err := lister.Error(); err != nil { + panic(err) + } + + // Copy a file + op.Copy("test", "test_copy") + + // Rename a file + op.Rename("test", "test_rename") + + // Delete a file + op.Delete("test_rename") +} ``` -For benchmark +## Run Tests -```shell -go test -bench=. -tags dynamic . +```bash +# Run all tests +CGO_ENABLE=0 go test -v -run TestBehavior +# Run specific test +CGO_ENABLE=0 go test -v -run TestBehavior/Write +# Run synchronously +CGO_ENABLE=0 GOMAXPROCS=1 go test -v -run TestBehavior ``` +## Capabilities + +- [x] OperatorInfo +- [x] Stat + - [x] Metadata +- [x] IsExist +- [x] Read + - [x] Read + - [x] Reader -- implement as `io.ReadCloser` +- [ ] Write + - [x] Write + - [ ] Writer -- Need support from the C binding +- [x] Delete +- [x] CreateDir +- [ ] Lister + - [x] Entry + - [ ] Metadata -- Need support from the C binding +- [x] Copy +- [x] Rename + + ## License and Trademarks Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 diff --git a/bindings/go/copy_test.go b/bindings/go/copy_test.go new file mode 100644 index 00000000000..1e9f33fa157 --- /dev/null +++ b/bindings/go/copy_test.go @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package opendal_test + +import ( + "fmt" + + "github.com/apache/opendal/bindings/go" + "github.com/google/uuid" + "github.com/stretchr/testify/require" +) + +func testsCopy(cap *opendal.Capability) []behaviorTest { + if !cap.Read() || !cap.Write() || !cap.Copy() { + return nil + } + return []behaviorTest{ + testCopyFileWithASCIIName, + testCopyFileWithNonASCIIName, + testCopyNonExistingSource, + testCopySourceDir, + testCopyTargetDir, + testCopySelf, + testCopyNested, + testCopyOverwrite, + } +} + +func testCopyFileWithASCIIName(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + sourcePath, sourceContent, _ := fixture.NewFile() + + assert.Nil(op.Write(sourcePath, sourceContent)) + + targetPath := fixture.NewFilePath() + + assert.Nil(op.Copy(sourcePath, targetPath)) + + targetContent, err := op.Read(targetPath) + assert.Nil(err, "read must succeed") + assert.Equal(sourceContent, targetContent) +} + +func testCopyFileWithNonASCIIName(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + sourcePath, sourceContent, _ := fixture.NewFileWithPath("🐂🍺中文.docx") + targetPath := fixture.PushPath("😈🐅Français.docx") + + assert.Nil(op.Write(sourcePath, sourceContent)) + assert.Nil(op.Copy(sourcePath, targetPath)) + + targetContent, err := op.Read(targetPath) + assert.Nil(err, "read must succeed") + assert.Equal(sourceContent, targetContent) +} + +func testCopyNonExistingSource(assert *require.Assertions, op *opendal.Operator, _ *fixture) { + sourcePath := uuid.NewString() + targetPath := uuid.NewString() + + err := op.Copy(sourcePath, targetPath) + assert.NotNil(err, "copy must fail") + assert.Equal(opendal.CodeNotFound, assertErrorCode(err)) +} + +func testCopySourceDir(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + if !op.Info().GetFullCapability().CreateDir() { + return + } + + sourcePath := fixture.NewDirPath() + targetPath := uuid.NewString() + + assert.Nil(op.CreateDir(sourcePath)) + + err := op.Copy(sourcePath, targetPath) + assert.NotNil(err, "copy must fail") + assert.Equal(opendal.CodeIsADirectory, assertErrorCode(err)) +} + +func testCopyTargetDir(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + if !op.Info().GetFullCapability().CreateDir() { + return + } + + sourcePath, sourceContent, _ := fixture.NewFile() + + assert.Nil(op.Write(sourcePath, sourceContent)) + + targetPath := fixture.NewDirPath() + + assert.Nil(op.CreateDir(targetPath)) + + err := op.Copy(sourcePath, targetPath) + assert.NotNil(err, "copy must fail") + assert.Equal(opendal.CodeIsADirectory, assertErrorCode(err)) +} + +func testCopySelf(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + sourcePath, sourceContent, _ := fixture.NewFile() + + assert.Nil(op.Write(sourcePath, sourceContent)) + + err := op.Copy(sourcePath, sourcePath) + assert.NotNil(err, "copy must fail") + assert.Equal(opendal.CodeIsSameFile, assertErrorCode(err)) +} + +func testCopyNested(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + sourcePath, sourceContent, _ := fixture.NewFile() + + assert.Nil(op.Write(sourcePath, sourceContent)) + + targetPath := fixture.PushPath(fmt.Sprintf( + "%s/%s/%s", + uuid.NewString(), + uuid.NewString(), + uuid.NewString(), + )) + + assert.Nil(op.Copy(sourcePath, targetPath)) + + targetContent, err := op.Read(targetPath) + assert.Nil(err, "read must succeed") + assert.Equal(sourceContent, targetContent) +} + +func testCopyOverwrite(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + sourcePath, sourceContent, _ := fixture.NewFile() + + assert.Nil(op.Write(sourcePath, sourceContent)) + + targetPath, targetContent, _ := fixture.NewFile() + assert.NotEqual(sourceContent, targetContent) + + assert.Nil(op.Write(targetPath, targetContent)) + + assert.Nil(op.Copy(sourcePath, targetPath)) + + targetContent, err := op.Read(targetPath) + assert.Nil(err, "read must succeed") + assert.Equal(sourceContent, targetContent) +} diff --git a/bindings/go/build_dynamic.go b/bindings/go/create_dir_test.go similarity index 50% rename from bindings/go/build_dynamic.go rename to bindings/go/create_dir_test.go index feb13dded3e..8b0937b6126 100644 --- a/bindings/go/build_dynamic.go +++ b/bindings/go/create_dir_test.go @@ -1,6 +1,3 @@ -//go:build dynamic -// +build dynamic - /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -20,11 +17,40 @@ * under the License. */ -package opendal +package opendal_test -/* -#cgo pkg-config: opendal_c -*/ -import "C" +import ( + "github.com/apache/opendal/bindings/go" + "github.com/stretchr/testify/require" +) + +func testsCreateDir(cap *opendal.Capability) []behaviorTest { + if !cap.CreateDir() || !cap.Stat() { + return nil + } + return []behaviorTest{ + testCreateDir, + testCreateDirExisting, + } +} + +func testCreateDir(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + path := fixture.NewDirPath() + + assert.Nil(op.CreateDir(path)) + + meta, err := op.Stat(path) + assert.Nil(err) + assert.True(meta.IsDir()) +} + +func testCreateDirExisting(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + path := fixture.NewDirPath() + + assert.Nil(op.CreateDir(path)) + assert.Nil(op.CreateDir(path)) -const LibopendalLinkInfo = "dynamically linked to libopendal_c" + meta, err := op.Stat(path) + assert.Nil(err) + assert.True(meta.IsDir()) +} diff --git a/bindings/go/delete.go b/bindings/go/delete.go new file mode 100644 index 00000000000..f92df487ad3 --- /dev/null +++ b/bindings/go/delete.go @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package opendal + +import ( + "context" + "unsafe" + + "github.com/jupiterrider/ffi" + "golang.org/x/sys/unix" +) + +// Delete removes the file or directory at the specified path. +// +// # Parameters +// +// - path: The path of the file or directory to delete. +// +// # Returns +// +// - error: An error if the deletion fails, or nil if successful. +// +// # Note +// +// Use with caution as this operation is irreversible. +func (op *Operator) Delete(path string) error { + delete := getFFI[operatorDelete](op.ctx, symOperatorDelete) + return delete(op.inner, path) +} + +type operatorDelete func(op *opendalOperator, path string) error + +const symOperatorDelete = "opendal_operator_delete" + +var withOperatorDelete = withFFI(ffiOpts{ + sym: symOperatorDelete, + rType: &ffi.TypePointer, + aTypes: []*ffi.Type{&ffi.TypePointer, &ffi.TypePointer}, +}, func(ctx context.Context, ffiCall func(rValue unsafe.Pointer, aValues ...unsafe.Pointer)) operatorDelete { + return func(op *opendalOperator, path string) error { + bytePath, err := unix.BytePtrFromString(path) + if err != nil { + return err + } + var e *opendalError + ffiCall( + unsafe.Pointer(&e), + unsafe.Pointer(&op), + unsafe.Pointer(&bytePath), + ) + return parseError(ctx, e) + } +}) diff --git a/bindings/go/delete_test.go b/bindings/go/delete_test.go new file mode 100644 index 00000000000..8f38ca73d0c --- /dev/null +++ b/bindings/go/delete_test.go @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package opendal_test + +import ( + "github.com/apache/opendal/bindings/go" + "github.com/google/uuid" + "github.com/stretchr/testify/require" +) + +func testsDelete(cap *opendal.Capability) []behaviorTest { + if !cap.Stat() || !cap.Delete() || !cap.Write() { + return nil + } + tests := []behaviorTest{ + testDeleteFile, + testDeleteEmptyDir, + testDeleteWithSpecialChars, + testDeleteNotExisting, + } + return tests +} + +func testDeleteFile(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + path, content, _ := fixture.NewFile() + + assert.Nil(op.Write(path, content), "write must succeed") + + assert.Nil(op.Delete(path)) + + assert.False(op.IsExist(path)) +} + +func testDeleteEmptyDir(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + if !op.Info().GetFullCapability().CreateDir() { + return + } + + path := fixture.NewDirPath() + + assert.Nil(op.CreateDir(path), "create must succeed") + + assert.Nil(op.Delete(path)) +} + +func testDeleteWithSpecialChars(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + path := uuid.NewString() + " !@#$%^&()_+-=;',.txt" + path, content, _ := fixture.NewFileWithPath(path) + + assert.Nil(op.Write(path, content), "write must succeed") + + assert.Nil(op.Delete(path)) + + assert.False(op.IsExist(path)) +} + +func testDeleteNotExisting(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + path := uuid.NewString() + + assert.Nil(op.Delete(path)) +} diff --git a/bindings/go/error.go b/bindings/go/error.go new file mode 100644 index 00000000000..33ed9b8c28f --- /dev/null +++ b/bindings/go/error.go @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package opendal + +import ( + "context" + "fmt" + "unsafe" + + "github.com/jupiterrider/ffi" +) + +// ErrorCode is all kinds of ErrorCode of opendal +type ErrorCode int32 + +const ( + // OpenDAL don't know what happened here, and no actions other than just + // returning it back. For example, s3 returns an internal service error. + CodeUnexpected ErrorCode = iota + // Underlying service doesn't support this operation. + CodeUnsupported + // The config for backend is invalid. + CodeConfigInvalid + // The given path is not found. + CodeNotFound + // The given path doesn't have enough permission for this operation + CodePermissioDenied + // The given path is a directory. + CodeIsADirectory + // The given path is not a directory. + CodeNotADirectory + // The given path already exists thus we failed to the specified operation on it. + CodeAlreadyExists + // Requests that sent to this path is over the limit, please slow down. + CodeRateLimited + // The given file paths are same. + CodeIsSameFile + // The condition of this operation is not match. + // + // The `condition` itself is context based. + // + // For example, in S3, the `condition` can be: + // 1. writing a file with If-Match header but the file's ETag is not match (will get a 412 Precondition Failed). + // 2. reading a file with If-None-Match header but the file's ETag is match (will get a 304 Not Modified). + // + // As OpenDAL cannot handle the `condition not match` error, it will always return this error to users. + // So users could to handle this error by themselves. + CodeConditionNotMatch + // The range of the content is not satisfied. + // + // OpenDAL returns this error to indicate that the range of the read request is not satisfied. + CodeRangeNotSatisfied +) + +func parseError(ctx context.Context, err *opendalError) error { + if err == nil { + return nil + } + free := getFFI[errorFree](ctx, symErrorFree) + defer free(err) + return &Error{ + code: ErrorCode(err.code), + message: string(parseBytes(&err.message)), + } +} + +type Error struct { + code ErrorCode + message string +} + +func (e *Error) Error() string { + return fmt.Sprintf("%d %s", e.code, e.message) +} + +func (e *Error) Code() ErrorCode { + return e.code +} + +func (e *Error) Message() string { + return e.message +} + +type errorFree func(e *opendalError) + +const symErrorFree = "opendal_error_free" + +var withErrorFree = withFFI(ffiOpts{ + sym: symErrorFree, + rType: &ffi.TypeVoid, + aTypes: []*ffi.Type{&ffi.TypePointer}, +}, func(_ context.Context, ffiCall func(rValue unsafe.Pointer, aValues ...unsafe.Pointer)) errorFree { + return func(e *opendalError) { + ffiCall( + nil, + unsafe.Pointer(&e), + ) + } +}) diff --git a/bindings/go/ffi.go b/bindings/go/ffi.go new file mode 100644 index 00000000000..802ae48582d --- /dev/null +++ b/bindings/go/ffi.go @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package opendal + +import ( + "context" + "errors" + "unsafe" + + "github.com/ebitengine/purego" + "github.com/jupiterrider/ffi" +) + +func contextWithFFIs(path string) (ctx context.Context, cancel context.CancelFunc, err error) { + libopendal, err := purego.Dlopen(path, purego.RTLD_LAZY|purego.RTLD_GLOBAL) + if err != nil { + return + } + ctx = context.Background() + for _, withFFI := range withFFIs { + ctx, err = withFFI(ctx, libopendal) + if err != nil { + return + } + } + cancel = func() { + purego.Dlclose(libopendal) + } + return +} + +type contextWithFFI func(ctx context.Context, libopendal uintptr) (context.Context, error) + +func getFFI[T any](ctx context.Context, key string) T { + return ctx.Value(key).(T) +} + +type ffiOpts struct { + sym string + rType *ffi.Type + aTypes []*ffi.Type +} + +func withFFI[T any]( + opts ffiOpts, + withFunc func( + ctx context.Context, + ffiCall func(rValue unsafe.Pointer, aValues ...unsafe.Pointer), + ) T, +) func(ctx context.Context, libopendal uintptr) (context.Context, error) { + return func(ctx context.Context, libopendal uintptr) (context.Context, error) { + var cif ffi.Cif + if status := ffi.PrepCif( + &cif, + ffi.DefaultAbi, + uint32(len(opts.aTypes)), + opts.rType, + opts.aTypes..., + ); status != ffi.OK { + return nil, errors.New(status.String()) + } + fn, err := purego.Dlsym(libopendal, opts.sym) + if err != nil { + return nil, err + } + return context.WithValue(ctx, opts.sym, + withFunc(ctx, func(rValue unsafe.Pointer, aValues ...unsafe.Pointer) { + ffi.Call(&cif, fn, rValue, aValues...) + }), + ), nil + } +} + +var withFFIs = []contextWithFFI{ + // free must be on top + withBytesFree, + withErrorFree, + + withOperatorOptionsNew, + withOperatorOptionsSet, + withOperatorOptionsFree, + + withOperatorNew, + withOperatorFree, + + withOperatorInfoNew, + withOperatorInfoGetFullCapability, + withOperatorInfoGetNativeCapability, + withOperatorInfoGetScheme, + withOperatorInfoGetRoot, + withOperatorInfoGetName, + withOperatorInfoFree, + + withOperatorCreateDir, + withOperatorRead, + withOperatorWrite, + withOperatorDelete, + withOperatorStat, + withOperatorIsExists, + withOperatorCopy, + withOperatorRename, + + withMetaContentLength, + withMetaIsFile, + withMetaIsDir, + withMetaLastModified, + withMetaFree, + + withOperatorList, + withListerNext, + withListerFree, + withEntryName, + withEntryPath, + withEntryFree, + + withOperatorReader, + withReaderRead, + withReaderFree, +} diff --git a/bindings/go/go.mod b/bindings/go/go.mod index 11bf83647be..21d89c79a1f 100644 --- a/bindings/go/go.mod +++ b/bindings/go/go.mod @@ -15,14 +15,24 @@ // specific language governing permissions and limitations // under the License. -module opendal.apache.org/go +module github.com/apache/opendal/bindings/go -go 1.20 +go 1.22.4 -require github.com/stretchr/testify v1.8.4 +toolchain go1.22.5 + +require ( + github.com/ebitengine/purego v0.7.1 + github.com/google/uuid v1.6.0 + github.com/jupiterrider/ffi v0.1.0-beta.9 + github.com/stretchr/testify v1.9.0 + github.com/yuchanns/opendal-go-services v0.0.1 + golang.org/x/sys v0.22.0 +) require ( github.com/davecgh/go-spew v1.1.1 // indirect + github.com/klauspost/compress v1.17.9 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/bindings/go/go.sum b/bindings/go/go.sum index 8cf66553bc7..729e933025b 100644 --- a/bindings/go/go.sum +++ b/bindings/go/go.sum @@ -1,9 +1,22 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/ebitengine/purego v0.7.1 h1:6/55d26lG3o9VCZX8lping+bZcmShseiqlh2bnUDiPA= +github.com/ebitengine/purego v0.7.1/go.mod h1:ah1In8AOtksoNK6yk5z1HTJeUkC1Ez4Wk2idgGslMwQ= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/jupiterrider/ffi v0.1.0-beta.9 h1:HCeAPTsTFgwvcfavyJwy1L2ANz0c85W+ZE7LfzjZi3A= +github.com/jupiterrider/ffi v0.1.0-beta.9/go.mod h1:sOp6VJGFaYyr4APi8gwy6g20QNHv5F8Iq1CVbtC900s= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/yuchanns/opendal-go-services v0.0.1 h1:qeKv0mOhypQNm97g+u94DnijJK5bdEAp5pdjBGf8N7w= +github.com/yuchanns/opendal-go-services v0.0.1/go.mod h1:tw8QXHu3hzsLpiUCQ+pOIhGw7FJFumnh++rKF/BK96I= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/bindings/go/list_test.go b/bindings/go/list_test.go new file mode 100644 index 00000000000..e31f8b212e0 --- /dev/null +++ b/bindings/go/list_test.go @@ -0,0 +1,254 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package opendal_test + +import ( + "fmt" + "slices" + "strings" + + "github.com/apache/opendal/bindings/go" + "github.com/google/uuid" + "github.com/stretchr/testify/require" +) + +func testsList(cap *opendal.Capability) []behaviorTest { + if !cap.Read() || !cap.Write() || !cap.List() { + return nil + } + return []behaviorTest{ + testListCheck, + testListDir, + testListPrefix, + testListRichDir, + testListEmptyDir, + testListNonExistDir, + testListSubDir, + testListNestedDir, + testListDirWithFilePath, + } +} + +func testListCheck(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + assert.Nil(op.Check(), "operator check must succeed") +} + +func testListDir(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + parent := fixture.NewDirPath() + path, content, size := fixture.NewFileWithPath(fmt.Sprintf("%s%s", parent, uuid.NewString())) + + assert.Nil(op.Write(path, content), "write must succeed") + + obs, err := op.List(parent) + assert.Nil(err) + defer obs.Close() + + var found bool + for obs.Next() { + entry := obs.Entry() + + if entry.Path() != path { + continue + } + + meta, err := op.Stat(entry.Path()) + assert.Nil(err) + assert.True(meta.IsFile()) + assert.Equal(uint64(size), meta.ContentLength()) + found = true + break + } + assert.Nil(obs.Error()) + assert.True(found, "file must be found in list") +} + +func testListPrefix(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + path, content, _ := fixture.NewFile() + + assert.Nil(op.Write(path, content), "write must succeed") + + obs, err := op.List(path[:len(path)-1]) + assert.Nil(err) + defer obs.Close() + assert.True(obs.Next()) + assert.Nil(obs.Error()) + + entry := obs.Entry() + assert.Equal(path, entry.Path()) +} + +func testListRichDir(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + parent := fixture.NewDirPath() + assert.Nil(op.CreateDir(parent)) + + var expected []string + for range 10 { + path, content, _ := fixture.NewFileWithPath(fmt.Sprintf("%s%s", parent, uuid.NewString())) + expected = append(expected, path) + assert.Nil(op.Write(path, content)) + } + + obs, err := op.List(parent) + assert.Nil(err) + defer obs.Close() + var actual []string + for obs.Next() { + entry := obs.Entry() + actual = append(actual, entry.Path()) + } + assert.Nil(obs.Error()) + + slices.Sort(expected) + slices.Sort(actual) + + assert.Equal(expected, actual) +} + +func testListEmptyDir(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + dir := fixture.NewDirPath() + + assert.Nil(op.CreateDir(dir), "create must succeed") + + obs, err := op.List(dir) + assert.Nil(err) + defer obs.Close() + var paths []string + for obs.Next() { + entry := obs.Entry() + paths = append(paths, entry.Path()) + } + assert.Nil(obs.Error()) + assert.Equal(0, len(paths), "dir should only return empty") + + obs, err = op.List(strings.TrimSuffix(dir, "/")) + assert.Nil(err) + defer obs.Close() + for obs.Next() { + entry := obs.Entry() + path := entry.Path() + paths = append(paths, path) + meta, err := op.Stat(path) + assert.Nil(err, "given dir should exist") + assert.True(meta.IsDir(), "given dir must be dir, but found: %v", path) + } + assert.Nil(obs.Error()) + assert.Equal(1, len(paths), "only return the dir itself, but found: %v", paths) +} + +func testListNonExistDir(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + dir := fixture.NewDirPath() + + obs, err := op.List(dir) + assert.Nil(err) + defer obs.Close() + assert.False(obs.Next(), "dir should only return empty") +} + +func testListSubDir(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + path := fixture.NewDirPath() + + assert.Nil(op.CreateDir(path), "create must succeed") + + obs, err := op.List("/") + assert.Nil(err) + defer obs.Close() + + var found bool + for obs.Next() { + entry := obs.Entry() + if path != entry.Path() { + continue + } + found = true + break + } + assert.Nil(obs.Error()) + assert.True(found, "dir should be found in list") +} + +func testListNestedDir(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + parent := fixture.NewDirPath() + dir := fixture.PushPath(fmt.Sprintf("%s%s/", parent, uuid.NewString())) + + filePath := fixture.PushPath(fmt.Sprintf("%s%s", dir, uuid.NewString())) + dirPath := fixture.PushPath(fmt.Sprintf("%s%s/", dir, uuid.NewString())) + + assert.Nil(op.CreateDir(dir), "create must succeed") + assert.Nil(op.Write(filePath, []byte("test_list_nested_dir")), "write must succeed") + assert.Nil(op.CreateDir(dirPath), "create must succeed") + + obs, err := op.List(parent) + assert.Nil(err) + defer obs.Close() + var paths []string + for obs.Next() { + entry := obs.Entry() + paths = append(paths, entry.Path()) + assert.Equal(dir, entry.Path()) + } + assert.Nil(obs.Error()) + assert.Equal(1, len(paths), "parent should only got 1 entry") + + obs, err = op.List(dir) + assert.Nil(err) + defer obs.Close() + paths = nil + var foundFile bool + var foundDir bool + for obs.Next() { + entry := obs.Entry() + paths = append(paths, entry.Path()) + if entry.Path() == filePath { + foundFile = true + } else if entry.Path() == dirPath { + foundDir = true + } + } + assert.Nil(obs.Error()) + assert.Equal(2, len(paths), "parent should only got 2 entries") + + assert.True(foundFile, "file should be found in list") + meta, err := op.Stat(filePath) + assert.Nil(err) + assert.True(meta.IsFile()) + assert.Equal(uint64(20), meta.ContentLength()) + + assert.True(foundDir, "dir should be found in list") + meta, err = op.Stat(dirPath) + assert.Nil(err) + assert.True(meta.IsDir()) +} + +func testListDirWithFilePath(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + parent := fixture.NewDirPath() + path, content, _ := fixture.NewFileWithPath(fmt.Sprintf("%s%s", parent, uuid.NewString())) + + assert.Nil(op.Write(path, content)) + + obs, err := op.List(strings.TrimSuffix(parent, "/")) + assert.Nil(err) + defer obs.Close() + + for obs.Next() { + entry := obs.Entry() + assert.Equal(parent, entry.Path()) + } + assert.Nil(obs.Error()) +} diff --git a/bindings/go/lister.go b/bindings/go/lister.go new file mode 100644 index 00000000000..bbd58f1f119 --- /dev/null +++ b/bindings/go/lister.go @@ -0,0 +1,424 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package opendal + +import ( + "context" + "unsafe" + + "github.com/jupiterrider/ffi" + "golang.org/x/sys/unix" +) + +// Check verifies if the operator is functioning correctly. +// +// This function performs a health check on the operator by sending a `list` request +// to the root path. It returns any errors encountered during this process. +// +// # Returns +// +// - error: An error if the check fails, or nil if the operator is working correctly. +// +// # Details +// +// The check is performed by attempting to list the contents of the root directory. +// This operation tests the basic functionality of the operator, including +// connectivity and permissions. +// +// # Example +// +// func exampleCheck(op *opendal.Operator) { +// err = op.Check() +// if err != nil { +// log.Printf("Operator check failed: %v", err) +// } else { +// log.Println("Operator is functioning correctly") +// } +// } +// +// Note: This example assumes proper error handling and import statements. +func (op *Operator) Check() (err error) { + ds, err := op.List("/") + if err != nil { + return + } + defer ds.Close() + ds.Next() + err = ds.Error() + if err, ok := err.(*Error); ok && err.Code() == CodeNotFound { + return nil + } + return +} + +// List returns a Lister to iterate over entries that start with the given path in the parent directory. +// +// This function creates a new Lister to enumerate entries in the specified path. +// +// # Parameters +// +// - path: The starting path for listing entries. +// +// # Returns +// +// - *Lister: A new Lister instance for iterating over entries. +// - error: An error if the listing operation fails, or nil if successful. +// +// # Notes +// +// 1. List is a wrapper around the C-binding function `opendal_operator_list`. Recursive listing is not currently supported. +// 2. Returned entries do not include metadata information. Use op.Stat to fetch metadata for individual entries. +// +// # Example +// +// func exampleList(op *opendal.Operator) { +// lister, err := op.List("test") +// if err != nil { +// log.Fatal(err) +// } +// +// for lister.Next() { +// entry := lister.Entry() +// +// meta, err := op.Stat(entry.Path()) +// if err != nil { +// log.Printf("Error fetching metadata for %s: %v", entry.Path(), err) +// continue +// } +// +// fmt.Printf("Name: %s\n", entry.Name()) +// fmt.Printf("Length: %d\n", meta.ContentLength()) +// fmt.Printf("Last Modified: %s\n", meta.LastModified()) +// fmt.Printf("Is Directory: %v, Is File: %v\n", meta.IsDir(), meta.IsFile()) +// fmt.Println("---") +// } +// if err := lister.Err(); err != nil { +// log.Printf("Error during listing: %v", err) +// } +// } +// +// Note: Always check lister.Err() after the loop to catch any errors that +// occurred during iteration. +func (op *Operator) List(path string) (*Lister, error) { + list := getFFI[operatorList](op.ctx, symOperatorList) + inner, err := list(op.inner, path) + if err != nil { + return nil, err + } + lister := &Lister{ + inner: inner, + ctx: op.ctx, + } + return lister, nil +} + +// Lister provides an mechanism for listing entries at a specified path. +// +// Lister is a wrapper around the C-binding function `opendal_operator_list`. It allows +// for efficient iteration over entries in a storage system. +// +// # Limitations +// +// - The current implementation does not support the `list_with` functionality. +// +// # Usage +// +// Lister should be used in conjunction with its Next() and Entry() methods to +// iterate through entries. The iteration ends when there are no more entries +// or when an error occurs. +// +// # Behavior +// +// - Next() returns false when there are no more entries or if an error has occurred. +// - Entry() returns nil if there are no more entries or if an error has been encountered. +// +// # Example +// +// lister, err := op.List("path/to/list") +// if err != nil { +// log.Fatal(err) +// } +// +// for lister.Next() { +// entry := lister.Entry() +// // Process the entry +// fmt.Println(entry.Name()) +// } +type Lister struct { + inner *opendalLister + ctx context.Context + entry *Entry + err error +} + +// This method implements the io.Closer interface. It should be called when +// the Lister is no longer needed to ensure proper resource cleanup. +func (l *Lister) Close() error { + free := getFFI[listerFree](l.ctx, symListerFree) + free(l.inner) + + return nil +} + +func (l *Lister) Error() error { + return l.err +} + +// Next advances the Lister to the next entry in the list. +// +// This method must be called before accessing the current entry. It prepares +// the next entry for reading and indicates whether there are more entries +// to process. +// +// # Returns +// +// - bool: true if there is another entry to process, false if the end of the list +// has been reached or an error occurred. +// +// # Usage +// +// Next should be used in a loop condition to iterate through all entries: +// +// for lister.Next() { +// entry := lister.Entry() +// // Process the entry +// } +// +// # Error Handling +// +// If an error occurs during iteration, Next will return false. The error +// can then be retrieved by calling the Err method on the Lister. +// +// # Example +// +// lister, err := op.List("path/to/list") +// if err != nil { +// log.Fatal(err) +// } +// +// for lister.Next() { +// entry := lister.Entry() +// fmt.Println(entry.Name()) +// } +func (l *Lister) Next() bool { + next := getFFI[listerNext](l.ctx, symListerNext) + inner, err := next(l.inner) + if inner == nil || err != nil { + l.err = err + l.entry = nil + return false + } + + entry := newEntry(l.ctx, inner) + + l.entry = entry + return true +} + +// Entry returns the current Entry in the list. +// Returns nil if there are no more entries +func (l *Lister) Entry() *Entry { + return l.entry +} + +// Entry represents a path and its associated metadata as returned by Lister. +// +// An Entry provides basic information about a file or directory encountered +// during a list operation. It contains the path of the item and minimal metadata. +// +// # Limitations +// +// The Entry itself does not contain comprehensive metadata. For detailed +// metadata information, use the op.Stat() method with the Entry's path. +// +// # Usage +// +// Entries are typically obtained through iteration of a Lister: +// +// for lister.Next() { +// entry := lister.Entry() +// // Process the entry +// fmt.Println(entry.Name()) +// } +// +// # Fetching Detailed Metadata +// +// To obtain comprehensive metadata for an Entry, use op.Stat(): +// +// meta, err := op.Stat(entry.Path()) +// if err != nil { +// log.Printf("Error fetching metadata: %v", err) +// return +// } +// fmt.Printf("Size: %d, Last Modified: %s\n", meta.ContentLength(), meta.LastModified()) +// +// # Methods +// +// Entry provides methods to access basic information: +// - Path(): Returns the full path of the entry. +// - Name(): Returns the name of the entry (last component of the path). +type Entry struct { + name string + path string +} + +func newEntry(ctx context.Context, inner *opendalEntry) *Entry { + name := getFFI[entryName](ctx, symEntryName) + path := getFFI[entryPath](ctx, symEntryPath) + free := getFFI[entryFree](ctx, symEntryFree) + + defer free(inner) + + return &Entry{ + name: name(inner), + path: path(inner), + } +} + +// Name returns the last component of the entry's path. +func (e *Entry) Name() string { + return e.name +} + +// Path returns the full path of the entry. +func (e *Entry) Path() string { + return e.path +} + +const symOperatorList = "opendal_operator_list" + +type operatorList func(op *opendalOperator, path string) (*opendalLister, error) + +var withOperatorList = withFFI(ffiOpts{ + sym: symOperatorList, + rType: &typeResultList, + aTypes: []*ffi.Type{&ffi.TypePointer, &ffi.TypePointer}, +}, func(ctx context.Context, ffiCall func(rValue unsafe.Pointer, aValues ...unsafe.Pointer)) operatorList { + return func(op *opendalOperator, path string) (*opendalLister, error) { + bytePath, err := unix.BytePtrFromString(path) + if err != nil { + return nil, err + } + var result opendalResultList + ffiCall( + unsafe.Pointer(&result), + unsafe.Pointer(&op), + unsafe.Pointer(&bytePath), + ) + if result.err != nil { + return nil, parseError(ctx, result.err) + } + return result.lister, nil + } +}) + +const symListerFree = "opendal_lister_free" + +type listerFree func(l *opendalLister) + +var withListerFree = withFFI(ffiOpts{ + sym: symListerFree, + rType: &ffi.TypeVoid, + aTypes: []*ffi.Type{&ffi.TypePointer}, +}, func(ctx context.Context, ffiCall func(rValue unsafe.Pointer, aValues ...unsafe.Pointer)) listerFree { + return func(l *opendalLister) { + ffiCall( + nil, + unsafe.Pointer(&l), + ) + } +}) + +const symListerNext = "opendal_lister_next" + +type listerNext func(l *opendalLister) (*opendalEntry, error) + +var withListerNext = withFFI(ffiOpts{ + sym: symListerNext, + rType: &typeResultListerNext, + aTypes: []*ffi.Type{&ffi.TypePointer}, +}, func(ctx context.Context, ffiCall func(rValue unsafe.Pointer, aValues ...unsafe.Pointer)) listerNext { + return func(l *opendalLister) (*opendalEntry, error) { + var result opendalResultListerNext + ffiCall( + unsafe.Pointer(&result), + unsafe.Pointer(&l), + ) + if result.err != nil { + return nil, parseError(ctx, result.err) + } + return result.entry, nil + } +}) + +const symEntryFree = "opendal_entry_free" + +type entryFree func(e *opendalEntry) + +var withEntryFree = withFFI(ffiOpts{ + sym: symEntryFree, + rType: &ffi.TypePointer, + aTypes: []*ffi.Type{&ffi.TypePointer}, +}, func(ctx context.Context, ffiCall func(rValue unsafe.Pointer, aValues ...unsafe.Pointer)) entryFree { + return func(e *opendalEntry) { + ffiCall( + nil, + unsafe.Pointer(&e), + ) + } +}) + +const symEntryName = "opendal_entry_name" + +type entryName func(e *opendalEntry) string + +var withEntryName = withFFI(ffiOpts{ + sym: symEntryName, + rType: &ffi.TypePointer, + aTypes: []*ffi.Type{&ffi.TypePointer}, +}, func(ctx context.Context, ffiCall func(rValue unsafe.Pointer, aValues ...unsafe.Pointer)) entryName { + return func(e *opendalEntry) string { + var bytePtr *byte + ffiCall( + unsafe.Pointer(&bytePtr), + unsafe.Pointer(&e), + ) + return unix.BytePtrToString(bytePtr) + } +}) + +const symEntryPath = "opendal_entry_path" + +type entryPath func(e *opendalEntry) string + +var withEntryPath = withFFI(ffiOpts{ + sym: symEntryPath, + rType: &ffi.TypePointer, + aTypes: []*ffi.Type{&ffi.TypePointer}, +}, func(ctx context.Context, ffiCall func(rValue unsafe.Pointer, aValues ...unsafe.Pointer)) entryPath { + return func(e *opendalEntry) string { + var bytePtr *byte + ffiCall( + unsafe.Pointer(&bytePtr), + unsafe.Pointer(&e), + ) + return unix.BytePtrToString(bytePtr) + } +}) diff --git a/bindings/go/metadata.go b/bindings/go/metadata.go new file mode 100644 index 00000000000..440653e65fd --- /dev/null +++ b/bindings/go/metadata.go @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package opendal + +import ( + "context" + "time" + "unsafe" + + "github.com/jupiterrider/ffi" +) + +// Metadata represents essential information about a file or directory. +// +// This struct contains basic attributes commonly used in file systems +// and object storage systems. +type Metadata struct { + contentLength uint64 + isFile bool + isDir bool + lastModified time.Time +} + +func newMetadata(ctx context.Context, inner *opendalMetadata) *Metadata { + getLength := getFFI[metaContentLength](ctx, symMetadataContentLength) + isFile := getFFI[metaIsFile](ctx, symMetadataIsFile) + isDir := getFFI[metaIsDir](ctx, symMetadataIsDir) + getLastModified := getFFI[metaLastModified](ctx, symMetadataLastModified) + + var lastModified time.Time + ms := getLastModified(inner) + if ms != -1 { + lastModified = time.UnixMilli(ms) + } + + free := getFFI[metaFree](ctx, symMetadataFree) + defer free(inner) + + return &Metadata{ + contentLength: getLength(inner), + isFile: isFile(inner), + isDir: isDir(inner), + lastModified: lastModified, + } +} + +// ContentLength returns the size of the file in bytes. +// +// For directories, this value may not be meaningful and could be zero. +func (m *Metadata) ContentLength() uint64 { + return m.contentLength +} + +// IsFile returns true if the metadata represents a file, false otherwise. +func (m *Metadata) IsFile() bool { + return m.isFile +} + +// IsDir returns true if the metadata represents a directory, false otherwise. +func (m *Metadata) IsDir() bool { + return m.isDir +} + +// LastModified returns the time when the file or directory was last modified. +// +// The returned time is in UTC. +func (m *Metadata) LastModified() time.Time { + return m.lastModified +} + +type metaContentLength func(m *opendalMetadata) uint64 + +const symMetadataContentLength = "opendal_metadata_content_length" + +var withMetaContentLength = withFFI(ffiOpts{ + sym: symMetadataContentLength, + rType: &ffi.TypeUint64, + aTypes: []*ffi.Type{&ffi.TypePointer}, +}, func(ctx context.Context, ffiCall func(rValue unsafe.Pointer, aValues ...unsafe.Pointer)) metaContentLength { + return func(m *opendalMetadata) uint64 { + var length uint64 + ffiCall( + unsafe.Pointer(&length), + unsafe.Pointer(&m), + ) + return length + } +}) + +type metaIsFile func(m *opendalMetadata) bool + +const symMetadataIsFile = "opendal_metadata_is_file" + +var withMetaIsFile = withFFI(ffiOpts{ + sym: symMetadataIsFile, + rType: &ffi.TypeUint8, + aTypes: []*ffi.Type{&ffi.TypePointer}, +}, func(ctx context.Context, ffiCall func(rValue unsafe.Pointer, aValues ...unsafe.Pointer)) metaIsFile { + return func(m *opendalMetadata) bool { + var result uint8 + ffiCall( + unsafe.Pointer(&result), + unsafe.Pointer(&m), + ) + return result == 1 + } +}) + +type metaIsDir func(m *opendalMetadata) bool + +const symMetadataIsDir = "opendal_metadata_is_dir" + +var withMetaIsDir = withFFI(ffiOpts{ + sym: symMetadataIsDir, + rType: &ffi.TypeUint8, + aTypes: []*ffi.Type{&ffi.TypePointer}, +}, func(ctx context.Context, ffiCall func(rValue unsafe.Pointer, aValues ...unsafe.Pointer)) metaIsDir { + return func(m *opendalMetadata) bool { + var result uint8 + ffiCall( + unsafe.Pointer(&result), + unsafe.Pointer(&m), + ) + return result == 1 + } +}) + +type metaLastModified func(m *opendalMetadata) int64 + +const symMetadataLastModified = "opendal_metadata_last_modified_ms" + +var withMetaLastModified = withFFI(ffiOpts{ + sym: symMetadataLastModified, + rType: &ffi.TypeSint64, + aTypes: []*ffi.Type{&ffi.TypePointer}, +}, func(ctx context.Context, ffiCall func(rValue unsafe.Pointer, aValues ...unsafe.Pointer)) metaLastModified { + return func(m *opendalMetadata) int64 { + var result int64 + ffiCall( + unsafe.Pointer(&result), + unsafe.Pointer(&m), + ) + return result + } +}) + +type metaFree func(m *opendalMetadata) + +const symMetadataFree = "opendal_metadata_free" + +var withMetaFree = withFFI(ffiOpts{ + sym: symMetadataFree, + rType: &ffi.TypeVoid, + aTypes: []*ffi.Type{&ffi.TypePointer}, +}, func(ctx context.Context, ffiCall func(rValue unsafe.Pointer, aValues ...unsafe.Pointer)) metaFree { + return func(m *opendalMetadata) { + ffiCall( + nil, + unsafe.Pointer(&m), + ) + } +}) diff --git a/bindings/go/opendal.go b/bindings/go/opendal.go index 027d87ad24e..9640e4ddcf5 100644 --- a/bindings/go/opendal.go +++ b/bindings/go/opendal.go @@ -17,85 +17,197 @@ * under the License. */ +// Package opendal provides a Go binding for Apache OpenDAL (Open Data Access Layer). +// +// OpenDAL is a data access layer that allows users to easily interact with various +// storage services using a unified API. This Go binding enables Go developers to +// leverage OpenDAL's capabilities without the need for CGO. +// +// Key features: +// - Unified interface for multiple storage backends (e.g., S3, Azure Blob, local filesystem) +// - Native Go implementation using purego and libffi +// - No CGO dependency, ensuring better portability and easier cross-compilation +// - Supports common operations like read, write, delete, list, and metadata retrieval +// +// Basic usage: +// +// import ( +// "github.com/apache/opendal/bindings/go +// "github.com/yuchanns/opendal-go-services/memory +// ) +// +// func main() { +// op, err := opendal.NewOperator(memory.Scheme, opendal.OperatorOptions{ +// "root": "/path/to/root", +// }) +// if err != nil { +// log.Fatal(err) +// } +// defer op.Close() +// +// // Perform operations using the operator +// err = op.Write("example.txt", []byte("Hello, OpenDAL!")) +// if err != nil { +// log.Fatal(err) +// } +// } +// +// This package aims to provide a seamless experience for Go developers working with +// various storage systems, combining the flexibility of OpenDAL with the performance +// and simplicity of native Go code. package opendal -/* -#include "opendal.h" -*/ -import "C" import ( - "errors" - "fmt" - "unsafe" + "context" ) -var ( - errInvalidScheme = errors.New("invalid scheme") - errValueEmpty = errors.New("value is empty") -) +// Scheme defines the interface for storage scheme implementations. +// +// A Scheme represents a specific storage backend (e.g., S3, filesystem, memory) +// and provides methods to identify and initialize the scheme. +// +// Implementations of this interface should be thread-safe, especially the LoadOnce method. +type Scheme interface { + // Name returns the unique identifier of the scheme. + Name() string + + // Path returns the filesystem path where the scheme's shared library (.so) is located. + Path() string + + // LoadOnce initializes the scheme. It ensures that initialization occurs only once, + // even if called multiple times. Subsequent calls after the first should be no-ops. + // + // Returns an error if initialization fails. + LoadOnce() error +} -type Options map[string]string +// OperatorOptions contains configuration parameters for creating an Operator. +// +// This struct allows users to specify various settings and credentials +// required for connecting to and interacting with different storage backends. +// +// Fields in this struct vary depending on the storage scheme being used. +// Refer to the documentation of specific storage backends for details on +// required and optional fields. +// +// Example usage: +// +// options := opendal.OperatorOptions{ +// "root": "/path/to/root", +// "endpoint": "https://example.com", +// "access_key_id": "your_access_key", +// "secret_access_key": "your_secret_key", +// } +type OperatorOptions map[string]string +// Operator is the entry point for all public APIs in OpenDAL. +// +// Operator provides a unified interface for interacting with various storage services. +// It encapsulates the underlying storage operations and presents a consistent API +// regardless of the storage backend. +// +// # Usage +// +// Create an Operator using NewOperator, perform operations, and always remember +// to Close the operator when finished to release resources. +// +// # Example +// +// func main() { +// // Create a new operator for the memory backend +// op, err := opendal.NewOperator(memory.Scheme, opendal.OperatorOptions{}) +// if err != nil { +// log.Fatal(err) +// } +// defer op.Close() // Ensure the operator is closed when done +// +// // Perform operations using the operator +// err = op.Write("example.txt", []byte("Hello, OpenDAL!")) +// if err != nil { +// log.Fatal(err) +// } +// +// data, err := op.Read("example.txt") +// if err != nil { +// log.Fatal(err) +// } +// fmt.Println(string(data)) +// } +// +// Note: Always use defer op.Close() to ensure proper resource cleanup. +// +// # Available Operations +// +// Operator provides methods for common storage operations including: +// - Read: Read data from a path +// - Write: Write data to a path +// - Stat: Get metadata for a path +// - Delete: Remove a file or directory +// - List: Enumerate entries in a directory +// - and more... +// +// Refer to the individual method documentation for detailed usage information. type Operator struct { - inner *C.opendal_operator + ctx context.Context + cancel context.CancelFunc + + inner *opendalOperator } -func NewOperator(scheme string, opt Options) (*Operator, error) { - if len(scheme) == 0 { - return nil, errInvalidScheme - } - opts := C.opendal_operator_options_new() - defer C.opendal_operator_options_free(opts) - for k, v := range opt { - C.opendal_operator_options_set(opts, C.CString(k), C.CString(v)) - } - ret := C.opendal_operator_new(C.CString(scheme), opts) - if ret.error != nil { - defer C.opendal_error_free(ret.error) - code, message := parseError(ret.error) - return nil, errors.New(fmt.Sprintf("create operator failed, error code: %d, error message: %s", code, message)) +// NewOperator creates and initializes a new Operator for the specified storage scheme. +// +// Parameters: +// - scheme: The storage scheme (e.g., "memory", "s3", "fs"). +// - options: Configuration options for the operator. +// +// Returns: +// - *Operator: A new Operator instance. +// - error: An error if initialization fails, or nil if successful. +// +// Note: Remember to call Close() on the returned Operator when it's no longer needed. +func NewOperator(scheme Scheme, opts OperatorOptions) (op *Operator, err error) { + err = scheme.LoadOnce() + if err != nil { + return } - return &Operator{ - inner: ret.op, - }, nil -} -func (o *Operator) Write(key string, value []byte) error { - if len(value) == 0 { - return errValueEmpty + ctx, cancel, err := contextWithFFIs(scheme.Path()) + if err != nil { + return } - bytes := C.opendal_bytes{data: (*C.uchar)(unsafe.Pointer(&value[0])), len: C.ulong(len(value))} - ret := C.opendal_operator_write(o.inner, C.CString(key), bytes) - if ret != nil { - defer C.opendal_error_free(ret) - code, message := parseError(ret) - return errors.New(fmt.Sprintf("write failed, error code: %d, error message: %s", code, message)) + + options := getFFI[operatorOptionsNew](ctx, symOperatorOptionsNew)() + setOptions := getFFI[operatorOptionsSet](ctx, symOperatorOptionSet) + optionsFree := getFFI[operatorOptionsFree](ctx, symOperatorOptionsFree) + + for key, value := range opts { + setOptions(options, key, value) } - return nil -} + defer optionsFree(options) -func (o *Operator) Read(key string) ([]byte, error) { - ret := C.opendal_operator_read(o.inner, C.CString(key)) - if ret.error != nil { - defer C.opendal_error_free(ret.error) - code, message := parseError(ret.error) - return nil, errors.New(fmt.Sprintf("read failed, error code: %d, error message: %s", code, message)) + inner, err := getFFI[operatorNew](ctx, symOperatorNew)(scheme, options) + if err != nil { + cancel() + return } - return C.GoBytes(unsafe.Pointer(ret.data.data), C.int(ret.data.len)), nil -} -func (o *Operator) Close() error { - C.opendal_operator_free(o.inner) - return nil -} + op = &Operator{ + inner: inner, + ctx: ctx, + cancel: cancel, + } -func decodeBytes(bs C.opendal_bytes) []byte { - return C.GoBytes(unsafe.Pointer(bs.data), C.int(bs.len)) + return } -func parseError(err *C.opendal_error) (int, string) { - code := int(err.code) - message := string(decodeBytes(err.message)) - - return code, message +// Close releases all resources associated with the Operator. +// +// It's important to call this method when the Operator is no longer needed +// to ensure proper cleanup of underlying resources. +// +// Note: It's recommended to use defer op.Close() immediately after creating an Operator. +func (op *Operator) Close() { + free := getFFI[operatorFree] + free(op.ctx, symOperatorFree)(op.inner) + op.cancel() } diff --git a/bindings/go/opendal_test.go b/bindings/go/opendal_test.go index 9ac76c27350..6ffcb674389 100644 --- a/bindings/go/opendal_test.go +++ b/bindings/go/opendal_test.go @@ -17,42 +17,213 @@ * under the License. */ -package opendal +package opendal_test import ( - "bytes" + "crypto/rand" + "fmt" + "math/big" + "os" + "reflect" + "runtime" + "strings" + "sync" "testing" - "github.com/stretchr/testify/assert" + "github.com/apache/opendal/bindings/go" + "github.com/google/uuid" + "github.com/stretchr/testify/require" + "github.com/yuchanns/opendal-go-services/aliyun_drive" + "github.com/yuchanns/opendal-go-services/memory" ) -func TestOperations(t *testing.T) { - opts := make(Options) - opts["root"] = "/myroot" - operator, err := NewOperator("memory", opts) - assert.NoError(t, err) - defer operator.Close() - err = operator.Write("test", []byte("Hello World from OpenDAL CGO!")) - assert.NoError(t, err) - value, err := operator.Read("test") - assert.NoError(t, err) - assert.Equal(t, "Hello World from OpenDAL CGO!", string(value)) +// Add more schemes for behavior tests here. +var schemes = []opendal.Scheme{ + aliyun_drive.Scheme, + memory.Scheme, +} + +type behaviorTest = func(assert *require.Assertions, op *opendal.Operator, fixture *fixture) + +func TestBehavior(t *testing.T) { + assert := require.New(t) + + op, closeFunc, err := newOperator() + assert.Nil(err) + + cap := op.Info().GetFullCapability() + + var tests []behaviorTest + + tests = append(tests, testsCopy(cap)...) + tests = append(tests, testsCreateDir(cap)...) + tests = append(tests, testsDelete(cap)...) + tests = append(tests, testsList(cap)...) + tests = append(tests, testsRead(cap)...) + tests = append(tests, testsRename(cap)...) + tests = append(tests, testsStat(cap)...) + tests = append(tests, testsWrite(cap)...) + + fixture := newFixture(op) - println(string(value)) + t.Cleanup(func() { + fixture.Cleanup(assert) + op.Close() + + if closeFunc != nil { + closeFunc() + } + }) + + for i := range tests { + test := tests[i] + + fullName := runtime.FuncForPC(reflect.ValueOf(test).Pointer()).Name() + parts := strings.Split(fullName, ".") + testName := strings.TrimPrefix(parts[len((parts))-1], "test") + + t.Run(testName, func(t *testing.T) { + // Run all tests in parallel by default. + // To run synchronously for specific services, set GOMAXPROCS=1. + t.Parallel() + assert := require.New(t) + + test(assert, op, fixture) + }) + } } -func BenchmarkOperator_Write(b *testing.B) { - opts := make(Options) - opts["root"] = "/myroot" - op, _ := NewOperator("memory", opts) - defer op.Close() +func newOperator() (op *opendal.Operator, closeFunc func(), err error) { + test := os.Getenv("OPENDAL_TEST") + var scheme opendal.Scheme + for _, s := range schemes { + if s.Name() != test { + continue + } + err = s.LoadOnce() + if err != nil { + return + } + closeFunc = func() { + os.Remove(s.Path()) + } + scheme = s + break + } + if scheme == nil { + err = fmt.Errorf("unsupported scheme: %s", test) + return + } + + prefix := fmt.Sprintf("OPENDAL_%s_", strings.ToUpper(scheme.Name())) - bs := bytes.Repeat([]byte("a"), 1024*1024) - op.Write("test", bs) + opts := opendal.OperatorOptions{} + for _, env := range os.Environ() { + pair := strings.SplitN(env, "=", 2) + if len(pair) != 2 { + continue + } + key := pair[0] + value := pair[1] + if !strings.HasPrefix(key, prefix) { + continue + } + opts[strings.ToLower(strings.TrimPrefix(key, prefix))] = value + } - b.SetBytes(1024 * 1024) - b.ResetTimer() - for i := 0; i < b.N; i++ { - op.Read("test") + op, err = opendal.NewOperator(scheme, opts) + if err != nil { + err = fmt.Errorf("create operator must succeed: %s", err) } + + return +} + +func assertErrorCode(err error) opendal.ErrorCode { + return err.(*opendal.Error).Code() +} + +func genBytesWithRange(min, max uint) ([]byte, uint) { + diff := max - min + n, _ := rand.Int(rand.Reader, big.NewInt(int64(diff+1))) + size := uint(n.Int64()) + min + + content := make([]byte, size) + + _, _ = rand.Read(content) + + return content, size +} + +func genFixedBytes(size uint) []byte { + content, _ := genBytesWithRange(size, size) + return content +} + +type fixture struct { + op *opendal.Operator + lock *sync.Mutex + + paths []string +} + +func newFixture(op *opendal.Operator) *fixture { + return &fixture{ + op: op, + lock: &sync.Mutex{}, + } +} + +func (f *fixture) NewDirPath() string { + path := fmt.Sprintf("%s/", uuid.NewString()) + f.PushPath(path) + + return path +} + +func (f *fixture) NewFilePath() string { + path := uuid.NewString() + f.PushPath(path) + + return path +} + +func (f *fixture) NewFile() (string, []byte, uint) { + return f.NewFileWithPath(uuid.NewString()) +} + +func (f *fixture) NewFileWithPath(path string) (string, []byte, uint) { + maxSize := f.op.Info().GetFullCapability().WriteTotalMaxSize() + if maxSize == 0 { + maxSize = 4 * 1024 * 1024 + } + return f.NewFileWithRange(path, 1, maxSize) +} + +func (f *fixture) NewFileWithRange(path string, min, max uint) (string, []byte, uint) { + f.PushPath(path) + + content, size := genBytesWithRange(min, max) + return path, content, size +} + +func (f *fixture) Cleanup(assert *require.Assertions) { + if !f.op.Info().GetFullCapability().Delete() { + return + } + f.lock.Lock() + defer f.lock.Unlock() + + for _, path := range f.paths { + assert.Nil(f.op.Delete(path), "delete must succeed: %s", path) + } +} + +func (f *fixture) PushPath(path string) string { + f.lock.Lock() + defer f.lock.Unlock() + + f.paths = append(f.paths, path) + + return path } diff --git a/bindings/go/operator.go b/bindings/go/operator.go new file mode 100644 index 00000000000..feb841e3ecc --- /dev/null +++ b/bindings/go/operator.go @@ -0,0 +1,298 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package opendal + +import ( + "context" + "unsafe" + + "github.com/jupiterrider/ffi" + "golang.org/x/sys/unix" +) + +// Copy duplicates a file from the source path to the destination path. +// +// This function copies the contents of the file at 'from' to a new or existing file at 'to'. +// +// # Parameters +// +// - from: The source file path. +// - to: The destination file path. +// +// # Returns +// +// - error: An error if the copy operation fails, or nil if successful. +// +// # Behavior +// +// - Both 'from' and 'to' must be file paths, not directories. +// - If 'to' already exists, it will be overwritten. +// - If 'from' and 'to' are identical, an 'IsSameFile' error will be returned. +// - The copy operation is idempotent; repeated calls with the same parameters will yield the same result. +// +// # Example +// +// func exampleCopy(op *operatorCopy) { +// err = op.Copy("path/from/file", "path/to/file") +// if err != nil { +// log.Printf("Copy operation failed: %v", err) +// } else { +// log.Println("File copied successfully") +// } +// } +// +// Note: This example assumes proper error handling and import statements. +func (op *Operator) Copy(src, dest string) error { + cp := getFFI[operatorCopy](op.ctx, symOperatorCopy) + return cp(op.inner, src, dest) +} + +// Rename changes the name or location of a file from the source path to the destination path. +// +// This function moves a file from 'from' to 'to', effectively renaming or relocating it. +// +// # Parameters +// +// - from: The current file path. +// - to: The new file path. +// +// # Returns +// +// - error: An error if the rename operation fails, or nil if successful. +// +// # Behavior +// +// - Both 'from' and 'to' must be file paths, not directories. +// - If 'to' already exists, it will be overwritten. +// - If 'from' and 'to' are identical, an 'IsSameFile' error will be returned. +// +// # Example +// +// func exampleRename(op *opendal.Operator) { +// err = op.Rename("path/from/file", "path/to/file") +// if err != nil { +// log.Printf("Rename operation failed: %v", err) +// } else { +// log.Println("File renamed successfully") +// } +// } +// +// Note: This example assumes proper error handling and import statements. +func (op *Operator) Rename(src, dest string) error { + rename := getFFI[operatorRename](op.ctx, symOperatorRename) + return rename(op.inner, src, dest) +} + +const symOperatorNew = "opendal_operator_new" + +type operatorNew func(scheme Scheme, opts *operatorOptions) (op *opendalOperator, err error) + +var withOperatorNew = withFFI(ffiOpts{ + sym: symOperatorNew, + rType: &typeResultOperatorNew, + aTypes: []*ffi.Type{&ffi.TypePointer, &ffi.TypePointer}, +}, func(ctx context.Context, ffiCall func(rValue unsafe.Pointer, aValues ...unsafe.Pointer)) operatorNew { + return func(scheme Scheme, opts *operatorOptions) (op *opendalOperator, err error) { + var byteName *byte + byteName, err = unix.BytePtrFromString(scheme.Name()) + if err != nil { + return + } + var result resultOperatorNew + ffiCall( + unsafe.Pointer(&result), + unsafe.Pointer(&byteName), + unsafe.Pointer(&opts), + ) + if result.error != nil { + err = parseError(ctx, result.error) + return + } + op = result.op + return + } +}) + +const symOperatorFree = "opendal_operator_free" + +type operatorFree func(op *opendalOperator) + +var withOperatorFree = withFFI(ffiOpts{ + sym: symOperatorFree, + rType: &ffi.TypeVoid, + aTypes: []*ffi.Type{&ffi.TypePointer}, +}, func(_ context.Context, ffiCall func(rValue unsafe.Pointer, aValues ...unsafe.Pointer)) operatorFree { + return func(op *opendalOperator) { + ffiCall( + nil, + unsafe.Pointer(&op), + ) + } +}) + +type operatorOptions struct { + inner uintptr +} + +const symOperatorOptionsNew = "opendal_operator_options_new" + +type operatorOptionsNew func() (opts *operatorOptions) + +var withOperatorOptionsNew = withFFI(ffiOpts{ + sym: symOperatorOptionsNew, + rType: &ffi.TypePointer, +}, func(_ context.Context, ffiCall func(rValue unsafe.Pointer, aValues ...unsafe.Pointer)) operatorOptionsNew { + return func() (opts *operatorOptions) { + ffiCall(unsafe.Pointer(&opts)) + return + } +}) + +const symOperatorOptionSet = "opendal_operator_options_set" + +type operatorOptionsSet func(opts *operatorOptions, key, value string) error + +var withOperatorOptionsSet = withFFI(ffiOpts{ + sym: symOperatorOptionSet, + rType: &ffi.TypeVoid, + aTypes: []*ffi.Type{&ffi.TypePointer, &ffi.TypePointer, &ffi.TypePointer}, +}, func(_ context.Context, ffiCall func(rValue unsafe.Pointer, aValues ...unsafe.Pointer)) operatorOptionsSet { + return func(opts *operatorOptions, key, value string) (err error) { + var ( + byteKey *byte + byteValue *byte + ) + byteKey, err = unix.BytePtrFromString(key) + if err != nil { + return err + } + byteValue, err = unix.BytePtrFromString(value) + if err != nil { + return err + } + ffiCall( + nil, + unsafe.Pointer(&opts), + unsafe.Pointer(&byteKey), + unsafe.Pointer(&byteValue), + ) + return nil + } +}) + +const symOperatorOptionsFree = "opendal_operator_options_free" + +type operatorOptionsFree func(opts *operatorOptions) + +var withOperatorOptionsFree = withFFI(ffiOpts{ + sym: symOperatorOptionsFree, + rType: &ffi.TypeVoid, + aTypes: []*ffi.Type{&ffi.TypePointer}, +}, func(_ context.Context, ffiCall func(rValue unsafe.Pointer, aValues ...unsafe.Pointer)) operatorOptionsFree { + return func(opts *operatorOptions) { + ffiCall( + nil, + unsafe.Pointer(&opts), + ) + } +}) + +const symOperatorCopy = "opendal_operator_copy" + +type operatorCopy func(op *opendalOperator, src, dest string) (err error) + +var withOperatorCopy = withFFI(ffiOpts{ + sym: symOperatorCopy, + rType: &ffi.TypePointer, + aTypes: []*ffi.Type{&ffi.TypePointer, &ffi.TypePointer, &ffi.TypePointer}, +}, func(ctx context.Context, ffiCall func(rValue unsafe.Pointer, aValues ...unsafe.Pointer)) operatorCopy { + return func(op *opendalOperator, src, dest string) (err error) { + var ( + byteSrc *byte + byteDest *byte + ) + byteSrc, err = unix.BytePtrFromString(src) + if err != nil { + return err + } + byteDest, err = unix.BytePtrFromString(dest) + if err != nil { + return err + } + var e *opendalError + ffiCall( + unsafe.Pointer(&e), + unsafe.Pointer(&op), + unsafe.Pointer(&byteSrc), + unsafe.Pointer(&byteDest), + ) + return parseError(ctx, e) + } +}) + +const symOperatorRename = "opendal_operator_rename" + +type operatorRename func(op *opendalOperator, src, dest string) (err error) + +var withOperatorRename = withFFI(ffiOpts{ + sym: symOperatorRename, + rType: &ffi.TypePointer, + aTypes: []*ffi.Type{&ffi.TypePointer, &ffi.TypePointer, &ffi.TypePointer}, +}, func(ctx context.Context, ffiCall func(rValue unsafe.Pointer, aValues ...unsafe.Pointer)) operatorRename { + return func(op *opendalOperator, src, dest string) (err error) { + var ( + byteSrc *byte + byteDest *byte + ) + byteSrc, err = unix.BytePtrFromString(src) + if err != nil { + return err + } + byteDest, err = unix.BytePtrFromString(dest) + if err != nil { + return err + } + var e *opendalError + ffiCall( + unsafe.Pointer(&e), + unsafe.Pointer(&op), + unsafe.Pointer(&byteSrc), + unsafe.Pointer(&byteDest), + ) + return parseError(ctx, e) + } +}) + +const symBytesFree = "opendal_bytes_free" + +type bytesFree func(b *opendalBytes) + +var withBytesFree = withFFI(ffiOpts{ + sym: symBytesFree, + rType: &ffi.TypeVoid, + aTypes: []*ffi.Type{&ffi.TypePointer}, +}, func(_ context.Context, ffiCall func(rValue unsafe.Pointer, aValues ...unsafe.Pointer)) bytesFree { + return func(b *opendalBytes) { + ffiCall( + nil, + unsafe.Pointer(&b), + ) + } +}) diff --git a/bindings/go/operator_info.go b/bindings/go/operator_info.go new file mode 100644 index 00000000000..4a3c32afe30 --- /dev/null +++ b/bindings/go/operator_info.go @@ -0,0 +1,381 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package opendal + +import ( + "context" + "unsafe" + + "github.com/jupiterrider/ffi" + "golang.org/x/sys/unix" +) + +// Info returns metadata about the Operator. +// +// This method provides access to essential information about the Operator, +// including its storage scheme, root path, name, and capabilities. +// +// Returns: +// - *OperatorInfo: A pointer to an OperatorInfo struct containing the Operator's metadata. +func (op *Operator) Info() *OperatorInfo { + newInfo := getFFI[operatorInfoNew](op.ctx, symOperatorInfoNew) + inner := newInfo(op.inner) + getFullCap := getFFI[operatorInfoGetFullCapability](op.ctx, symOperatorInfoGetFullCapability) + getNativeCap := getFFI[operatorInfoGetNativeCapability](op.ctx, symOperatorInfoGetNativeCapability) + getScheme := getFFI[operatorInfoGetScheme](op.ctx, symOperatorInfoGetScheme) + getRoot := getFFI[operatorInfoGetRoot](op.ctx, symOperatorInfoGetRoot) + getName := getFFI[operatorInfoGetName](op.ctx, symOperatorInfoGetName) + + free := getFFI[operatorInfoFree](op.ctx, symOperatorInfoFree) + defer free(inner) + + return &OperatorInfo{ + scheme: getScheme(inner), + root: getRoot(inner), + name: getName(inner), + fullCap: &Capability{inner: getFullCap(inner)}, + nativeCap: &Capability{inner: getNativeCap(inner)}, + } +} + +// OperatorInfo provides metadata about an Operator instance. +// +// This struct contains essential information about the storage backend +// and its capabilities, allowing users to query details about the +// Operator they are working with. +type OperatorInfo struct { + scheme string + root string + name string + fullCap *Capability + nativeCap *Capability +} + +func (i *OperatorInfo) GetFullCapability() *Capability { + return i.fullCap +} + +func (i *OperatorInfo) GetNativeCapability() *Capability { + return i.nativeCap +} + +func (i *OperatorInfo) GetScheme() string { + return i.scheme +} + +func (i *OperatorInfo) GetRoot() string { + return i.root +} + +func (i *OperatorInfo) GetName() string { + return i.name +} + +// Capability represents the set of operations and features supported by an Operator. +// +// Each field indicates the support level for a specific capability: +// - bool fields: false indicates no support, true indicates support. +// - uint fields: Represent size limits or thresholds for certain operations. +// +// This struct covers a wide range of capabilities including: +// - Basic operations: stat, read, write, delete, copy, rename, list +// - Advanced features: multipart uploads, presigned URLs, batch operations +// - Operation modifiers: cache control, content type, if-match conditions +// +// The capability information helps in understanding the functionalities +// available for a specific storage backend or Operator configuration. +type Capability struct { + inner *opendalCapability +} + +func (c *Capability) Stat() bool { + return c.inner.stat == 1 +} + +func (c *Capability) StatWithIfmatch() bool { + return c.inner.statWithIfmatch == 1 +} + +func (c *Capability) StatWithIfNoneMatch() bool { + return c.inner.statWithIfNoneMatch == 1 +} + +func (c *Capability) Read() bool { + return c.inner.read == 1 +} + +func (c *Capability) ReadWithIfmatch() bool { + return c.inner.readWithIfmatch == 1 +} + +func (c *Capability) ReadWithIfMatchNone() bool { + return c.inner.readWithIfMatchNone == 1 +} + +func (c *Capability) ReadWithOverrideCacheControl() bool { + return c.inner.readWithOverrideCacheControl == 1 +} + +func (c *Capability) ReadWithOverrideContentDisposition() bool { + return c.inner.readWithOverrideContentDisposition == 1 +} + +func (c *Capability) ReadWithOverrideContentType() bool { + return c.inner.readWithOverrideContentType == 1 +} + +func (c *Capability) Write() bool { + return c.inner.write == 1 +} + +func (c *Capability) WriteCanMulti() bool { + return c.inner.writeCanMulti == 1 +} + +func (c *Capability) WriteCanEmpty() bool { + return c.inner.writeCanEmpty == 1 +} + +func (c *Capability) WriteCanAppend() bool { + return c.inner.writeCanAppend == 1 +} + +func (c *Capability) WriteWithContentType() bool { + return c.inner.writeWithContentType == 1 +} + +func (c *Capability) WriteWithContentDisposition() bool { + return c.inner.writeWithContentDisposition == 1 +} + +func (c *Capability) WriteWithCacheControl() bool { + return c.inner.writeWithCacheControl == 1 +} + +func (c *Capability) WriteMultiMaxSize() uint { + return c.inner.writeMultiMaxSize +} + +func (c *Capability) WriteMultiMinSize() uint { + return c.inner.writeMultiMinSize +} + +func (c *Capability) WriteMultiAlignSize() uint { + return c.inner.writeMultiAlignSize +} + +func (c *Capability) WriteTotalMaxSize() uint { + return c.inner.writeTotalMaxSize +} + +func (c *Capability) CreateDir() bool { + return c.inner.createDir == 1 +} + +func (c *Capability) Delete() bool { + return c.inner.delete == 1 +} + +func (c *Capability) Copy() bool { + return c.inner.copy == 1 +} + +func (c *Capability) Rename() bool { + return c.inner.rename == 1 +} + +func (c *Capability) List() bool { + return c.inner.list == 1 +} + +func (c *Capability) ListWithLimit() bool { + return c.inner.listWithLimit == 1 +} + +func (c *Capability) ListWithStartAfter() bool { + return c.inner.listWithStartAfter == 1 +} + +func (c *Capability) ListWithRecursive() bool { + return c.inner.listWithRecursive == 1 +} + +func (c *Capability) Presign() bool { + return c.inner.presign == 1 +} + +func (c *Capability) PresignRead() bool { + return c.inner.presignRead == 1 +} + +func (c *Capability) PresignStat() bool { + return c.inner.presignStat == 1 +} + +func (c *Capability) PresignWrite() bool { + return c.inner.presignWrite == 1 +} + +func (c *Capability) Batch() bool { + return c.inner.batch == 1 +} + +func (c *Capability) BatchDelete() bool { + return c.inner.batchDelete == 1 +} + +func (c *Capability) BatchMaxOperations() uint { + return c.inner.batchMaxOperations +} + +func (c *Capability) Blocking() bool { + return c.inner.blocking == 1 +} + +const symOperatorInfoNew = "opendal_operator_info_new" + +type operatorInfoNew func(op *opendalOperator) *opendalOperatorInfo + +var withOperatorInfoNew = withFFI(ffiOpts{ + sym: symOperatorInfoNew, + rType: &ffi.TypePointer, + aTypes: []*ffi.Type{&ffi.TypePointer}, +}, func(ctx context.Context, ffiCall func(rValue unsafe.Pointer, aValues ...unsafe.Pointer)) operatorInfoNew { + return func(op *opendalOperator) *opendalOperatorInfo { + var result *opendalOperatorInfo + ffiCall( + unsafe.Pointer(&result), + unsafe.Pointer(&op), + ) + return result + } +}) + +const symOperatorInfoFree = "opendal_operator_info_free" + +type operatorInfoFree func(info *opendalOperatorInfo) + +var withOperatorInfoFree = withFFI(ffiOpts{ + sym: symOperatorInfoFree, + rType: &ffi.TypeVoid, + aTypes: []*ffi.Type{&ffi.TypePointer}, +}, func(ctx context.Context, ffiCall func(rValue unsafe.Pointer, aValues ...unsafe.Pointer)) operatorInfoFree { + return func(info *opendalOperatorInfo) { + ffiCall( + nil, + unsafe.Pointer(&info), + ) + } +}) + +const symOperatorInfoGetFullCapability = "opendal_operator_info_get_full_capability" + +type operatorInfoGetFullCapability func(info *opendalOperatorInfo) *opendalCapability + +var withOperatorInfoGetFullCapability = withFFI(ffiOpts{ + sym: symOperatorInfoGetFullCapability, + rType: &typeCapability, + aTypes: []*ffi.Type{&ffi.TypePointer}, +}, func(ctx context.Context, ffiCall func(rValue unsafe.Pointer, aValues ...unsafe.Pointer)) operatorInfoGetFullCapability { + return func(info *opendalOperatorInfo) *opendalCapability { + var cap opendalCapability + ffiCall( + unsafe.Pointer(&cap), + unsafe.Pointer(&info), + ) + return &cap + } +}) + +const symOperatorInfoGetNativeCapability = "opendal_operator_info_get_native_capability" + +type operatorInfoGetNativeCapability func(info *opendalOperatorInfo) *opendalCapability + +var withOperatorInfoGetNativeCapability = withFFI(ffiOpts{ + sym: symOperatorInfoGetNativeCapability, + rType: &typeCapability, + aTypes: []*ffi.Type{&ffi.TypePointer}, +}, func(ctx context.Context, ffiCall func(rValue unsafe.Pointer, aValues ...unsafe.Pointer)) operatorInfoGetNativeCapability { + return func(info *opendalOperatorInfo) *opendalCapability { + var cap opendalCapability + ffiCall( + unsafe.Pointer(&cap), + unsafe.Pointer(&info), + ) + return &cap + } +}) + +const symOperatorInfoGetScheme = "opendal_operator_info_get_scheme" + +type operatorInfoGetScheme func(info *opendalOperatorInfo) string + +var withOperatorInfoGetScheme = withFFI(ffiOpts{ + sym: symOperatorInfoGetScheme, + rType: &ffi.TypePointer, + aTypes: []*ffi.Type{&ffi.TypePointer}, +}, func(ctx context.Context, ffiCall func(rValue unsafe.Pointer, aValues ...unsafe.Pointer)) operatorInfoGetScheme { + return func(info *opendalOperatorInfo) string { + var bytePtr *byte + ffiCall( + unsafe.Pointer(&bytePtr), + unsafe.Pointer(&info), + ) + return unix.BytePtrToString(bytePtr) + } +}) + +const symOperatorInfoGetRoot = "opendal_operator_info_get_root" + +type operatorInfoGetRoot func(info *opendalOperatorInfo) string + +var withOperatorInfoGetRoot = withFFI(ffiOpts{ + sym: symOperatorInfoGetRoot, + rType: &ffi.TypePointer, + aTypes: []*ffi.Type{&ffi.TypePointer}, +}, func(ctx context.Context, ffiCall func(rValue unsafe.Pointer, aValues ...unsafe.Pointer)) operatorInfoGetRoot { + return func(info *opendalOperatorInfo) string { + var bytePtr *byte + ffiCall( + unsafe.Pointer(&bytePtr), + unsafe.Pointer(&info), + ) + return unix.BytePtrToString(bytePtr) + } +}) + +const symOperatorInfoGetName = "opendal_operator_info_get_name" + +type operatorInfoGetName func(info *opendalOperatorInfo) string + +var withOperatorInfoGetName = withFFI(ffiOpts{ + sym: symOperatorInfoGetName, + rType: &ffi.TypePointer, + aTypes: []*ffi.Type{&ffi.TypePointer}, +}, func(ctx context.Context, ffiCall func(rValue unsafe.Pointer, aValues ...unsafe.Pointer)) operatorInfoGetName { + return func(info *opendalOperatorInfo) string { + var bytePtr *byte + ffiCall( + unsafe.Pointer(&bytePtr), + unsafe.Pointer(&info), + ) + return unix.BytePtrToString(bytePtr) + } +}) diff --git a/bindings/go/read_test.go b/bindings/go/read_test.go new file mode 100644 index 00000000000..ff125cf1d1e --- /dev/null +++ b/bindings/go/read_test.go @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package opendal_test + +import ( + "github.com/apache/opendal/bindings/go" + "github.com/google/uuid" + "github.com/stretchr/testify/require" +) + +func testsRead(cap *opendal.Capability) []behaviorTest { + if !cap.Read() || !cap.Write() { + return nil + } + return []behaviorTest{ + testReadFull, + testReader, + testReadNotExist, + testReadWithDirPath, + testReadWithSpecialChars, + } +} + +func testReadFull(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + path, content, size := fixture.NewFile() + + assert.Nil(op.Write(path, content), "write must succeed") + + bs, err := op.Read(path) + assert.Nil(err) + assert.Equal(size, uint(len(bs)), "read size") + assert.Equal(content, bs, "read content") +} + +func testReader(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + path, content, size := fixture.NewFile() + + assert.Nil(op.Write(path, content), "write must succeed") + + r, err := op.Reader(path) + assert.Nil(err) + defer r.Close() + bs := make([]byte, size) + n, err := r.Read(bs) + assert.Nil(err) + assert.Equal(size, uint(n), "read size") + assert.Equal(content, bs[:n], "read content") +} + +func testReadNotExist(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + path := fixture.NewFilePath() + + _, err := op.Read(path) + assert.NotNil(err) + assert.Equal(opendal.CodeNotFound, assertErrorCode(err)) +} + +func testReadWithDirPath(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + if !op.Info().GetFullCapability().CreateDir() { + return + } + + path := fixture.NewDirPath() + + assert.Nil(op.CreateDir(path), "create must succeed") + + _, err := op.Read(path) + assert.NotNil(err) + assert.Equal(opendal.CodeIsADirectory, assertErrorCode(err)) +} + +func testReadWithSpecialChars(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + path, content, size := fixture.NewFileWithPath(uuid.NewString() + " !@#$%^&()_+-=;',.txt") + + assert.Nil(op.Write(path, content), "write must succeed") + + bs, err := op.Read(path) + assert.Nil(err) + assert.Equal(size, uint(len(bs))) + assert.Equal(content, bs) +} diff --git a/bindings/go/reader.go b/bindings/go/reader.go new file mode 100644 index 00000000000..c6fbb68238a --- /dev/null +++ b/bindings/go/reader.go @@ -0,0 +1,299 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package opendal + +import ( + "context" + "io" + "unsafe" + + "github.com/jupiterrider/ffi" + "golang.org/x/sys/unix" +) + +// Read reads the entire contents of the file at the specified path into a byte slice. +// +// This function is a wrapper around the C-binding function `opendal_operator_read`. +// +// # Parameters +// +// - path: The path of the file to read. +// +// # Returns +// +// - []byte: The contents of the file as a byte slice. +// - error: An error if the read operation fails, or nil if successful. +// +// # Notes +// +// - This implementation does not support the `read_with` functionality. +// - Read allocates a new byte slice internally. For more precise memory control +// or lazy reading, consider using the Reader() method instead. +// +// # Example +// +// func exampleRead(op *opendal.Operator) { +// data, err := op.Read("test") +// if err != nil { +// log.Fatal(err) +// } +// fmt.Printf("Read: %s\n", data) +// } +// +// Note: This example assumes proper error handling and import statements. +func (op *Operator) Read(path string) ([]byte, error) { + read := getFFI[operatorRead](op.ctx, symOperatorRead) + bytes, err := read(op.inner, path) + if err != nil { + return nil, err + } + + data := parseBytes(bytes) + if len(data) > 0 { + free := getFFI[bytesFree](op.ctx, symBytesFree) + free(bytes) + + } + return data, nil +} + +// Reader creates a new Reader for reading the contents of a file at the specified path. +// +// This function is a wrapper around the C-binding function `opendal_operator_reader`. +// +// # Parameters +// +// - path: The path of the file to read. +// +// # Returns +// +// - *Reader: A reader for accessing the file's contents. It implements `io.ReadCloser`. +// - error: An error if the reader creation fails, or nil if successful. +// +// # Notes +// +// - This implementation does not support the `reader_with` functionality. +// - The returned reader allows for more controlled and efficient reading of large files. +// +// # Example +// +// func exampleReader(op *opendal.Operator) { +// r, err := op.Reader("path/to/file") +// if err != nil { +// log.Fatal(err) +// } +// defer r.Close() +// +// size := 1024 // Read 1KB at a time +// buffer := make([]byte, size) +// +// for { +// n, err := r.Read(buffer) +// if err != nil { +// log.Fatal(err) +// } +// fmt.Printf("Read %d bytes: %s\n", n, buffer[:n]) +// } +// } +// +// Note: This example assumes proper error handling and import statements. +func (op *Operator) Reader(path string) (*Reader, error) { + getReader := getFFI[operatorReader](op.ctx, symOperatorReader) + inner, err := getReader(op.inner, path) + if err != nil { + return nil, err + } + reader := &Reader{ + inner: inner, + ctx: op.ctx, + } + return reader, nil +} + +type Reader struct { + inner *opendalReader + ctx context.Context +} + +var _ io.ReadCloser = (*Reader)(nil) + +// Read reads data from the underlying storage into the provided buffer. +// +// This method implements the io.Reader interface for OperatorReader. +// +// # Parameters +// +// - buf: A pre-allocated byte slice where the read data will be stored. +// The length of buf determines the maximum number of bytes to read. +// +// # Returns +// +// - int: The number of bytes read. Returns 0 if no data is available or the end of the file is reached. +// - error: An error if the read operation fails, or nil if successful. +// Note that this method does not return io.EOF; it returns nil at the end of the file. +// +// # Notes +// +// - This method only returns OpenDAL-specific errors, not io.EOF. +// - If no data is read (end of file), it returns (0, nil) instead of (0, io.EOF). +// - The caller is responsible for pre-allocating the buffer and determining its size. +// +// # Example +// +// reader, err := op.Reader("path/to/file") +// if err != nil { +// log.Fatal(err) +// } +// defer reader.Close() +// +// buf := make([]byte, 1024) +// for { +// n, err := reader.Read(buf) +// if err != nil { +// log.Fatal(err) +// } +// if n == 0 { +// break // End of file +// } +// // Process buf[:n] +// } +// +// Note: Always check the number of bytes read (n) as it may be less than len(buf). +func (r *Reader) Read(buf []byte) (int, error) { + length := uint(len(buf)) + read := getFFI[readerRead](r.ctx, symReaderRead) + var ( + totalSize uint + size uint + err error + ) + for { + size, err = read(r.inner, buf[totalSize:]) + totalSize += size + if size == 0 || err != nil || totalSize >= length { + break + } + } + return int(totalSize), err +} + +// Close releases resources associated with the OperatorReader. +func (r *Reader) Close() error { + free := getFFI[readerFree](r.ctx, symReaderFree) + free(r.inner) + return nil +} + +const symOperatorRead = "opendal_operator_read" + +type operatorRead func(op *opendalOperator, path string) (*opendalBytes, error) + +var withOperatorRead = withFFI(ffiOpts{ + sym: symOperatorRead, + rType: &typeResultRead, + aTypes: []*ffi.Type{&ffi.TypePointer, &ffi.TypePointer}, +}, func(ctx context.Context, ffiCall func(rValue unsafe.Pointer, aValues ...unsafe.Pointer)) operatorRead { + return func(op *opendalOperator, path string) (*opendalBytes, error) { + bytePath, err := unix.BytePtrFromString(path) + if err != nil { + return nil, err + } + var result resultRead + ffiCall( + unsafe.Pointer(&result), + unsafe.Pointer(&op), + unsafe.Pointer(&bytePath), + ) + return result.data, parseError(ctx, result.error) + } +}) + +const symOperatorReader = "opendal_operator_reader" + +type operatorReader func(op *opendalOperator, path string) (*opendalReader, error) + +var withOperatorReader = withFFI(ffiOpts{ + sym: symOperatorReader, + rType: &typeResultOperatorReader, + aTypes: []*ffi.Type{&ffi.TypePointer, &ffi.TypePointer}, +}, func(ctx context.Context, ffiCall func(rValue unsafe.Pointer, aValues ...unsafe.Pointer)) operatorReader { + return func(op *opendalOperator, path string) (*opendalReader, error) { + bytePath, err := unix.BytePtrFromString(path) + if err != nil { + return nil, err + } + var result resultOperatorReader + ffiCall( + unsafe.Pointer(&result), + unsafe.Pointer(&op), + unsafe.Pointer(&bytePath), + ) + if result.error != nil { + return nil, parseError(ctx, result.error) + } + return result.reader, nil + } +}) + +const symReaderFree = "opendal_reader_free" + +type readerFree func(r *opendalReader) + +var withReaderFree = withFFI(ffiOpts{ + sym: symReaderFree, + rType: &ffi.TypeVoid, + aTypes: []*ffi.Type{&ffi.TypePointer}, +}, func(ctx context.Context, ffiCall func(rValue unsafe.Pointer, aValues ...unsafe.Pointer)) readerFree { + return func(r *opendalReader) { + ffiCall( + nil, + unsafe.Pointer(&r), + ) + } +}) + +const symReaderRead = "opendal_reader_read" + +type readerRead func(r *opendalReader, buf []byte) (size uint, err error) + +var withReaderRead = withFFI(ffiOpts{ + sym: symReaderRead, + rType: &typeResultReaderRead, + aTypes: []*ffi.Type{&ffi.TypePointer, &ffi.TypePointer, &ffi.TypePointer}, +}, func(ctx context.Context, ffiCall func(rValue unsafe.Pointer, aValues ...unsafe.Pointer)) readerRead { + return func(r *opendalReader, buf []byte) (size uint, err error) { + var length = len(buf) + if length == 0 { + return 0, nil + } + bytePtr := &buf[0] + var result resultReaderRead + ffiCall( + unsafe.Pointer(&result), + unsafe.Pointer(&r), + unsafe.Pointer(&bytePtr), + unsafe.Pointer(&length), + ) + if result.error != nil { + return 0, parseError(ctx, result.error) + } + return result.size, nil + } +}) diff --git a/bindings/go/rename_test.go b/bindings/go/rename_test.go new file mode 100644 index 00000000000..3a71e1b578e --- /dev/null +++ b/bindings/go/rename_test.go @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package opendal_test + +import ( + "fmt" + + "github.com/apache/opendal/bindings/go" + "github.com/google/uuid" + "github.com/stretchr/testify/require" +) + +func testsRename(cap *opendal.Capability) []behaviorTest { + if !cap.Read() || !cap.Write() || !cap.Rename() { + return nil + } + return []behaviorTest{ + testRenameFile, + testRenameNonExistingSource, + testRenameSourceDir, + testRenameTargetDir, + testRenameSelf, + testRenameNested, + testRenameOverwrite, + } +} + +func testRenameFile(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + sourcePath, sourceContent, _ := fixture.NewFile() + + assert.Nil(op.Write(sourcePath, sourceContent), "write must succeed") + + targetPath := fixture.NewFilePath() + + assert.Nil(op.Rename(sourcePath, targetPath)) + + _, err := op.Stat(sourcePath) + assert.NotNil(err, "stat must fail") + assert.Equal(opendal.CodeNotFound, assertErrorCode(err)) + + targetContent, err := op.Read(targetPath) + assert.Nil(err) + assert.Equal(sourceContent, targetContent) +} + +func testRenameNonExistingSource(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + sourcePath := fixture.NewFilePath() + targetPath := fixture.NewFilePath() + + err := op.Rename(sourcePath, targetPath) + assert.NotNil(err, "rename must fail") + assert.Equal(opendal.CodeNotFound, assertErrorCode(err)) +} + +func testRenameSourceDir(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + if !op.Info().GetFullCapability().CreateDir() { + return + } + + sourcePath := fixture.NewDirPath() + targetPth := fixture.NewFilePath() + + assert.Nil(op.CreateDir(sourcePath), "create must succeed") + + err := op.Rename(sourcePath, targetPth) + assert.NotNil(err, "rename must fail") + assert.Equal(opendal.CodeIsADirectory, assertErrorCode(err)) +} + +func testRenameTargetDir(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + if !op.Info().GetFullCapability().CreateDir() { + return + } + + sourcePath, sourceContent, _ := fixture.NewFile() + + assert.Nil(op.Write(sourcePath, sourceContent), "write must succeed") + + targetPath := fixture.NewDirPath() + assert.Nil(op.CreateDir(targetPath)) + + err := op.Rename(sourcePath, targetPath) + assert.NotNil(err) + assert.Equal(opendal.CodeIsADirectory, assertErrorCode(err)) +} + +func testRenameSelf(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + sourcePath, sourceContent, _ := fixture.NewFile() + + assert.Nil(op.Write(sourcePath, sourceContent), "write must succeed") + + err := op.Rename(sourcePath, sourcePath) + assert.NotNil(err) + assert.Equal(opendal.CodeIsSameFile, assertErrorCode(err)) +} + +func testRenameNested(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + sourcePath, sourceContent, _ := fixture.NewFile() + + assert.Nil(op.Write(sourcePath, sourceContent), "write must succeed") + + targetPath := fixture.PushPath(fmt.Sprintf( + "%s/%s/%s", + uuid.NewString(), + uuid.NewString(), + uuid.NewString(), + )) + + assert.Nil(op.Rename(sourcePath, targetPath)) + + _, err := op.Stat(sourcePath) + assert.NotNil(err, "stat must fail") + assert.Equal(opendal.CodeNotFound, assertErrorCode(err)) + + targetContent, err := op.Read(targetPath) + assert.Nil(err, "read must succeed") + assert.Equal(sourceContent, targetContent) +} + +func testRenameOverwrite(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + sourcePath, sourceContent, _ := fixture.NewFile() + + assert.Nil(op.Write(sourcePath, sourceContent), "write must succeed") + + targetPath, targetContent, _ := fixture.NewFile() + assert.NotEqual(sourceContent, targetContent) + + assert.Nil(op.Write(targetPath, targetContent), "write must succeed") + + assert.Nil(op.Rename(sourcePath, targetPath)) + + _, err := op.Stat(sourcePath) + assert.NotNil(err, "stat must fail") + assert.Equal(opendal.CodeNotFound, assertErrorCode(err)) + + targetContent, err = op.Read(targetPath) + assert.Nil(err, "read must succeed") + assert.Equal(sourceContent, targetContent) +} diff --git a/bindings/go/stat.go b/bindings/go/stat.go new file mode 100644 index 00000000000..6b83ad605c8 --- /dev/null +++ b/bindings/go/stat.go @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package opendal + +import ( + "context" + "unsafe" + + "github.com/jupiterrider/ffi" + "golang.org/x/sys/unix" +) + +// Stat retrieves metadata for the specified path. +// +// This function is a wrapper around the C-binding function `opendal_operator_stat`. +// +// # Parameters +// +// - path: The path of the file or directory to get metadata for. +// +// # Returns +// +// - *Metadata: Metadata of the specified path. +// - error: An error if the operation fails, or nil if successful. +// +// # Notes +// +// - The current implementation does not support `stat_with` functionality. +// - If the path does not exist, an error with code opendal.CodeNotFound will be returned. +// +// # Example +// +// func exampleStat(op *opendal.Operator) { +// meta, err := op.Stat("/path/to/file") +// if err != nil { +// if e, ok := err.(*opendal.Error); ok && e.Code() == opendal.CodeNotFound { +// fmt.Println("File not found") +// return +// } +// log.Fatalf("Stat operation failed: %v", err) +// } +// fmt.Printf("File size: %d bytes\n", meta.ContentLength()) +// fmt.Printf("Last modified: %v\n", meta.LastModified()) +// } +// +// Note: This example assumes proper error handling and import statements. +func (op *Operator) Stat(path string) (*Metadata, error) { + stat := getFFI[operatorStat](op.ctx, symOperatorStat) + meta, err := stat(op.inner, path) + if err != nil { + return nil, err + } + return newMetadata(op.ctx, meta), nil +} + +// IsExist checks if a file or directory exists at the specified path. +// +// This method provides a convenient way to determine the existence of a resource +// without fetching its full metadata. +// +// # Parameters +// +// - path: The path of the file or directory to check. +// +// # Returns +// +// - bool: true if the resource exists, false otherwise. +// - error: An error if the check operation fails, or nil if the check is successful. +// Note that a false return value with a nil error indicates that the resource does not exist. +// +// # Example +// +// exists, err := op.IsExist("path/to/file") +// if err != nil { +// log.Fatalf("Error checking existence: %v", err) +// } +// if exists { +// fmt.Println("The file exists") +// } else { +// fmt.Println("The file does not exist") +// } +// +func (op *Operator) IsExist(path string) (bool, error) { + isExist := getFFI[operatorIsExist](op.ctx, symOperatorIsExist) + return isExist(op.inner, path) +} + +const symOperatorStat = "opendal_operator_stat" + +type operatorStat func(op *opendalOperator, path string) (*opendalMetadata, error) + +var withOperatorStat = withFFI(ffiOpts{ + sym: symOperatorStat, + rType: &typeResultStat, + aTypes: []*ffi.Type{&ffi.TypePointer, &ffi.TypePointer}, +}, func(ctx context.Context, ffiCall func(rValue unsafe.Pointer, aValues ...unsafe.Pointer)) operatorStat { + return func(op *opendalOperator, path string) (*opendalMetadata, error) { + bytePath, err := unix.BytePtrFromString(path) + if err != nil { + return nil, err + } + var result resultStat + ffiCall( + unsafe.Pointer(&result), + unsafe.Pointer(&op), + unsafe.Pointer(&bytePath), + ) + if result.error != nil { + return nil, parseError(ctx, result.error) + } + return result.meta, nil + } +}) + +const symOperatorIsExist = "opendal_operator_is_exist" + +type operatorIsExist func(op *opendalOperator, path string) (bool, error) + +var withOperatorIsExists = withFFI(ffiOpts{ + sym: symOperatorIsExist, + rType: &typeResultIsExist, + aTypes: []*ffi.Type{&ffi.TypePointer, &ffi.TypePointer}, +}, func(ctx context.Context, ffiCall func(rValue unsafe.Pointer, aValues ...unsafe.Pointer)) operatorIsExist { + return func(op *opendalOperator, path string) (bool, error) { + bytePath, err := unix.BytePtrFromString(path) + if err != nil { + return false, err + } + var result resultIsExist + ffiCall( + unsafe.Pointer(&result), + unsafe.Pointer(&op), + unsafe.Pointer(&bytePath), + ) + if result.error != nil { + return false, parseError(ctx, result.error) + } + return result.is_exist == 1, nil + } +}) diff --git a/bindings/go/stat_test.go b/bindings/go/stat_test.go new file mode 100644 index 00000000000..f59b71be129 --- /dev/null +++ b/bindings/go/stat_test.go @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package opendal_test + +import ( + "fmt" + "strings" + + "github.com/apache/opendal/bindings/go" + "github.com/google/uuid" + "github.com/stretchr/testify/require" +) + +func testsStat(cap *opendal.Capability) []behaviorTest { + if !cap.Write() || !cap.Stat() { + return nil + } + return []behaviorTest{ + testStatFile, + testStatDir, + testStatNestedParentDir, + testStatWithSpecialChars, + testStatNotCleanedPath, + testStatNotExist, + testStatRoot, + } +} + +func testStatFile(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + path, content, size := fixture.NewFile() + + assert.Nil(op.Write(path, content)) + + meta, err := op.Stat(path) + assert.Nil(err) + assert.True(meta.IsFile()) + assert.Equal(meta.ContentLength(), uint64(size)) + + if op.Info().GetFullCapability().CreateDir() { + _, err := op.Stat(fmt.Sprintf("%s/", path)) + assert.NotNil(err) + assert.Equal(opendal.CodeNotFound, assertErrorCode(err)) + } +} + +func testStatDir(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + if !op.Info().GetFullCapability().CreateDir() { + return + } + + path := fixture.NewDirPath() + assert.Nil(op.CreateDir(path)) + + meta, err := op.Stat(path) + assert.Nil(err) + assert.True(meta.IsDir()) + + meta, err = op.Stat(strings.TrimSuffix(path, "/")) + if err != nil { + assert.Equal(opendal.CodeNotFound, assertErrorCode(err)) + } else { + assert.True(meta.IsDir()) + } +} + +func testStatNestedParentDir(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + if !op.Info().GetFullCapability().CreateDir() { + return + } + + parent := fixture.NewDirPath() + path, content, _ := fixture.NewFileWithPath(fmt.Sprintf("%s%s", parent, uuid.NewString())) + + assert.Nil(op.Write(path, content), "write must succeed") + + meta, err := op.Stat(parent) + assert.Nil(err) + assert.True(meta.IsDir()) +} + +func testStatWithSpecialChars(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + path, content, size := fixture.NewFileWithPath(uuid.NewString() + " !@#$%^&()_+-=;',.txt") + + assert.Nil(op.Write(path, content), "write must succeed") + + meta, err := op.Stat(path) + assert.Nil(err) + assert.True(meta.IsFile()) + assert.Equal(uint64(size), meta.ContentLength()) +} + +func testStatNotCleanedPath(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + path, content, size := fixture.NewFile() + + assert.Nil(op.Write(path, content), "write must succeed") + + meta, err := op.Stat(fmt.Sprintf("//%s", path)) + assert.Nil(err) + assert.True(meta.IsFile()) + assert.Equal(uint64(size), meta.ContentLength()) +} + +func testStatNotExist(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + path := fixture.NewFilePath() + + _, err := op.Stat(path) + assert.NotNil(err) + assert.Equal(opendal.CodeNotFound, assertErrorCode(err)) + + if op.Info().GetFullCapability().CreateDir() { + _, err := op.Stat(fmt.Sprintf("%s/", path)) + assert.NotNil(err) + assert.Equal(opendal.CodeNotFound, assertErrorCode(err)) + } +} + +func testStatRoot(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + meta, err := op.Stat("") + assert.Nil(err) + assert.True(meta.IsDir()) + + meta, err = op.Stat("/") + assert.Nil(err) + assert.True(meta.IsDir()) + +} diff --git a/bindings/go/types.go b/bindings/go/types.go new file mode 100644 index 00000000000..9fe21597234 --- /dev/null +++ b/bindings/go/types.go @@ -0,0 +1,286 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package opendal + +import ( + "unsafe" + + "github.com/jupiterrider/ffi" +) + +var ( + typeResultOperatorNew = ffi.Type{ + Type: ffi.Struct, + Elements: &[]*ffi.Type{ + &ffi.TypePointer, + &ffi.TypePointer, + nil, + }[0], + } + + typeResultRead = ffi.Type{ + Type: ffi.Struct, + Elements: &[]*ffi.Type{ + &ffi.TypePointer, + &ffi.TypePointer, + nil, + }[0], + } + + typeBytes = ffi.Type{ + Type: ffi.Struct, + Elements: &[]*ffi.Type{ + &ffi.TypePointer, + &ffi.TypePointer, + nil, + }[0], + } + + typeResultStat = ffi.Type{ + Type: ffi.Struct, + Elements: &[]*ffi.Type{ + &ffi.TypePointer, + &ffi.TypePointer, + nil, + }[0], + } + + typeResultList = ffi.Type{ + Type: ffi.Struct, + Elements: &[]*ffi.Type{ + &ffi.TypePointer, + &ffi.TypePointer, + nil, + }[0], + } + + typeResultListerNext = ffi.Type{ + Type: ffi.Struct, + Elements: &[]*ffi.Type{ + &ffi.TypePointer, + &ffi.TypePointer, + nil, + }[0], + } + + typeResultOperatorReader = ffi.Type{ + Type: ffi.Struct, + Elements: &[]*ffi.Type{ + &ffi.TypePointer, + &ffi.TypePointer, + nil, + }[0], + } + + typeResultReaderRead = ffi.Type{ + Type: ffi.Struct, + Elements: &[]*ffi.Type{ + &ffi.TypePointer, + &ffi.TypePointer, + nil, + }[0], + } + + typeResultIsExist = ffi.Type{ + Type: ffi.Struct, + Elements: &[]*ffi.Type{ + &ffi.TypeUint8, + &ffi.TypePointer, + nil, + }[0], + } + + typeCapability = ffi.Type{ + Type: ffi.Struct, + Elements: &[]*ffi.Type{ + &ffi.TypeUint8, // stat + &ffi.TypeUint8, // stat_with_if_match + &ffi.TypeUint8, // stat_with_if_none_match + &ffi.TypeUint8, // read + &ffi.TypeUint8, // read_with_if_match + &ffi.TypeUint8, // read_with_if_match_none + &ffi.TypeUint8, // read_with_override_cache_control + &ffi.TypeUint8, // read_with_override_content_disposition + &ffi.TypeUint8, // read_with_override_content_type + &ffi.TypeUint8, // write + &ffi.TypeUint8, // write_can_multi + &ffi.TypeUint8, // write_can_empty + &ffi.TypeUint8, // write_can_append + &ffi.TypeUint8, // write_with_content_type + &ffi.TypeUint8, // write_with_content_disposition + &ffi.TypeUint8, // write_with_cache_control + &ffi.TypePointer, // write_multi_max_size + &ffi.TypePointer, // write_multi_min_size + &ffi.TypePointer, // write_multi_align_size + &ffi.TypePointer, // write_total_max_size + &ffi.TypeUint8, // create_dir + &ffi.TypeUint8, // delete + &ffi.TypeUint8, // copy + &ffi.TypeUint8, // rename + &ffi.TypeUint8, // list + &ffi.TypeUint8, // list_with_limit + &ffi.TypeUint8, // list_with_start_after + &ffi.TypeUint8, // list_with_recursive + &ffi.TypeUint8, // presign + &ffi.TypeUint8, // presign_read + &ffi.TypeUint8, // presign_stat + &ffi.TypeUint8, // presign_write + &ffi.TypeUint8, // batch + &ffi.TypeUint8, // batch_delete + &ffi.TypePointer, // batch_max_operations + &ffi.TypeUint8, // blocking + nil, + }[0], + } +) + +type opendalCapability struct { + stat uint8 + statWithIfmatch uint8 + statWithIfNoneMatch uint8 + read uint8 + readWithIfmatch uint8 + readWithIfMatchNone uint8 + readWithOverrideCacheControl uint8 + readWithOverrideContentDisposition uint8 + readWithOverrideContentType uint8 + write uint8 + writeCanMulti uint8 + writeCanEmpty uint8 + writeCanAppend uint8 + writeWithContentType uint8 + writeWithContentDisposition uint8 + writeWithCacheControl uint8 + writeMultiMaxSize uint + writeMultiMinSize uint + writeMultiAlignSize uint + writeTotalMaxSize uint + createDir uint8 + delete uint8 + copy uint8 + rename uint8 + list uint8 + listWithLimit uint8 + listWithStartAfter uint8 + listWithRecursive uint8 + presign uint8 + presignRead uint8 + presignStat uint8 + presignWrite uint8 + batch uint8 + batchDelete uint8 + batchMaxOperations uint + blocking uint8 +} + +type resultOperatorNew struct { + op *opendalOperator + error *opendalError +} + +type opendalOperator struct { + ptr uintptr +} + +type resultRead struct { + data *opendalBytes + error *opendalError +} + +type opendalReader struct { + inner uintptr +} + +type resultOperatorReader struct { + reader *opendalReader + error *opendalError +} + +type resultReaderRead struct { + size uint + error *opendalError +} + +type resultIsExist struct { + is_exist uint8 + error *opendalError +} + +type resultStat struct { + meta *opendalMetadata + error *opendalError +} + +type opendalMetadata struct { + inner uintptr +} + +type opendalBytes struct { + data *byte + len uintptr +} + +type opendalError struct { + code int32 + message opendalBytes +} + +type opendalOperatorInfo struct { + inner uintptr +} + +type opendalResultList struct { + lister *opendalLister + err *opendalError +} + +type opendalLister struct { + inner uintptr +} + +type opendalResultListerNext struct { + entry *opendalEntry + err *opendalError +} + +type opendalEntry struct { + inner uintptr +} + +func toOpendalBytes(data []byte) opendalBytes { + var ptr *byte + l := len(data) + if l > 0 { + ptr = &data[0] + } + return opendalBytes{ + data: ptr, + len: uintptr(l), + } +} + +func parseBytes(b *opendalBytes) (data []byte) { + if b == nil || b.len == 0 { + return nil + } + data = make([]byte, b.len) + copy(data, unsafe.Slice(b.data, b.len)) + return +} diff --git a/bindings/go/write.go b/bindings/go/write.go new file mode 100644 index 00000000000..e1d18a13223 --- /dev/null +++ b/bindings/go/write.go @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package opendal + +import ( + "context" + "unsafe" + + "github.com/jupiterrider/ffi" + "golang.org/x/sys/unix" +) + +// Write writes the given bytes to the specified path. +// +// Write is a wrapper around the C-binding function `opendal_operator_write`. It provides a simplified +// interface for writing data to the storage. Currently, this implementation does not support the +// `Operator::write_with` method from the original Rust library, nor does it support streaming writes +// or multipart uploads. +// +// # Parameters +// +// - path: The destination path where the bytes will be written. +// - data: The byte slice containing the data to be written. +// +// # Returns +// +// - error: An error if the write operation fails, or nil if successful. +// +// # Example +// +// func exampleWrite(op *opendal.Operator) { +// err = op.Write("test", []byte("Hello opendal go binding!")) +// if err != nil { +// log.Fatal(err) +// } +// } +// +// Note: This example assumes proper error handling and import statements. +func (op *Operator) Write(path string, data []byte) error { + write := getFFI[operatorWrite](op.ctx, symOperatorWrite) + return write(op.inner, path, data) +} + +// CreateDir creates a directory at the specified path. +// +// CreateDir is a wrapper around the C-binding function `opendal_operator_create_dir`. +// It provides a way to create directories in the storage system. +// +// # Parameters +// +// - path: The path where the directory should be created. +// +// # Returns +// +// - error: An error if the directory creation fails, or nil if successful. +// +// # Notes +// +// It is mandatory to include a trailing slash (/) in the path to indicate +// that it is a directory. Failing to do so may result in a `CodeNotADirectory` +// error being returned by OpenDAL. +// +// # Behavior +// +// - Creating a directory that already exists will succeed without error. +// - Directory creation is always recursive, similar to the `mkdir -p` command. +// +// # Example +// +// func exampleCreateDir(op *opendal.Operator) { +// err = op.CreateDir("test/") +// if err != nil { +// log.Fatal(err) +// } +// } +// +// Note: This example assumes proper error handling and import statements. +// The trailing slash in "test/" is important to indicate it's a directory. +func (op *Operator) CreateDir(path string) error { + createDir := getFFI[operatorCreateDir](op.ctx, symOperatorCreateDir) + return createDir(op.inner, path) +} + +const symOperatorWrite = "opendal_operator_write" + +type operatorWrite func(op *opendalOperator, path string, data []byte) error + +var withOperatorWrite = withFFI(ffiOpts{ + sym: symOperatorWrite, + rType: &ffi.TypePointer, + aTypes: []*ffi.Type{&ffi.TypePointer, &ffi.TypePointer, &typeBytes}, +}, func(ctx context.Context, ffiCall func(rValue unsafe.Pointer, aValues ...unsafe.Pointer)) operatorWrite { + return func(op *opendalOperator, path string, data []byte) error { + bytePath, err := unix.BytePtrFromString(path) + if err != nil { + return err + } + bytes := toOpendalBytes(data) + if len(data) > 0 { + bytes.data = &data[0] + } + var e *opendalError + ffiCall( + unsafe.Pointer(&e), + unsafe.Pointer(&op), + unsafe.Pointer(&bytePath), + unsafe.Pointer(&bytes), + ) + return parseError(ctx, e) + } +}) + +const symOperatorCreateDir = "opendal_operator_create_dir" + +type operatorCreateDir func(op *opendalOperator, path string) error + +var withOperatorCreateDir = withFFI(ffiOpts{ + sym: symOperatorCreateDir, + rType: &ffi.TypePointer, + aTypes: []*ffi.Type{&ffi.TypePointer, &ffi.TypePointer}, +}, func(ctx context.Context, ffiCall func(rValue unsafe.Pointer, aValues ...unsafe.Pointer)) operatorCreateDir { + return func(op *opendalOperator, path string) error { + bytePath, err := unix.BytePtrFromString(path) + if err != nil { + return err + } + var e *opendalError + ffiCall( + unsafe.Pointer(&e), + unsafe.Pointer(&op), + unsafe.Pointer(&bytePath), + ) + return parseError(ctx, e) + } +}) diff --git a/bindings/go/write_test.go b/bindings/go/write_test.go new file mode 100644 index 00000000000..69cece7d436 --- /dev/null +++ b/bindings/go/write_test.go @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package opendal_test + +import ( + "github.com/apache/opendal/bindings/go" + "github.com/google/uuid" + "github.com/stretchr/testify/require" +) + +func testsWrite(cap *opendal.Capability) []behaviorTest { + if !cap.Write() || !cap.Stat() || !cap.Read() { + return nil + } + return []behaviorTest{ + testWriteOnly, + testWriteWithEmptyContent, + testWriteWithDirPath, + testWriteWithSpecialChars, + testWriteOverwrite, + } +} + +func testWriteOnly(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + path, content, size := fixture.NewFile() + + assert.Nil(op.Write(path, content)) + + meta, err := op.Stat(path) + assert.Nil(err, "stat must succeed") + assert.Equal(uint64(size), meta.ContentLength()) +} + +func testWriteWithEmptyContent(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + if !op.Info().GetFullCapability().WriteCanEmpty() { + return + } + + path := fixture.NewFilePath() + assert.Nil(op.Write(path, []byte{})) + + meta, err := op.Stat(path) + assert.Nil(err, "stat must succeed") + assert.Equal(uint64(0), meta.ContentLength()) +} + +func testWriteWithDirPath(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + path := fixture.NewDirPath() + + err := op.Write(path, []byte("1")) + assert.NotNil(err) + assert.Equal(opendal.CodeIsADirectory, assertErrorCode(err)) +} + +func testWriteWithSpecialChars(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + path, content, size := fixture.NewFileWithPath(uuid.NewString() + " !@#$%^&()_+-=;',.txt") + + assert.Nil(op.Write(path, content)) + + meta, err := op.Stat(path) + assert.Nil(err, "stat must succeed") + assert.Equal(uint64(size), meta.ContentLength()) +} + +func testWriteOverwrite(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + if !op.Info().GetFullCapability().WriteCanMulti() { + return + } + + path := fixture.NewFilePath() + size := uint(5 * 1024 * 1024) + contentOne, contentTwo := genFixedBytes(size), genFixedBytes(size) + + assert.Nil(op.Write(path, contentOne)) + bs, err := op.Read(path) + assert.Nil(err, "read must succeed") + assert.Equal(contentOne, bs, "read content_one") + + assert.Nil(op.Write(path, contentTwo)) + bs, err = op.Read(path) + assert.Nil(err, "read must succeed") + assert.NotEqual(contentOne, bs, "content_one must be overwrote") + assert.Equal(contentTwo, bs, "read content_two") +}