diff --git a/.github/scripts/test_go_binding/matrix.yaml b/.github/scripts/test_go_binding/matrix.yaml index b64fa19b6a21..eb5eee32e559 100644 --- a/.github/scripts/test_go_binding/matrix.yaml +++ b/.github/scripts/test_go_binding/matrix.yaml @@ -26,6 +26,11 @@ build: goos: "darwin" goarch: "arm64" os: "macos-latest" + - target: "x86_64-pc-windows-msvc" + cc: "cl.exe" + goos: "windows" + goarch: "amd64" + os: "windows-latest" service: - "fs" diff --git a/.github/workflows/ci_bindings_go.yml b/.github/workflows/ci_bindings_go.yml index e611e98e9510..dc46378b598e 100644 --- a/.github/workflows/ci_bindings_go.yml +++ b/.github/workflows/ci_bindings_go.yml @@ -31,6 +31,7 @@ on: - "bindings/c/**" - "bindings/go/**" - ".github/workflows/ci_bindings_go.yml" + - ".github/scripts/test_go_binding/matrix.yaml" workflow_dispatch: concurrency: @@ -82,10 +83,17 @@ jobs: path: "tools" - name: Setup Rust toolchain uses: ./.github/actions/setup - - name: Setup Target + - name: Setup Target (Linux/macOS) + if: runner.os != 'Windows' env: TARGET: ${{ matrix.build.target }} run: rustup target add $TARGET + - name: Setup Target (Windows) + if: runner.os == 'Windows' + env: + TARGET: ${{ matrix.build.target }} + run: | + rustup target add $env:TARGET - uses: actions/setup-go@v5 with: go-version: stable @@ -104,8 +112,12 @@ jobs: - name: Install dependencies (macOS) if: ${{ matrix.build.os == 'macos-latest' }} run: brew install zstd libffi - - name: Build C Binding + - name: Install dependencies (Windows) + if: ${{ matrix.build.os == 'windows-latest' }} + uses: ilammy/msvc-dev-cmd@v1 + - name: Build C Binding (Linux/macOS) working-directory: bindings/c + if: runner.os != 'Windows' env: VERSION: "latest" SERVICE: ${{ matrix.service }} @@ -113,7 +125,7 @@ jobs: CC: ${{ matrix.build.cc }} OS: ${{ matrix.build.os }} run: | - cargo build --target $TARGET --release + cargo build --target $TARGET --release DIR=$GITHUB_WORKSPACE/libopendal_c_${VERSION}_${SERVICE}_$TARGET mkdir $DIR if [ ${OS} == 'ubuntu-latest' ]; then @@ -122,6 +134,21 @@ jobs: SO=dylib fi zstd -19 ./target/$TARGET/release/libopendal_c.$SO -o $DIR/libopendal_c.$TARGET.$SO.zst + - name: Build C Binding (Windows) + working-directory: bindings/c + if: runner.os == 'Windows' + env: + VERSION: "latest" + SERVICE: ${{ matrix.service }} + TARGET: ${{ matrix.build.target }} + CC: ${{ matrix.build.cc }} + OS: ${{ matrix.build.os }} + run: | + cargo build --target $env:TARGET --release + $DIR="$env:GITHUB_WORKSPACE\libopendal_c_${env:VERSION}_${env:SERVICE}_${env:TARGET}" + Rename-Item -Path "./target/$env:TARGET/release/opendal_c.dll" -NewName "libopendal_c.dll" + New-Item -ItemType Directory -Force -Path $DIR + zstd -19 "./target/${env:TARGET}/release/libopendal_c.dll" -o "$DIR/libopendal_c.${env:TARGET}.dll.zst" - name: Build Go Artifact working-directory: tools/internal/generate env: @@ -129,22 +156,43 @@ jobs: VERSION: "latest" run: | go run generate.go - - name: Setup Go Workspace + - name: Setup Go Workspace (Linux/macOS) env: SERVICE: ${{ matrix.service }} working-directory: bindings/go/tests + if: runner.os != 'Windows' run: | go work init go work use .. go work use ./behavior_tests go work use $GITHUB_WORKSPACE/$(echo $SERVICE | sed 's/-/_/g') - - name: Run tests + - name: Setup Go Workspace (Windows) + env: + SERVICE: ${{ matrix.service }} + working-directory: bindings/go/tests + if: runner.os == 'Windows' + run: | + go work init + go work use .. + go work use ./behavior_tests + go work use $env:GITHUB_WORKSPACE/$($env:SERVICE -replace '-','_') + - name: Run tests (Linux/macOS) env: OPENDAL_TEST: ${{ matrix.service }} - OPENDAL_FS_ROOT: "/tmp/opendal/" + OPENDAL_FS_ROOT: runner.temp working-directory: bindings/go/tests/behavior_tests + if: runner.os != 'Windows' run: | if [ ${{ matrix.build.os }} == 'macos-latest' ]; then export DYLD_FALLBACK_LIBRARY_PATH=$DYLD_FALLBACK_LIBRARY_PATH:/opt/homebrew/opt/libffi/lib fi CGO_ENABLE=0 go test -v -run TestBehavior + - name: Run tests (Windows) + env: + OPENDAL_TEST: ${{ matrix.service }} + OPENDAL_FS_ROOT: runner.temp + working-directory: bindings/go/tests/behavior_tests + if: runner.os == 'Windows' + run: | + $env:CGO_ENABLE = "0" + go test -v -run TestBehavior diff --git a/bindings/go/delete.go b/bindings/go/delete.go index f92df487ad30..1a8b64317bdf 100644 --- a/bindings/go/delete.go +++ b/bindings/go/delete.go @@ -24,7 +24,6 @@ import ( "unsafe" "github.com/jupiterrider/ffi" - "golang.org/x/sys/unix" ) // Delete removes the file or directory at the specified path. @@ -55,7 +54,7 @@ var withOperatorDelete = withFFI(ffiOpts{ aTypes: []*ffi.Type{&ffi.TypePointer, &ffi.TypePointer}, }, func(ctx context.Context, ffiCall func(rValue unsafe.Pointer, aValues ...unsafe.Pointer)) operatorDelete { return func(op *opendalOperator, path string) error { - bytePath, err := unix.BytePtrFromString(path) + bytePath, err := BytePtrFromString(path) if err != nil { return err } diff --git a/bindings/go/ffi.go b/bindings/go/ffi.go index 931c345f2a07..4cca16170014 100644 --- a/bindings/go/ffi.go +++ b/bindings/go/ffi.go @@ -24,12 +24,11 @@ import ( "errors" "unsafe" - "github.com/ebitengine/purego" "github.com/jupiterrider/ffi" ) func contextWithFFIs(path string) (ctx context.Context, cancel context.CancelFunc, err error) { - libopendal, err := purego.Dlopen(path, purego.RTLD_LAZY|purego.RTLD_GLOBAL) + libopendal, err := LoadLibrary(path) if err != nil { return } @@ -41,7 +40,7 @@ func contextWithFFIs(path string) (ctx context.Context, cancel context.CancelFun } } cancel = func() { - purego.Dlclose(libopendal) + _ = FreeLibrary(libopendal) } return } @@ -83,7 +82,7 @@ func withFFI[T any]( ); status != ffi.OK { return nil, errors.New(status.String()) } - fn, err := purego.Dlsym(libopendal, opts.sym.String()) + fn, err := GetProcAddress(libopendal, opts.sym.String()) if err != nil { return nil, err } diff --git a/bindings/go/go.mod b/bindings/go/go.mod index ac1f39f5c3b4..17bc50945628 100644 --- a/bindings/go/go.mod +++ b/bindings/go/go.mod @@ -22,7 +22,7 @@ go 1.22.4 toolchain go1.22.5 require ( - github.com/ebitengine/purego v0.7.1 - github.com/jupiterrider/ffi v0.1.0 + github.com/ebitengine/purego v0.8.2 + github.com/jupiterrider/ffi v0.4.0 golang.org/x/sys v0.24.0 ) diff --git a/bindings/go/go.sum b/bindings/go/go.sum index b08aa464ca32..ac2837037129 100644 --- a/bindings/go/go.sum +++ b/bindings/go/go.sum @@ -1,6 +1,10 @@ github.com/ebitengine/purego v0.7.1 h1:6/55d26lG3o9VCZX8lping+bZcmShseiqlh2bnUDiPA= github.com/ebitengine/purego v0.7.1/go.mod h1:ah1In8AOtksoNK6yk5z1HTJeUkC1Ez4Wk2idgGslMwQ= +github.com/ebitengine/purego v0.8.2 h1:jPPGWs2sZ1UgOSgD2bClL0MJIqu58nOmIcBuXr62z1I= +github.com/ebitengine/purego v0.8.2/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= github.com/jupiterrider/ffi v0.1.0 h1:OI6ZHZJW1Io1PcfqGeLk/CBLj+//8aNdOuo558ykQQo= github.com/jupiterrider/ffi v0.1.0/go.mod h1:tyr9EitV+PW99I6137IDwdO6ZzNyFp/noXNSfU3OYqk= +github.com/jupiterrider/ffi v0.4.0 h1:7mhlrfiBZa0kHhh2DV7mGAdXN/D8zDeu8UlaBO+ZSko= +github.com/jupiterrider/ffi v0.4.0/go.mod h1:1QCaf2VVPpGyIeU3RqQ2rHYrAPT8m9l0GhQupVYQB24= golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= diff --git a/bindings/go/lister.go b/bindings/go/lister.go index bbd58f1f1196..a328f3d05b1f 100644 --- a/bindings/go/lister.go +++ b/bindings/go/lister.go @@ -24,7 +24,6 @@ import ( "unsafe" "github.com/jupiterrider/ffi" - "golang.org/x/sys/unix" ) // Check verifies if the operator is functioning correctly. @@ -312,7 +311,7 @@ var withOperatorList = withFFI(ffiOpts{ aTypes: []*ffi.Type{&ffi.TypePointer, &ffi.TypePointer}, }, func(ctx context.Context, ffiCall func(rValue unsafe.Pointer, aValues ...unsafe.Pointer)) operatorList { return func(op *opendalOperator, path string) (*opendalLister, error) { - bytePath, err := unix.BytePtrFromString(path) + bytePath, err := BytePtrFromString(path) if err != nil { return nil, err } @@ -400,7 +399,7 @@ var withEntryName = withFFI(ffiOpts{ unsafe.Pointer(&bytePtr), unsafe.Pointer(&e), ) - return unix.BytePtrToString(bytePtr) + return BytePtrToString(bytePtr) } }) @@ -419,6 +418,6 @@ var withEntryPath = withFFI(ffiOpts{ unsafe.Pointer(&bytePtr), unsafe.Pointer(&e), ) - return unix.BytePtrToString(bytePtr) + return BytePtrToString(bytePtr) } }) diff --git a/bindings/go/operator.go b/bindings/go/operator.go index d63bacda61ee..304b05d06af7 100644 --- a/bindings/go/operator.go +++ b/bindings/go/operator.go @@ -24,7 +24,6 @@ import ( "unsafe" "github.com/jupiterrider/ffi" - "golang.org/x/sys/unix" ) // Copy duplicates a file from the source path to the destination path. @@ -111,7 +110,7 @@ var withOperatorNew = withFFI(ffiOpts{ }, func(ctx context.Context, ffiCall func(rValue unsafe.Pointer, aValues ...unsafe.Pointer)) operatorNew { return func(scheme Scheme, opts *operatorOptions) (op *opendalOperator, err error) { var byteName *byte - byteName, err = unix.BytePtrFromString(scheme.Name()) + byteName, err = BytePtrFromString(scheme.Name()) if err != nil { return } @@ -177,11 +176,11 @@ var withOperatorOptionsSet = withFFI(ffiOpts{ byteKey *byte byteValue *byte ) - byteKey, err = unix.BytePtrFromString(key) + byteKey, err = BytePtrFromString(key) if err != nil { return err } - byteValue, err = unix.BytePtrFromString(value) + byteValue, err = BytePtrFromString(value) if err != nil { return err } @@ -226,11 +225,11 @@ var withOperatorCopy = withFFI(ffiOpts{ byteSrc *byte byteDest *byte ) - byteSrc, err = unix.BytePtrFromString(src) + byteSrc, err = BytePtrFromString(src) if err != nil { return err } - byteDest, err = unix.BytePtrFromString(dest) + byteDest, err = BytePtrFromString(dest) if err != nil { return err } @@ -259,11 +258,11 @@ var withOperatorRename = withFFI(ffiOpts{ byteSrc *byte byteDest *byte ) - byteSrc, err = unix.BytePtrFromString(src) + byteSrc, err = BytePtrFromString(src) if err != nil { return err } - byteDest, err = unix.BytePtrFromString(dest) + byteDest, err = BytePtrFromString(dest) if err != nil { return err } diff --git a/bindings/go/operator_info.go b/bindings/go/operator_info.go index 6ea7f15fcd10..9fffe909c5b9 100644 --- a/bindings/go/operator_info.go +++ b/bindings/go/operator_info.go @@ -24,7 +24,6 @@ import ( "unsafe" "github.com/jupiterrider/ffi" - "golang.org/x/sys/unix" ) // Info returns metadata about the Operator. @@ -325,7 +324,7 @@ var withOperatorInfoGetScheme = withFFI(ffiOpts{ unsafe.Pointer(&bytePtr), unsafe.Pointer(&info), ) - return unix.BytePtrToString(bytePtr) + return BytePtrToString(bytePtr) } }) @@ -344,7 +343,7 @@ var withOperatorInfoGetRoot = withFFI(ffiOpts{ unsafe.Pointer(&bytePtr), unsafe.Pointer(&info), ) - return unix.BytePtrToString(bytePtr) + return BytePtrToString(bytePtr) } }) @@ -363,6 +362,6 @@ var withOperatorInfoGetName = withFFI(ffiOpts{ unsafe.Pointer(&bytePtr), unsafe.Pointer(&info), ) - return unix.BytePtrToString(bytePtr) + return BytePtrToString(bytePtr) } }) diff --git a/bindings/go/reader.go b/bindings/go/reader.go index ce26ce663ecd..52810ddebac9 100644 --- a/bindings/go/reader.go +++ b/bindings/go/reader.go @@ -25,7 +25,6 @@ import ( "unsafe" "github.com/jupiterrider/ffi" - "golang.org/x/sys/unix" ) // Read reads the entire contents of the file at the specified path into a byte slice. @@ -210,7 +209,7 @@ var withOperatorRead = withFFI(ffiOpts{ aTypes: []*ffi.Type{&ffi.TypePointer, &ffi.TypePointer}, }, func(ctx context.Context, ffiCall func(rValue unsafe.Pointer, aValues ...unsafe.Pointer)) operatorRead { return func(op *opendalOperator, path string) (opendalBytes, error) { - bytePath, err := unix.BytePtrFromString(path) + bytePath, err := BytePtrFromString(path) if err != nil { return opendalBytes{}, err } @@ -234,7 +233,7 @@ var withOperatorReader = withFFI(ffiOpts{ aTypes: []*ffi.Type{&ffi.TypePointer, &ffi.TypePointer}, }, func(ctx context.Context, ffiCall func(rValue unsafe.Pointer, aValues ...unsafe.Pointer)) operatorReader { return func(op *opendalOperator, path string) (*opendalReader, error) { - bytePath, err := unix.BytePtrFromString(path) + bytePath, err := BytePtrFromString(path) if err != nil { return nil, err } diff --git a/bindings/go/stat.go b/bindings/go/stat.go index 7eb7345939a0..feb37d860a5a 100644 --- a/bindings/go/stat.go +++ b/bindings/go/stat.go @@ -24,7 +24,6 @@ import ( "unsafe" "github.com/jupiterrider/ffi" - "golang.org/x/sys/unix" ) // Stat retrieves metadata for the specified path. @@ -111,7 +110,7 @@ var withOperatorStat = withFFI(ffiOpts{ aTypes: []*ffi.Type{&ffi.TypePointer, &ffi.TypePointer}, }, func(ctx context.Context, ffiCall func(rValue unsafe.Pointer, aValues ...unsafe.Pointer)) operatorStat { return func(op *opendalOperator, path string) (*opendalMetadata, error) { - bytePath, err := unix.BytePtrFromString(path) + bytePath, err := BytePtrFromString(path) if err != nil { return nil, err } @@ -138,7 +137,7 @@ var withOperatorIsExists = withFFI(ffiOpts{ aTypes: []*ffi.Type{&ffi.TypePointer, &ffi.TypePointer}, }, func(ctx context.Context, ffiCall func(rValue unsafe.Pointer, aValues ...unsafe.Pointer)) operatorIsExist { return func(op *opendalOperator, path string) (bool, error) { - bytePath, err := unix.BytePtrFromString(path) + bytePath, err := BytePtrFromString(path) if err != nil { return false, err } diff --git a/bindings/go/util_unix.go b/bindings/go/util_unix.go new file mode 100644 index 000000000000..a8f67740683a --- /dev/null +++ b/bindings/go/util_unix.go @@ -0,0 +1,67 @@ +//go:build !windows + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package opendal + +import ( + "github.com/ebitengine/purego" + "golang.org/x/sys/unix" +) + +func BytePtrFromString(s string) (*byte, error) { + if s == "" { + return new(byte), nil + } + return unix.BytePtrFromString(s) +} + +func BytePtrToString(p *byte) string { + if p == nil { + return "" + } + return unix.BytePtrToString(p) +} + +func LoadLibrary(path string) (uintptr, error) { + return purego.Dlopen(path, purego.RTLD_LAZY|purego.RTLD_GLOBAL) +} + +func FreeLibrary(handle uintptr) error { + if handle == 0 { + return nil + } + err := purego.Dlclose(handle) + if err != nil { + return err + } + return nil +} + +func GetProcAddress(handle uintptr, name string) (uintptr, error) { + if handle == 0 { + return 0, nil + } + addr, err := purego.Dlsym(handle, name) + if err != nil { + return 0, err + } + return addr, nil +} diff --git a/bindings/go/util_windows.go b/bindings/go/util_windows.go new file mode 100644 index 000000000000..3d095b4a0d99 --- /dev/null +++ b/bindings/go/util_windows.go @@ -0,0 +1,70 @@ +//go:build windows + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package opendal + +import ( + "golang.org/x/sys/windows" +) + +func BytePtrFromString(s string) (*byte, error) { + if s == "" { + return new(byte), nil + } + return windows.BytePtrFromString(s) +} + +func BytePtrToString(p *byte) string { + if p == nil { + return "" + } + return windows.BytePtrToString(p) +} + +func LoadLibrary(path string) (uintptr, error) { + handle, err := windows.LoadLibrary(path) + if err != nil { + return 0, err + } + return uintptr(handle), nil +} + +func FreeLibrary(handle uintptr) error { + if handle == 0 { + return nil + } + err := windows.FreeLibrary(windows.Handle(handle)) + if err != nil { + return err + } + return nil +} + +func GetProcAddress(handle uintptr, name string) (uintptr, error) { + if handle == 0 { + return 0, nil + } + proc, err := windows.GetProcAddress(windows.Handle(handle), name) + if err != nil { + return 0, err + } + return proc, nil +} diff --git a/bindings/go/write.go b/bindings/go/write.go index f0c23b816f5e..a426c9eaee90 100644 --- a/bindings/go/write.go +++ b/bindings/go/write.go @@ -25,7 +25,6 @@ import ( "unsafe" "github.com/jupiterrider/ffi" - "golang.org/x/sys/unix" ) // Write writes the given bytes to the specified path. @@ -199,7 +198,7 @@ var withOperatorWrite = withFFI(ffiOpts{ aTypes: []*ffi.Type{&ffi.TypePointer, &ffi.TypePointer, &ffi.TypePointer}, }, func(ctx context.Context, ffiCall func(rValue unsafe.Pointer, aValues ...unsafe.Pointer)) operatorWrite { return func(op *opendalOperator, path string, data []byte) error { - bytePath, err := unix.BytePtrFromString(path) + bytePath, err := BytePtrFromString(path) if err != nil { return err } @@ -225,7 +224,7 @@ var withOperatorCreateDir = withFFI(ffiOpts{ aTypes: []*ffi.Type{&ffi.TypePointer, &ffi.TypePointer}, }, func(ctx context.Context, ffiCall func(rValue unsafe.Pointer, aValues ...unsafe.Pointer)) operatorCreateDir { return func(op *opendalOperator, path string) error { - bytePath, err := unix.BytePtrFromString(path) + bytePath, err := BytePtrFromString(path) if err != nil { return err } @@ -249,7 +248,7 @@ var withOperatorWriter = withFFI(ffiOpts{ aTypes: []*ffi.Type{&ffi.TypePointer, &ffi.TypePointer}, }, func(ctx context.Context, ffiCall func(rValue unsafe.Pointer, aValues ...unsafe.Pointer)) operatorWriter { return func(op *opendalOperator, path string) (*opendalWriter, error) { - bytePath, err := unix.BytePtrFromString(path) + bytePath, err := BytePtrFromString(path) if err != nil { return nil, err }