Skip to content

fix(c): Generate versioned DLLs and import LIBs when building with MSVC #2858

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 22 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 32 additions & 0 deletions c/CMakePresets.json
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,26 @@
"ADBC_USE_UBSAN": "ON"
}
},
{
"name": "multi-config-windows",
"displayName": "Multi config for Windows without ASAN/UBSAN",
"generator": "Ninja Multi-Config",
"toolchainFile": "$env{VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake",
"cacheVariables": {
"CMAKE_EXPORT_COMPILE_COMMANDS": "ON",
"ADBC_BUILD_TESTS": "ON",
"ADBC_DRIVER_FLIGHTSQL": "ON",
"ADBC_DRIVER_MANAGER": "ON",
"ADBC_DRIVER_POSTGRESQL": "ON",
"ADBC_DRIVER_SNOWFLAKE": "ON",
"ADBC_DRIVER_SQLITE": "ON",
"ADBC_BUILD_SHARED": "ON",
"ADBC_BUILD_STATIC": "OFF",
"ADBC_USE_ASAN": "OFF",
"ADBC_USE_UBSAN": "OFF",
"CMAKE_TOOLCHAIN_FILE": "$env{VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake"
}
},
{
"name": "debug-python",
"displayName": "debug, all drivers, with tests, without ASan/UBSan (usable from Python)",
Expand All @@ -41,6 +61,18 @@
}
}
],
"buildPresets": [
{
"name": "debug-windows",
"configurePreset": "multi-config-windows",
"configuration": "Debug"
},
{
"name": "release-windows",
"configurePreset": "multi-config-windows",
"configuration": "Release"
}
],
"testPresets": [
{
"name": "debug",
Expand Down
14 changes: 13 additions & 1 deletion c/cmake_modules/AdbcDefines.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ if(MSVC)
add_compile_options(/wd5027)
add_compile_options(/wd5039)
add_compile_options(/wd5045)
add_compile_options(/wd5246)
add_compile_options(/wd5246)
elseif(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang"
OR CMAKE_CXX_COMPILER_ID STREQUAL "Clang"
OR CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
Expand Down Expand Up @@ -143,6 +143,18 @@ endmacro()
# Common testing setup
add_custom_target(all-tests)
if(ADBC_BUILD_TESTS)
if(MSVC)
# MSVC emitted warnings for testing code
add_compile_options(/wd5204)
add_compile_options(/wd5026)
add_compile_options(/wd4266)
add_compile_options(/wd4265)
add_compile_options(/wd5262)
add_compile_options(/wd4146)
add_compile_options(/wd4244)
add_compile_options(/wd4307)
endif()

find_package(GTest)
if(NOT GTest_FOUND)
message(STATUS "Building googletest from source")
Expand Down
15 changes: 15 additions & 0 deletions c/cmake_modules/BuildUtils.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -240,6 +240,21 @@ function(ADD_ARROW_LIB LIB_NAME)
OUTPUT_NAME ${LIB_NAME}
VERSION "${ADBC_FULL_SO_VERSION}"
SOVERSION "${ADBC_SO_VERSION}")

if(MSVC)
# Binaries generated on Windows need file version information, otherwise when the binary is part of a Windows installer
# the installer won't know to update a previously installed version.
set(VERSION_RC_TEMPLATE "${CMAKE_SOURCE_DIR}/cmake_modules/version.rc.in")
configure_file(
"${VERSION_RC_TEMPLATE}"
"${CMAKE_CURRENT_BINARY_DIR}/${LIB_NAME}_version.rc"
@ONLY
)

target_sources(${LIB_NAME}_shared PRIVATE
"${CMAKE_CURRENT_BINARY_DIR}/${LIB_NAME}_version.rc"
)
endif()

# https://github.com/apache/arrow-adbc/issues/81
target_compile_features(${LIB_NAME}_shared PRIVATE cxx_std_11)
Expand Down
129 changes: 98 additions & 31 deletions c/cmake_modules/GoUtils.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,13 @@
find_program(GO_BIN "go" REQUIRED)
message(STATUS "Detecting Go executable: Found ${GO_BIN}")

# Find tools for generating import libraries on Windows
if(MSVC)
# msys64 is needed for dlltool and gendef
find_program(DLLTOOL_BIN "dlltool")
find_program(GENDEF_BIN "gendef")
endif()

set(ADBC_GO_PACKAGE_INIT
[=[
get_filename_component(_IMPORT_PREFIX "${CMAKE_CURRENT_LIST_FILE}" PATH)
Expand Down Expand Up @@ -65,6 +72,19 @@ function(adbc_add_static_library target_name base_name)
endfunction()
]=])

# Function to generate import library from DLL on Windows
function(generate_import_library DLL_PATH LIB_PATH DEF_PATH)
add_custom_command(OUTPUT "${DEF_PATH}"
COMMAND ${GENDEF_BIN} - "${DLL_PATH}" > "${DEF_PATH}"
DEPENDS "${DLL_PATH}"
COMMENT "Generating .def file from ${DLL_PATH}")

add_custom_command(OUTPUT "${LIB_PATH}"
COMMAND ${DLLTOOL_BIN} -d "${DEF_PATH}" -l "${LIB_PATH}" -D "${DLL_PATH}"
DEPENDS "${DEF_PATH}"
COMMENT "Generating import library ${LIB_PATH}")
endfunction()

function(add_go_lib GO_MOD_DIR GO_LIBNAME)
set(options)
set(one_value_args
Expand Down Expand Up @@ -180,36 +200,78 @@ function(add_go_lib GO_MOD_DIR GO_LIBNAME)
list(APPEND GO_ENV_VARS "GOARCH=arm64")
endif()

add_custom_command(OUTPUT "${LIBOUT_SHARED}.${ADBC_FULL_SO_VERSION}"
WORKING_DIRECTORY ${GO_MOD_DIR}
DEPENDS ${ARG_SOURCES}
COMMAND ${CMAKE_COMMAND} -E env ${GO_ENV_VARS} ${GO_BIN} build
${GO_BUILD_TAGS} "${GO_BUILD_FLAGS}" -o
${LIBOUT_SHARED}.${ADBC_FULL_SO_VERSION}
-buildmode=c-shared ${GO_LDFLAGS} .
COMMAND ${CMAKE_COMMAND} -E remove -f
"${LIBOUT_SHARED}.${ADBC_SO_VERSION}.0.h"
COMMENT "Building Go Shared lib ${GO_LIBNAME}"
COMMAND_EXPAND_LISTS)
if(MSVC)
# On Windows with MSVC, generate a lib from the DLL file created by go. Binaries generated on Windows have their version
# as part of the DLL/file details, therefore the generated DLL file name does not need to have the version info.
set(LIBOUT_IMPORT_LIB "${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_IMPORT_LIBRARY_PREFIX}${GO_LIBNAME}${CMAKE_IMPORT_LIBRARY_SUFFIX}")
set(LIBOUT_DEF_FILE "${CMAKE_CURRENT_BINARY_DIR}/${GO_LIBNAME}.def")

add_custom_command(OUTPUT "${LIBOUT_SHARED}"
WORKING_DIRECTORY ${GO_MOD_DIR}
DEPENDS ${ARG_SOURCES}
COMMAND ${CMAKE_COMMAND} -E env ${GO_ENV_VARS} ${GO_BIN} build
${GO_BUILD_TAGS} "${GO_BUILD_FLAGS}" -o
${LIBOUT_SHARED}
-buildmode=c-shared ${GO_LDFLAGS} .
COMMAND ${CMAKE_COMMAND} -E remove -f
"${LIBOUT_SHARED}.h"
COMMENT "Building Go Shared lib ${GO_LIBNAME}"
COMMAND_EXPAND_LISTS)

# Generate import library if tools are available
if(GENDEF_BIN AND DLLTOOL_BIN)
generate_import_library("${LIBOUT_SHARED}" "${LIBOUT_IMPORT_LIB}" "${LIBOUT_DEF_FILE}")
set(IMPORT_LIB_OUTPUTS "${LIBOUT_IMPORT_LIB}" "${LIBOUT_DEF_FILE}")
else()
set(IMPORT_LIB_OUTPUTS)
message(WARNING "Import library generation tools not found. You may need to link directly to the DLL or use delay-loading.")
endif()

add_custom_command(OUTPUT "${LIBOUT_SHARED}.${ADBC_SO_VERSION}" "${LIBOUT_SHARED}"
DEPENDS "${LIBOUT_SHARED}.${ADBC_FULL_SO_VERSION}"
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
COMMAND ${CMAKE_COMMAND} -E create_symlink
"${LIB_NAME_SHARED}.${ADBC_FULL_SO_VERSION}"
"${LIB_NAME_SHARED}.${ADBC_SO_VERSION}"
COMMAND ${CMAKE_COMMAND} -E create_symlink
"${LIB_NAME_SHARED}.${ADBC_SO_VERSION}"
"${LIB_NAME_SHARED}")

add_custom_target(${GO_LIBNAME}_target ALL
DEPENDS "${LIBOUT_SHARED}.${ADBC_FULL_SO_VERSION}"
"${LIBOUT_SHARED}.${ADBC_SO_VERSION}" "${LIBOUT_SHARED}")
add_library(${GO_LIBNAME}_shared SHARED IMPORTED GLOBAL)
set_target_properties(${GO_LIBNAME}_shared
PROPERTIES IMPORTED_LOCATION
"${LIBOUT_SHARED}.${ADBC_FULL_SO_VERSION}"
IMPORTED_SONAME "${LIB_NAME_SHARED}")
add_custom_target(${GO_LIBNAME}_target ALL
DEPENDS "${LIBOUT_SHARED}"
${IMPORT_LIB_OUTPUTS})

add_library(${GO_LIBNAME}_shared SHARED IMPORTED GLOBAL)
set_target_properties(${GO_LIBNAME}_shared
PROPERTIES IMPORTED_LOCATION "${LIBOUT_SHARED}")

# Set import library if it was generated
if(IMPORT_LIB_OUTPUTS)
set_target_properties(${GO_LIBNAME}_shared
PROPERTIES IMPORTED_IMPLIB "${LIBOUT_IMPORT_LIB}")
endif()
else()
add_custom_command(OUTPUT "${LIBOUT_SHARED}.${ADBC_FULL_SO_VERSION}"
WORKING_DIRECTORY ${GO_MOD_DIR}
DEPENDS ${ARG_SOURCES}
COMMAND ${CMAKE_COMMAND} -E env ${GO_ENV_VARS} ${GO_BIN} build
${GO_BUILD_TAGS} "${GO_BUILD_FLAGS}" -o
${LIBOUT_SHARED}.${ADBC_FULL_SO_VERSION}
-buildmode=c-shared ${GO_LDFLAGS} .
COMMAND ${CMAKE_COMMAND} -E remove -f
"${LIBOUT_SHARED}.${ADBC_SO_VERSION}.0.h"
COMMENT "Building Go Shared lib ${GO_LIBNAME}"
COMMAND_EXPAND_LISTS)

add_custom_command(OUTPUT "${LIBOUT_SHARED}.${ADBC_SO_VERSION}" "${LIBOUT_SHARED}"
DEPENDS "${LIBOUT_SHARED}.${ADBC_FULL_SO_VERSION}"
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
COMMAND ${CMAKE_COMMAND} -E create_symlink
"${LIB_NAME_SHARED}.${ADBC_FULL_SO_VERSION}"
"${LIB_NAME_SHARED}.${ADBC_SO_VERSION}"
COMMAND ${CMAKE_COMMAND} -E create_symlink
"${LIB_NAME_SHARED}.${ADBC_SO_VERSION}"
"${LIB_NAME_SHARED}")

add_custom_target(${GO_LIBNAME}_target ALL
DEPENDS "${LIBOUT_SHARED}.${ADBC_FULL_SO_VERSION}"
"${LIBOUT_SHARED}.${ADBC_SO_VERSION}" "${LIBOUT_SHARED}")
add_library(${GO_LIBNAME}_shared SHARED IMPORTED GLOBAL)
set_target_properties(${GO_LIBNAME}_shared
PROPERTIES IMPORTED_LOCATION
"${LIBOUT_SHARED}.${ADBC_FULL_SO_VERSION}"
IMPORTED_SONAME "${LIB_NAME_SHARED}")
endif()
add_dependencies(${GO_LIBNAME}_shared ${GO_LIBNAME}_target)
if(ARG_OUTPUTS)
list(APPEND ${ARG_OUTPUTS} ${GO_LIBNAME}_shared)
Expand Down Expand Up @@ -239,6 +301,9 @@ function(add_go_lib GO_MOD_DIR GO_LIBNAME)
if(WIN32)
install(PROGRAMS $<TARGET_FILE:${GO_LIBNAME}_shared> ${INSTALL_IS_OPTIONAL}
DESTINATION ${RUNTIME_INSTALL_DIR})
if(IMPORT_LIB_OUTPUTS)
install(FILES "${LIBOUT_IMPORT_LIB}" TYPE LIB)
endif()
else()
install(PROGRAMS $<TARGET_FILE:${GO_LIBNAME}_shared> ${INSTALL_IS_OPTIONAL}
TYPE LIB)
Expand All @@ -255,8 +320,10 @@ function(add_go_lib GO_MOD_DIR GO_LIBNAME)
${CMAKE_INSTALL_LIBDIR})
endif()
if(WIN32)
# This symlink doesn't get installed
install(FILES "${LIBOUT_SHARED}.${ADBC_SO_VERSION}" TYPE BIN)
install(FILES "${LIBOUT_SHARED}" TYPE BIN)
if(IMPORT_LIB_OUTPUTS)
install(FILES "${LIBOUT_IMPORT_LIB}" TYPE LIB)
endif()
else()
install(FILES "${LIBOUT_SHARED}" "${LIBOUT_SHARED}.${ADBC_SO_VERSION}" TYPE LIB)
endif()
Expand Down
27 changes: 27 additions & 0 deletions c/cmake_modules/version.rc.in
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
#include <windows.h>

VS_VERSION_INFO VERSIONINFO
FILEVERSION @ADBC_VERSION_MAJOR@,@ADBC_VERSION_MINOR@,@ADBC_VERSION_PATCH@,0
PRODUCTVERSION @ADBC_VERSION_MAJOR@,@ADBC_VERSION_MINOR@,@ADBC_VERSION_PATCH@,0
FILEFLAGSMASK VS_FFI_FILEFLAGSMASK
FILEFLAGS 0x0L
FILEOS VOS__WINDOWS32
FILETYPE VFT_DLL
FILESUBTYPE VFT2_UNKNOWN
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we need to specify this with VFT_DLL?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's the main file type value that makes sense for the produced files (DLLs). The rest are unrelated to the DLL file type

BEGIN
BLOCK "StringFileInfo"
BEGIN
BLOCK "040904b0"
BEGIN
VALUE "FileDescription", "ADBC: Arrow Database Connectivity"
VALUE "FileVersion", "@ADBC_FULL_SO_VERSION@"
VALUE "ProductVersion", "@ADBC_FULL_SO_VERSION@"
VALUE "CompanyName", "Apache"
VALUE "ProductName", "@LIB_NAME@"
END
END
BLOCK "VarFileInfo"
BEGIN
VALUE "Translation", 0x409, 1200
END
END
12 changes: 12 additions & 0 deletions c/driver/flightsql/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,18 @@ if(ADBC_BUILD_TESTS)
adbc_driver_common
adbc_validation
${TEST_LINK_LIBS})

if(ADBC_TEST_LINKAGE STREQUAL "shared")
# The go build is not copying the DLL to the test location, causing the test to fail on Windows.
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm, how is this handled for the C/C++ builds? I wouldn't think CMake copies the DLL over...

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It seems by default cmake copies the dll when it is referenced as a library by another cmake project (the test one in this case). However, because the flightsql and snowflake ones are generated manually via GoUtils, they are not getting copied to the test bin folder.

if (MSVC)
add_custom_command(TARGET adbc-driver-flightsql-test POST_BUILD
COMMAND ${CMAKE_COMMAND} -E copy_if_different
"$<TARGET_FILE:adbc_driver_flightsql_shared>"
"$<TARGET_FILE_DIR:adbc-driver-flightsql-test>"
)
endif()
endif()

target_compile_features(adbc-driver-flightsql-test PRIVATE cxx_std_17)
target_include_directories(adbc-driver-flightsql-test SYSTEM
PRIVATE ${REPOSITORY_ROOT}/c/ ${REPOSITORY_ROOT}/c/include/
Expand Down
4 changes: 2 additions & 2 deletions c/driver/postgresql/copy/postgres_copy_reader_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -237,10 +237,10 @@ TEST(PostgresCopyUtilsTest, PostgresCopyReadReal) {
ASSERT_TRUE(ArrowBitGet(validity, 3));
ASSERT_FALSE(ArrowBitGet(validity, 4));

ASSERT_FLOAT_EQ(data_buffer[0], -123.456);
ASSERT_FLOAT_EQ(data_buffer[0], -123.456f);
ASSERT_EQ(data_buffer[1], -1);
ASSERT_EQ(data_buffer[2], 1);
ASSERT_FLOAT_EQ(data_buffer[3], 123.456);
ASSERT_FLOAT_EQ(data_buffer[3], 123.456f);
ASSERT_EQ(data_buffer[4], 0);
}

Expand Down
12 changes: 12 additions & 0 deletions c/driver/snowflake/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,18 @@ if(ADBC_BUILD_TESTS)
adbc_validation
nanoarrow
${TEST_LINK_LIBS})

if(ADBC_TEST_LINKAGE STREQUAL "shared")
# The go build is not copying the DLL to the test location, causing the test to fail on Windows.
if (MSVC)
add_custom_command(TARGET adbc-driver-snowflake-test POST_BUILD
COMMAND ${CMAKE_COMMAND} -E copy_if_different
"$<TARGET_FILE:adbc_driver_snowflake_shared>"
"$<TARGET_FILE_DIR:adbc-driver-snowflake-test>"
)
endif()
endif()

target_compile_features(adbc-driver-snowflake-test PRIVATE cxx_std_17)
target_include_directories(adbc-driver-snowflake-test SYSTEM
PRIVATE ${REPOSITORY_ROOT}/c/ ${REPOSITORY_ROOT}/c/include/
Expand Down
4 changes: 2 additions & 2 deletions c/driver/sqlite/sqlite_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -446,7 +446,7 @@ class SqliteReaderTest : public ::testing::Test {

void Exec(const std::string& query) {
SCOPED_TRACE(query);
int rc = sqlite3_prepare_v2(db, query.c_str(), query.size(), &stmt,
int rc = sqlite3_prepare_v2(db, query.c_str(), static_cast<int>(query.size()), &stmt,
/*pzTail=*/nullptr);
ASSERT_EQ(SQLITE_OK, rc) << "Failed to prepare query: " << sqlite3_errmsg(db);
ASSERT_EQ(SQLITE_DONE, sqlite3_step(stmt));
Expand Down Expand Up @@ -478,7 +478,7 @@ class SqliteReaderTest : public ::testing::Test {

void Exec(const std::string& query, size_t infer_rows,
adbc_validation::StreamReader* reader) {
ASSERT_EQ(SQLITE_OK, sqlite3_prepare_v2(db, query.c_str(), query.size(), &stmt,
ASSERT_EQ(SQLITE_OK, sqlite3_prepare_v2(db, query.c_str(), static_cast<int>(query.size()), &stmt,
/*pzTail=*/nullptr));
struct AdbcSqliteBinder* binder =
this->binder.schema.release ? &this->binder : nullptr;
Expand Down
7 changes: 7 additions & 0 deletions c/driver/sqlite/statement_reader.c
Original file line number Diff line number Diff line change
Expand Up @@ -426,8 +426,15 @@ AdbcStatusCode InternalAdbcSqliteBinderBindNext(struct AdbcSqliteBinder* binder,
}
case NANOARROW_TYPE_TIMESTAMP: {
struct ArrowSchemaView bind_schema_view;
#ifdef _MSC_VER
#pragma warning(push)
#pragma warning(disable : 4244) // RAISE_NA returns ArrowErrorCode, but this function returns AdbcStatusCode
#endif
RAISE_NA(ArrowSchemaViewInit(&bind_schema_view, binder->schema.children[col],
&arrow_error));
#ifdef _MSC_VER
#pragma warning(pop)
#endif
enum ArrowTimeUnit unit = bind_schema_view.time_unit;
int64_t value =
ArrowArrayViewGetIntUnsafe(binder->batch.children[col], binder->next_row);
Expand Down
2 changes: 1 addition & 1 deletion c/driver_manager/adbc_driver_manager.cc
Original file line number Diff line number Diff line change
Expand Up @@ -675,7 +675,7 @@ std::string AdbcDriverManagerDefaultEntrypoint(const std::string& driver) {
// if pos == npos this is the entire filename
std::string token = filename.substr(prev, pos - prev);
// capitalize first letter
token[0] = std::toupper(static_cast<unsigned char>(token[0]));
token[0] = static_cast<char>(std::toupper(static_cast<unsigned char>(token[0])));

entrypoint += token;

Expand Down
8 changes: 4 additions & 4 deletions c/validation/adbc_validation.cc
Original file line number Diff line number Diff line change
Expand Up @@ -51,11 +51,11 @@ AdbcStatusCode DoIngestSampleTable(struct AdbcConnection* connection,
Handle<struct ArrowSchema> schema;
Handle<struct ArrowArray> array;
struct ArrowError na_error;
CHECK_OK(MakeSchema(&schema.value, {{"int64s", NANOARROW_TYPE_INT64},
{"strings", NANOARROW_TYPE_STRING}}));
CHECK_OK((MakeBatch<int64_t, std::string>(&schema.value, &array.value, &na_error,
CHECK_OK(static_cast<AdbcStatusCode>(MakeSchema(&schema.value, {{"int64s", NANOARROW_TYPE_INT64},
{"strings", NANOARROW_TYPE_STRING}})));
CHECK_OK((static_cast<AdbcStatusCode>(MakeBatch<int64_t, std::string>(&schema.value, &array.value, &na_error,
{42, -42, std::nullopt},
{"foo", std::nullopt, ""})));
{"foo", std::nullopt, ""}))));

Handle<struct AdbcStatement> statement;
CHECK_OK(AdbcStatementNew(connection, &statement.value, error));
Expand Down
Loading
Loading