Migrate third_party/ to git submodules.

This replaces third_party/absl/ and third_party/protobuf-lite/ with
submodules checked out at specific versions for abseil-cpp, googletest,
benchmark, and protobuf libraries.

This change allows removing local code copies of absl and googltest
functionality.

Author: ZacharyGarrett

.github/workflows/cifuzz.yml

@@ -23,7 +23,7 @@ jobs:
         dry-run: false
         language: c++
     - name: Upload Crash
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       if: failure() && steps.build.outcome == 'success'
       with:
         name: artifacts

.github/workflows/cmake.yml

@@ -27,6 +27,8 @@ jobs:
 
     steps:
     - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      with:
+        submodules: 'recursive'
     - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1
       with:
         python-version: '3.x'
@@ -40,11 +42,11 @@ jobs:
         } else {
           $msbuildPlatform = "Win32"
         }
-        cmake -A $msbuildPlatform -B ${{github.workspace}}/build -DSPM_BUILD_TEST=ON -DSPM_ENABLE_SHARED=OFF -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/build/root
+        cmake -A $msbuildPlatform -B ${{github.workspace}}/build -DSPM_BUILD_TEST=ON -DBUILD_SHARED_LIBS=OFF -DSPM_ENABLE_SHARED=OFF -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/build/root
 
     - name: Config for Unix
       if: runner.os != 'Windows'
-      run: cmake -B ${{github.workspace}}/build -DSPM_BUILD_TEST=ON -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/build/root
+      run: cmake -B ${{github.workspace}}/build -DSPM_BUILD_TEST=ON -DBUILD_SHARED_LIBS=OFF -DSPM_ENABLE_SHARED=OFF -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/build/root
       env:
         CMAKE_OSX_ARCHITECTURES: arm64;x86_64
 
@@ -54,6 +56,8 @@ jobs:
     - name: Test
       working-directory: ${{github.workspace}}/build
       run: ctest -C Release --output-on-failure
+      env:
+        TEST_SRCDIR: ${{github.workspace}}/data
 
     - name: Package
       working-directory: ${{github.workspace}}/build
@@ -65,10 +69,14 @@ jobs:
         python -m pip install --require-hashes --no-dependencies -r ../.github/workflows/requirements/base.txt
         python setup.py build
         python setup.py bdist_wheel
+        pip install dist/*.whl
+        pushd test
         python -m pytest
+      env:
+        GITHUB_REF_NAME: "${{ github.ref_name }}"
 
     - name: Upload artifcacts
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: artifcacts
         path: ./build/*.7z

.github/workflows/cross_build.yml

@@ -21,6 +21,8 @@ jobs:
 
     steps:
     - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      with:
+        submodules: 'recursive'
 
     - name: Install cross tools
       run: |
@@ -34,7 +36,7 @@ jobs:
       run: |
         mkdir -p ${{github.workspace}}/build
         cd ${{github.workspace}}/build
-        env CXX=/usr/bin/${{matrix.arch}}-linux-gnu-g++-10 CC=/usr/bin/${{matrix.arch}}-linux-gnu-gcc-10 cmake .. -DSPM_BUILD_TEST=ON -DSPM_ENABLE_SHARED=OFF -DCMAKE_FIND_ROOT_PATH=/usr/${{matrix.arch}}-linux-gnu -DSPM_CROSS_SYSTEM_PROCESSOR=${{matrix.arch}}
+        env CXX=/usr/bin/${{matrix.arch}}-linux-gnu-g++-10 CC=/usr/bin/${{matrix.arch}}-linux-gnu-gcc-10 cmake .. -DSPM_BUILD_TEST=ON -DBUILD_SHARED_LIBS=OFF -DSPM_ENABLE_SHARED=OFF -DCMAKE_FIND_ROOT_PATH=/usr/${{matrix.arch}}-linux-gnu -DSPM_CROSS_SYSTEM_PROCESSOR=${{matrix.arch}}
         make -j$(nproc)
 
     - name: Test on QEMU

.github/workflows/wheel.yml

@@ -28,9 +28,11 @@ jobs:
 
     steps:
       - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        with:
+          submodules: 'recursive'
       - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1
         with:
-          python-version: "3.x"
+          python-version: ">=3.8"
 
       - name: Set up QEMU
         if: runner.os == 'Linux'
@@ -41,17 +43,17 @@ jobs:
       - name: Build for Windows
         if: runner.os == 'Windows'
         run: |
-          cmake -A Win32 -B ${{github.workspace}}/build_win32 -DSPM_ENABLE_SHARED=OFF -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/build/root_win32
+          cmake -A Win32 -B ${{github.workspace}}/build_win32 -DBUILD_SHARED_LIBS=OFF -DSPM_ENABLE_SHARED=OFF -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/build/root_win32
           cmake --build ${{github.workspace}}/build_win32 --config Release --target install --parallel 8
-          cmake -A x64   -B ${{github.workspace}}/build_amd64 -DSPM_ENABLE_SHARED=OFF -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/build/root_amd64
+          cmake -A x64   -B ${{github.workspace}}/build_amd64 -DBUILD_SHARED_LIBS=OFF -DSPM_ENABLE_SHARED=OFF -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/build/root_amd64
           cmake --build ${{github.workspace}}/build_amd64 --config Release --target install --parallel 8
-          cmake -A arm64 -B ${{github.workspace}}/build_arm64 -DSPM_ENABLE_SHARED=OFF -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/build/root_arm64
+          cmake -A arm64 -B ${{github.workspace}}/build_arm64 -DBUILD_SHARED_LIBS=OFF -DSPM_ENABLE_SHARED=OFF -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/build/root_arm64
           cmake --build ${{github.workspace}}/build_arm64 --config Release --target install --parallel 8
 
       - name: Build for Mac
         if: runner.os == 'macOS'
         run: |
-          cmake -B ${{github.workspace}}/build -DSPM_ENABLE_SHARED=OFF -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/build/root
+          cmake -B ${{github.workspace}}/build -DSPM_ENABLE_SHARED=OFF -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/build/root
           cmake --build ${{github.workspace}}/build --config Release --target install --parallel 8
         env:
           CMAKE_OSX_ARCHITECTURES: arm64;x86_64
@@ -66,10 +68,11 @@ jobs:
         working-directory: ${{github.workspace}}/python
         run: python -m cibuildwheel --output-dir wheelhouse
         env:
+          CIBW_ENVIRONMENT: "GITHUB_REF_NAME=${{ github.ref_name }}"
           CIBW_ARCHS_LINUX: auto aarch64
           CIBW_ARCHS_MACOS: x86_64 universal2 arm64
           CIBW_ARCHS_WINDOWS: auto ARM64
-          CIBW_SKIP: "pp* *-musllinux_*"
+          CIBW_SKIP: "pp* *-musllinux_* cp3{6,7}-*"
           CIBW_BUILD_VERBOSITY: 1
 
       - name: Build sdist archive
@@ -91,7 +94,7 @@ jobs:
         run: cp -f dist/*.tar.gz wheelhouse/
 
       - name: Upload artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: artifacts
           path: |

.gitignore

@@ -63,6 +63,7 @@ dist/
 *.swp
 *.swo
 *.pyc
+*.whl
 
 m.model
 m.vocab
@@ -72,6 +73,8 @@ libsentencepiece.so*
 libsentencepiece_train.so*
 python/bundled
 _sentencepiece.*.so
-third_party/abseil-cpp
+
+# Generated proto files.
+src/*.pb.*
 
 python/sentencepiece

.gitmodules

@@ -0,0 +1,12 @@
+[submodule "third_party/googletest"]
+	path = third_party/googletest
+	url = https://github.com/google/googletest.git
+[submodule "third_party/protobuf"]
+	path = third_party/protobuf
+	url = https://github.com/protocolbuffers/protobuf.git
+[submodule "third_party/abseil-cpp"]
+	path = third_party/abseil-cpp
+	url = https://github.com/abseil/abseil-cpp.git
+[submodule "third_party/benchmark"]
+	path = third_party/benchmark
+	url = https://github.com/google/benchmark.git

CMakeLists.txt

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.!
 
-cmake_minimum_required(VERSION 3.5 FATAL_ERROR)
+cmake_minimum_required(VERSION 3.16 FATAL_ERROR)
 file(STRINGS "VERSION.txt" SPM_VERSION)
 message(STATUS "VERSION: ${SPM_VERSION}")
 
@@ -30,13 +30,13 @@ option(SPM_ENABLE_TENSORFLOW_SHARED "Makes a tensorflow compatible shared file."
 option(SPM_ENABLE_TCMALLOC "Enable TCMalloc if available." ON)
 option(SPM_TCMALLOC_STATIC "Link static library of TCMALLOC." OFF)
 option(SPM_NO_THREADLOCAL "Disable thread_local operator" OFF)
-option(SPM_ENABLE_MSVC_MT_BUILD, "Use /MT flag in MSVC build" OFF)
+option(SPM_ENABLE_MSVC_MT_BUILD, "Use /MT (static runtime) flag in MSVC build. Otherwise uses /MD (DLL runtime)" ON)
 option(SPM_CROSS_SYSTEM_PROCESSOR, "Override system processor" "")
 
 set(SPM_PROTOBUF_PROVIDER "internal" CACHE STRING "Provider of protobuf library")
 set_property(CACHE SPM_PROTOBUF_PROVIDER PROPERTY STRINGS "internal" "package")
 set(SPM_ABSL_PROVIDER "internal" CACHE STRING "Provider of absl library")
-set_property(CACHE SPM_ABSL_PROVIDER PROPERTY STRINGS "internal" "module" "package")
+set_property(CACHE SPM_ABSL_PROVIDER PROPERTY STRINGS "internal" "package")
 
 if (SPM_CROSS_SYSTEM_PROCESSOR)
  set(CMAKE_SYSTEM_PROCESSOR ${SPM_CROSS_SYSTEM_PROCESSOR})
@@ -45,10 +45,12 @@ endif()
 # Disable shared build on windows
 if(WIN32)
   set(SPM_ENABLE_SHARED OFF)
+  set(SPM_ENABLE_MSVC_MT_BUILD ON)
 endif()
 
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_POSITION_INDEPENDENT_CODE TRUE)
 
 if((CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND
      CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 10.0) OR
@@ -96,10 +98,9 @@ endif()
 if (MSVC)
   add_definitions("/wd4267 /wd4244 /wd4305 /Zc:strictStrings /utf-8")
   if (SPM_ENABLE_MSVC_MT_BUILD)
-    string(REPLACE "/MD" "/MT" CMAKE_CXX_FLAGS_DEBUG          ${CMAKE_CXX_FLAGS_DEBUG})
-    string(REPLACE "/MD" "/MT" CMAKE_CXX_FLAGS_MINSIZEREL     ${CMAKE_CXX_FLAGS_MINSIZEREL})
-    string(REPLACE "/MD" "/MT" CMAKE_CXX_FLAGS_RELEASE        ${CMAKE_CXX_FLAGS_RELEASE})
-    string(REPLACE "/MD" "/MT" CMAKE_CXX_FLAGS_RELWITHDEBINFO ${CMAKE_CXX_FLAGS_RELWITHDEBINFO})
+    set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>")
+  else()
+    set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>DLL")
   endif()
 endif()
 
@@ -144,7 +145,7 @@ configure_file("${PROJECT_SOURCE_DIR}/sentencepiece.pc.in" "sentencepiece.pc" @O
 
 if (NOT MSVC)
   # suppress warning for C++11 features.
-#  add_definitions("-Wno-deprecated-declarations -Wno-deprecated-enum-enum-conversion")
+  # add_definitions("-Wno-deprecated-declarations -Wno-deprecated-enum-enum-conversion")
   install(FILES "${CMAKE_CURRENT_BINARY_DIR}/sentencepiece.pc" DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig)
 endif()
 
@@ -154,21 +155,24 @@ if (SPM_BUILD_TEST)
   enable_testing()
 endif()
 
+set(ABSL_PROPAGATE_CXX_STD ON)
 if (SPM_ABSL_PROVIDER STREQUAL "internal")
-  include_directories(${CMAKE_CURRENT_SOURCE_DIR}/third_party/absl)
-elseif (SPM_ABSL_PROVIDER STREQUAL "module")
   include(FetchContent)
-  FetchContent_Populate(abseil-cpp
-        GIT_REPOSITORY  https://github.com/abseil/abseil-cpp.git
-        SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/abseil-cpp
-        GIT_PROGRESS TRUE)
-  add_subdirectory(third_party/abseil-cpp)
-  if (NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/third_party/absl.org)
-    file(RENAME ${CMAKE_CURRENT_SOURCE_DIR}/third_party/absl ${CMAKE_CURRENT_SOURCE_DIR}/third_party/absl.org)
-    execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink
-      ${CMAKE_CURRENT_SOURCE_DIR}/third_party/abseil-cpp/absl
-      ${CMAKE_CURRENT_SOURCE_DIR}/third_party/absl)
-  endif()
+  FetchContent_Declare(
+	googletest
+        SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/googletest)
+  set(ABSL_ENABLE_INSTALL ON)
+  set(ASBL_BUILD_TESTING OFF)
+  FetchContent_Declare(
+	absl
+        SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/abseil-cpp)
+  FetchContent_MakeAvailable(googletest absl)
+  FetchContent_GetProperties(googletest SOURCE_DIR googletest_SOURCE_DIR)
+  include_directories("${googletest_SOURCE_DIR}/googletest/include/")
+  include_directories("${googletest_SOURCE_DIR}/googlemock/include")
+
+  FetchContent_GetProperties(absl SOURCE_DIR absl_SOURCE_DIR)
+  include_directories(${absl_SOURCE_DIR})
 elseif (SPM_ABSL_PROVIDER STREQUAL "package")
   find_package(absl REQUIRED)
   get_target_property(ABSL_INCLUDE_DIRS absl::base INTERFACE_INCLUDE_DIRECTORIES)
@@ -180,6 +184,42 @@ elseif (SPM_ABSL_PROVIDER STREQUAL "package")
   include_directories(${ABSL_INCLUDE_DIRS})
 endif()
 
+if ((${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm") OR
+    (${CMAKE_SYSTEM_PROCESSOR} MATCHES "mips") OR
+    (${CMAKE_SYSTEM_PROCESSOR} MATCHES "m68k") OR
+    (${CMAKE_SYSTEM_PROCESSOR} MATCHES "ppc") OR
+    (${CMAKE_SYSTEM_PROCESSOR} MATCHES "powerpc") OR
+    (${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch") OR
+    (${CMAKE_SYSTEM_PROCESSOR} MATCHES "sh4"))
+  set(protobuf_LINK_LIBATOMIC ON)
+endif()
+
+# Note: Protobuf must occuring after abseil-cpp due to dependency.
+if (SPM_PROTOBUF_PROVIDER STREQUAL "internal")
+  set(protobuf_INSTALL ON)
+  set(utf8_range_ENABLE_INSTALL ON)
+  set(protobuf_BUILD_TESTS OFF)
+  # https://github.com/protocolbuffers/protobuf/issues/14549
+  if (${CMAKE_SYSTEM_PROCCESSOR} MATCHES "riscv|powerpc")
+          set(protobuf_LINK_LIBATOMIC true)
+  endif()
+  FetchContent_Declare(
+        Protobuf
+        SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/protobuf/
+  )
+  if (SPM_ENABLE_SHARED)
+        set(protobuf_BUILD_SHARED_LIBS ON)
+        set(Protobuf_USE_STATIC_LIBS OFF)
+  else ()
+        set(protobuf_BUILD_SHARED_LIBS OFF)
+        set(Protobuf_USE_STATIC_LIBS ON)
+  endif()
+  FetchContent_MakeAvailable(Protobuf)
+  FetchContent_GetProperties(Protobuf SOURCE_DIR Protobuf_SOURCE_DIR)
+  include(${Protobuf_SOURCE_DIR}/cmake/protobuf-generate.cmake)
+  include_directories("${Protobuf_SOURCE_DIR}/src")
+endif()
+
 add_subdirectory(src)
 add_subdirectory(third_party)
 

python/build_bundled.sh

@@ -1,6 +1,8 @@
 #!/bin/sh
 
-VERSION="$1"
+set -x
+
+REF_NAME="${1}"
 
 mkdir -p build
 
@@ -10,13 +12,25 @@ INSTALL_DIR=./build/root
 if [ -f ./sentencepiece/src/CMakeLists.txt ]; then
   SRC_DIR=./sentencepiece
 elif [ -f ../src/CMakeLists.txt ]; then
-  SRC_DIR=..  
+  SRC_DIR=..
 else
   # Try taged version. Othewise, use head.
-  git clone https://github.com/google/sentencepiece.git -b v"${VERSION}" --depth 1 || \
+  echo -e "\033[0;35mSource not found, cloning from github...\033[0m"
   git clone https://github.com/google/sentencepiece.git --depth 1
+  pushd sentencepiece
+  git checkout ${REF_NAME}
+  if [[ $? > 0 ]]; then
+    echo -e "\033[0;35mRef named ${REF_NAME} not found, trying it as a pull request...\033[0m"
+    PR_NUM=${REF_NAME%/merge}
+    git pull origin pull/${PR_NUM}/head:pr_${PR_NUM} && git checkout -q pr_${PR_NUM}
+  fi
+  git submodule init && git submodule update
+  echo -e "\033[0;35mWorking from ref:\033[0m"
+  git --no-pager log -n 1 --decorate=short --pretty=oneline
+  popd
   SRC_DIR=./sentencepiece
 fi
+echo -e "\033[0;32mSource at ${SRC_DIR}\033[0m"
 
-cmake ${SRC_DIR} -B ${BUILD_DIR} -DSPM_ENABLE_SHARED=OFF -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR}
+cmake ${SRC_DIR} -B ${BUILD_DIR} -DBUILD_SHARED_LIBS=OFF -DSPM_ENABLE_SHARED=OFF -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR}
 cmake --build ${BUILD_DIR} --config Release --target install --parallel $(nproc)