build: Remove bundled protoc and build from source (tokio-rs#610)

Author: LucioFranco

.github/workflows/continuous-integration-workflow.yaml

@@ -8,6 +8,8 @@ jobs:
     steps:
       - name: checkout
         uses: actions/checkout@v2
+        with:
+          submodules: recursive
       - name: install toolchain
         uses: actions-rs/toolchain@v1
         with:
@@ -24,7 +26,10 @@ jobs:
   clippy:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - name: checkout
+        uses: actions/checkout@v2
+        with:
+          submodules: recursive
       - name: install toolchain
         uses: actions-rs/toolchain@v1
         with:
@@ -54,6 +59,8 @@ jobs:
     steps:
       - name: checkout
         uses: actions/checkout@v2
+        with:
+          submodules: recursive
       - name: install toolchain
         uses: actions-rs/toolchain@v1
         with:
@@ -78,6 +85,8 @@ jobs:
     steps:
       - name: checkout
         uses: actions/checkout@v2
+        with:
+          submodules: recursive
       - name: install toolchain
         uses: actions-rs/toolchain@v1
         with:
@@ -109,3 +118,20 @@ jobs:
         with:
           command: check
           args: --manifest-path prost-build/Cargo.toml
+
+  vendored:
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2
+        with:
+          submodules: recursive
+      - name: install toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+            toolchain: stable
+            default: true
+            profile: minimal
+      - name: cargo check
+        run: cd test-vendored && cargo check
+

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "prost-build/third-party/protobuf"]
+	path = prost-build/third-party/protobuf
+	url = [email protected]:protocolbuffers/protobuf

Cargo.toml

@@ -33,6 +33,7 @@ exclude = [
   "fuzz",
   # Same counts for the afl fuzz targets
   "afl",
+  "test-vendored"
 ]
 
 [lib]

README.md

@@ -47,6 +47,13 @@ start-to-finish example.
 `proto3` syntax. `prost`'s goal is to make the generated code as simple as
 possible.
 
+### `protoc`
+
+It's recommended to install `protoc` locally in your path to improve build times.
+Prost uses `protoc` to parse protobuf files and will attempt to compile protobuf
+from source requiring a C++ toolchain. For more info checkout the [`prost-build`](prost-build)
+docs.
+
 ### Packages
 
 Prost can now generate code for `.proto` files that don't have a package spec. 

prost-build/Cargo.toml

@@ -12,6 +12,10 @@ readme = "README.md"
 description = "A Protocol Buffers implementation for the Rust Language."
 edition = "2018"
 
+[features]
+default = []
+vendored = []
+
 [dependencies]
 bytes = { version = "1", default-features = false }
 heck = "0.4"
@@ -27,6 +31,8 @@ regex = { version = "1.5.5", default-features = false, features = ["std", "unico
 
 [build-dependencies]
 which = { version = "4", default-features = false }
+cfg-if = "1"
+cmake = "0.1"
 
 [dev-dependencies]
 env_logger = { version = "0.8", default-features = false }

prost-build/README.md

@@ -7,6 +7,28 @@
 a Cargo build. See the crate [documentation](https://docs.rs/prost-build/) for examples
 of how to integrate `prost-build` into a Cargo project.
 
+## `protoc`
+
+`prost-build` uses `protoc` to parse the proto files. There are a few ways to make `protoc`
+available for `prost-build`. 
+
+The first option is to include `protoc` in your `PATH` this
+can be done by following the [`protoc` install instructions]. In addition, its possible to
+pass the `PROTOC=<my/path/to/protoc>` environment variable.
+
+[`protoc` install instructions]: https://github.com/protocolbuffers/protobuf#protocol-compiler-installation
+
+The second option is to provide the `vendored` feature flag to `prost-build`. This will
+force `prost-build` to compile `protoc` from the bundled source. This will require that
+you have the correct dependencies installed include a C++ toolchain, cmake, etc. For
+more info on what the required dependencies are check [here].
+
+[here]: https://github.com/protocolbuffers/protobuf/blob/master/src/README.md
+
+If you would like to always ignore vendoring `protoc` you can additionally pass
+`PROTOC_NO_VENDOR` and this will always check the `PATH`/`PROTOC` environment
+variables and never compile `protoc` from source.
+
 ## License
 
 `prost-build` is distributed under the terms of the Apache License (Version 2.0).

prost-build/build.rs

@@ -1,72 +1,29 @@
 //! Finds the appropriate `protoc` binary and Protobuf include directory for this host, and outputs
 //! build directives so that the main `prost-build` crate can use them.
 //!
-//! The following locations are checked for `protoc` in decreasing priority:
+//! This build script attempts to find `protoc` in a few ways:
 //!
-//!     1. The `PROTOC` environment variable.
-//!     2. The bundled `protoc`.
-//!     3. The `protoc` on the `PATH`.
-//!
-//! If no `protoc` binary is available in these locations, the build fails.
+//!     1. If `PROTOC_NO_VENDOR` is enabled, it will check the `PROTOC` environment variable
+//!         then check the `PATH` for a `protoc` or `protoc.exe`.
+//!     2. If the `vendored` feature flag is enabled or `protoc` can't be found via the environment
+//!         variable or in the `PATH` then `prost-build` will attempt to build `protoc` from the
+//!         bundled source code.
+//!     3. Otherwise, it will attempt to execute from the `PATH` and fail if it does not exist.
 //!
 //! The following locations are checked for the Protobuf include directory in decreasing priority:
 //!
 //!     1. The `PROTOC_INCLUDE` environment variable.
 //!     2. The bundled Protobuf include directory.
+//!
 
+use cfg_if::cfg_if;
 use std::env;
 use std::path::PathBuf;
+use which::which;
 
 /// Returns the path to the location of the bundled Protobuf artifacts.
 fn bundle_path() -> PathBuf {
-    env::current_dir()
-        .unwrap()
-        .join("third-party")
-        .join("protobuf")
-}
-
-/// Returns the path to the `protoc` pointed to by the `PROTOC` environment variable, if it is set.
-fn env_protoc() -> Option<PathBuf> {
-    let protoc = match env::var_os("PROTOC") {
-        Some(path) => PathBuf::from(path),
-        None => return None,
-    };
-
-    Some(protoc)
-}
-
-/// We can only use a bundled protoc if the interpreter necessary to load the binary is available.
-///
-/// The interpreter is specific to the binary and can be queried via e.g. `patchelf
-/// --print-interpreter`, or via readelf, or similar.
-fn is_interpreter(path: &'static str) -> bool {
-    // Here we'd check for it being executable and other things, but for now it being present is
-    // probably good enough.
-    std::fs::metadata(path).is_ok()
-}
-
-/// Returns the path to the bundled `protoc`, if it is available for the host platform.
-fn bundled_protoc() -> Option<PathBuf> {
-    let protoc_bin_name = match (env::consts::OS, env::consts::ARCH) {
-        ("linux", "x86") if is_interpreter("/lib/ld-linux.so.2") => "protoc-linux-x86_32",
-        ("linux", "x86_64") if is_interpreter("/lib64/ld-linux-x86-64.so.2") => {
-            "protoc-linux-x86_64"
-        }
-        ("linux", "aarch64") if is_interpreter("/lib/ld-linux-aarch64.so.1") => {
-            "protoc-linux-aarch_64"
-        }
-        ("macos", "x86_64") => "protoc-osx-x86_64",
-        ("macos", "aarch64") => "protoc-osx-aarch64",
-        ("windows", _) => "protoc-win32.exe",
-        _ => return None,
-    };
-
-    Some(bundle_path().join(protoc_bin_name))
-}
-
-/// Returns the path to the `protoc` included on the `PATH`, if it exists.
-fn path_protoc() -> Option<PathBuf> {
-    which::which("protoc").ok()
+    env::current_dir().unwrap().join("third-party")
 }
 
 /// Returns the path to the Protobuf include directory pointed to by the `PROTOC_INCLUDE`
@@ -98,14 +55,56 @@ fn bundled_protoc_include() -> PathBuf {
     bundle_path().join("include")
 }
 
+/// Check for `protoc` via the `PROTOC` env var or in the `PATH`.
+fn path_protoc() -> Option<PathBuf> {
+    env::var_os("PROTOC")
+        .map(PathBuf::from)
+        .or_else(|| which("protoc").ok())
+}
+
+/// Returns true if the vendored flag is enabled.
+fn vendored() -> bool {
+    cfg_if! {
+        if #[cfg(feature = "vendored")] {
+            true
+        } else {
+            false
+        }
+    }
+}
+
+/// Compile `protoc` via `cmake`.
+fn compile() -> Option<PathBuf> {
+    let protobuf_src = bundle_path().join("protobuf").join("cmake");
+
+    println!("cargo:rerun-if-changed={}", protobuf_src.display());
+
+    let dst = cmake::Config::new(protobuf_src).build();
+
+    Some(dst.join("bin").join("protoc"))
+}
+
+/// Try to find a `protoc` through a few methods.
+///
+/// Check module docs for more info.
+fn protoc() -> Option<PathBuf> {
+    if env::var_os("PROTOC_NO_VENDOR").is_some() {
+        path_protoc()
+    } else if vendored() {
+        compile()
+    } else {
+        path_protoc().or_else(compile)
+    }
+}
+
 fn main() {
-    let protoc = env_protoc()
-        .or_else(bundled_protoc)
-        .or_else(path_protoc)
-        .expect(
-            "Failed to find the protoc binary. The PROTOC environment variable is not set, \
-             there is no bundled protoc for this platform, and protoc is not in the PATH",
-        );
+    let protoc = protoc().expect(
+        "Failed to find or build the protoc binary. The PROTOC environment \
+    is not set, `protoc` is not in PATH or you are missing the requirements to compile protobuf \
+    from source. \n \
+    Check out the `prost-build` README for instructions on the requirements: \
+    https://github.com/tokio-rs/prost#generated-code",
+    );
 
     let protoc_include = env_protoc_include().unwrap_or_else(bundled_protoc_include);
 

prost-build/src/lib.rs

@@ -97,17 +97,28 @@
 //! PROTOC_INCLUDE=/usr/include
 //! ```
 //!
-//! If `PROTOC` is not found in the environment, then a pre-compiled `protoc` binary bundled in the
-//! prost-build crate is used. Pre-compiled `protoc` binaries exist for Linux (non-musl), macOS,
-//! and Windows systems. If no pre-compiled `protoc` is available for the host platform, then the
-//! `protoc` or `protoc.exe` binary on the `PATH` is used. If `protoc` is not available in any of
-//! these fallback locations, then the build fails.
+//! If no `PROTOC` environment variable is set then `prost-build` will search the
+//! current path for `protoc` or `protoc.exe`. If `protoc` is not found via these
+//! two methods then `prost-build` will attempt to compile `protoc` from the bundled
+//! source.
+//!
+//! If you would not like `prost-build` to not compile `protoc` from source ever then
+//! ensure you have set `PROTO_NO_VENDOR` environment variable as this will disable
+//! compiling from source even if the `vendored` feature flag is enabled.
+//!
+//! If you would like to always compile from source then setting the `vendored` feature
+//! flag will force `prost-build` to always build `protoc` from source.
 //!
 //! If `PROTOC_INCLUDE` is not found in the environment, then the Protobuf include directory
 //! bundled in the prost-build crate is be used.
 //!
-//! To force `prost-build` to use the `protoc` on the `PATH`, add `PROTOC=protoc` to the
-//! environment.
+//! ### Compiling `protoc` from source
+//!
+//! Compiling `protoc` from source requires a few external dependencies. Currently,
+//! `prost-build` uses `cmake` to build `protoc`. For more information check out the
+//! [protobuf build instructions](protobuf-build).
+//!
+//! [protobuf-build]: https://github.com/protocolbuffers/protobuf/blob/master/src/README.md
 
 mod ast;
 mod code_generator;

prost-build/third-party/protobuf/include/google/protobuf/any.proto renamed to prost-build/third-party/include/google/protobuf/any.proto

@@ -348,17 +348,17 @@ message FileOptions {
   optional string java_package = 1;
 
 
-  // If set, all the classes from the .proto file are wrapped in a single
-  // outer class with the given name.  This applies to both Proto1
-  // (equivalent to the old "--one_java_file" option) and Proto2 (where
-  // a .proto always translates to a single class, but you may want to
-  // explicitly choose the class name).
+  // Controls the name of the wrapper Java class generated for the .proto file.
+  // That class will always contain the .proto file's getDescriptor() method as
+  // well as any top-level extensions defined in the .proto file.
+  // If java_multiple_files is disabled, then all the other classes from the
+  // .proto file will be nested inside the single wrapper outer class.
   optional string java_outer_classname = 8;
 
-  // If set true, then the Java code generator will generate a separate .java
+  // If enabled, then the Java code generator will generate a separate .java
   // file for each top-level message, enum, and service defined in the .proto
-  // file.  Thus, these types will *not* be nested inside the outer class
-  // named by java_outer_classname.  However, the outer class will still be
+  // file.  Thus, these types will *not* be nested inside the wrapper class
+  // named by java_outer_classname.  However, the wrapper class will still be
   // generated to contain the file's getDescriptor() method as well as any
   // top-level extensions defined in the file.
   optional bool java_multiple_files = 10 [default = false];
@@ -496,6 +496,8 @@ message MessageOptions {
   // this is a formalization for deprecating messages.
   optional bool deprecated = 3 [default = false];
 
+  reserved 4, 5, 6;
+
   // Whether the message is an automatically generated map entry type for the
   // maps field.
   //

prost-build/third-party/protobuf/include/google/protobuf/api.proto renamed to prost-build/third-party/include/google/protobuf/api.proto

@@ -55,8 +55,8 @@ message Struct {
 
 // `Value` represents a dynamically typed value which can be either
 // null, a number, a string, a boolean, a recursive struct value, or a
-// list of values. A producer of value is expected to set one of that
-// variants, absence of any variant indicates an error.
+// list of values. A producer of value is expected to set one of these
+// variants. Absence of any variant indicates an error.
 //
 // The JSON representation for `Value` is JSON value.
 message Value {