Part 2 of eliminating most of the per-project boilerplate

Summary:
After this lands, adding new `fbcode_builder` projects will be much easier.

**Bistro configs before:**

```
bistro/build/ci$ wc *
     51     124    1863 build_steps.py
wc: fbcode_builder: Is a directory
      0       0       0 fbcode_builder
     30      97     855 make_docker_context.py
     17      85     593 README
     15      61     611 travis_docker_build.sh
    144     490    5045 total
```

**Bistro configs after:**

```
public_tld/build$ wc *
  41  113 1376 fbcode_builder_config.py
  10   45  397 README.md
  64  200 2220 total
```

Reviewed By: sdwilsh

Differential Revision: D4511822

fbshipit-source-id: 471d6f312f3688ef0114bc5f96fb28271cfef1f4

Author: snarkmaster

.gitignore

@@ -1,3 +1,2 @@
 *~
 *.pyc
-

.travis.yml

@@ -21,4 +21,4 @@ script:
   - sudo apt-get install python2.7
   # We don't want to write the script inline because of Travis kludginess --
   # it looks like it escapes " and \ in scripts when using `matrix:`.
-  - ./build/travis_docker_build.sh
+  - ./build/fbcode_builder/travis_docker_build.sh

README.md

@@ -41,16 +41,16 @@ The CLI tools and web UI will be shipping shortly.
 Bistro needs a 64-bit Linux, Folly, FBThrift, Proxygen, boost, and
 libsqlite3.  You need 2-3GB of RAM to build, as well as GCC 4.9 or above.
 
-`bistro/build/ci/README` documents the usage of Docker-based scripts that
-build Bistro on Ubuntu 14.04, 16.04, and Debian 8.6.  You should be able to
-follow very similar steps on most modern Linux distributions.
+`build/README.md` documents the usage of Docker-based scripts that build
+Bistro on Ubuntu 14.04, 16.04, and Debian 8.6.  You should be able to follow
+very similar steps on most modern Linux distributions.
 
-If you run into dependency problems, look at bistro/build/CMakeLists.txt for
+If you run into dependency problems, look at `bistro/cmake/setup.cmake` for
 a full list of Bistro's external dependencies (direct and indirect).  We
 gratefully accept patches that improve Bistro's builds, or add support for
 various flavors of Linux and Mac OS.
 
-The binaries will be in bistro/build/{Debug,Release}.  Available build
+The binaries will be in bistro/cmake/{Debug,Release}.  Available build
 targets are explained here:
    http://cmake.org/Wiki/CMake_Useful_Variables#Compilers_and_Tools
 You can start Bistro's unit tests by running `ctest` in those directories.
@@ -88,13 +88,13 @@ Open two terminals, one for the scheduler, and one for the worker.
 # In both terminals
 cd bistro/bistro
 # Start the scheduler in one terminal
-./build/Debug/server/bistro_scheduler \
+./cmake/Debug/server/bistro_scheduler \
   --server_port=6789 --http_server_port=6790 \
   --config_file=scripts/test_configs/simple --clean_statuses \
   --CAUTION_startup_wait_for_workers=1 --instance_node_name=scheduler
 # Start the worker in another
 mkdir /tmp/bistro_worker
-./build/Debug/worker/bistro_worker --server_port=27182 --scheduler_host=:: \
+./cmake/Debug/worker/bistro_worker --server_port=27182 --scheduler_host=:: \
   --scheduler_port=6789 --worker_command="$HOME/demo_bistro_task.sh" \
   --data_dir=/tmp/bistro_worker
 ```
@@ -113,7 +113,7 @@ put the SQLite database, via `--data_dir=/tmp/bistro_scheduler` and
 
 ```
 mkdir /tmp/bistro_scheduler
-./build/Debug/server/bistro_scheduler \
+./cmake/Debug/server/bistro_scheduler \
   --server_port=6789 --http_server_port=6790 \
   --config_file=scripts/test_configs/simple \
   --data_dir=/tmp/bistro_scheduler --status_table=task_statuses \
@@ -140,7 +140,7 @@ less scripts/test_configs/simple
 For debugging, we typically invoke the binaries like this:
 
 ```
-gdb build/Debug/worker/bistro_worker -ex "r ..." 2>&1 | tee WORKER.txt
+gdb cmake/Debug/worker/bistro_worker -ex "r ..." 2>&1 | tee WORKER.txt
 ```
 
 When configuring a real deployment, be sure to carefully review the `--help`

bistro/build/ci/build_steps.py

@@ -1,5 +1,5 @@
 /*
- *  Copyright (c) 2015, Facebook, Inc.
+ *  Copyright (c) 2015-present, Facebook, Inc.
  *  All rights reserved.
  *
  *  This source code is licensed under the BSD-style license found in the

bistro/build/ci/fbcode_builder

@@ -9,7 +9,7 @@
 #
 
 #
-# Runs `cmake` for Bistro, directing output to build/Release/ or build/Debug/.
+# Runs `cmake` for Bistro, directing output to cmake/Release/ or cmake/Debug/.
 #
 # If Bistro's dependencies are installed in a non-default prefix, it will
 # need some help finding them:
@@ -87,10 +87,10 @@ fetch_gtest
 
 # Hacky way to build Thrift until we have a proper CMake module to do it.
 cd "$build_dir/.."
-update_thrift if if/*.thrift build/fbinclude/common/fb303/if/fb303.thrift
-./build/targets_to_cmake_lists.py .
+update_thrift if if/*.thrift cmake/fbinclude/common/fb303/if/fb303.thrift
+./cmake/targets_to_cmake_lists.py .
 
-# Put our artifacts in build/{Debug,Release}
+# Put our artifacts in cmake/{Debug,Release}
 make_dir="$build_dir/$build_type"
 mkdir -p "$make_dir"
 cd "$make_dir"

bistro/build/ci/make_docker_context.py

@@ -7,13 +7,13 @@
 # of patent rights can be found in the PATENTS file in the same directory.
 #
 
-# Do NOT use this -- run build/run-cmake.sh instead & read its docblock.
+# Do NOT use this -- run cmake/run-cmake.sh instead & read its docblock.
 
 include_directories(
   # Our includes start with "bistro/bistro/"
   "${PROJECT_SOURCE_DIR}/../.."
   # A hack to include a stub for some FB-specific includes under "common/".
-  "${PROJECT_SOURCE_DIR}/build/fbinclude"
+  "${PROJECT_SOURCE_DIR}/cmake/fbinclude"
   "${CMAKE_INSTALL_PREFIX}/include"
 )
 
@@ -59,7 +59,7 @@ macro(bistro_link_libraries name)
   )
 endmacro(bistro_link_libraries)
 
-add_subdirectory(build/deps/gtest-1.7.0)
+add_subdirectory(cmake/deps/gtest-1.7.0)
 enable_testing()
 include_directories("${gtest_SOURCE_DIR}/include" "${gtest_SOURCE_DIR}")
 

bistro/build/.gitignore bistro/cmake/.gitignore

@@ -111,7 +111,7 @@ def cpp_library(name, **kwargs):
         _validate_keys(kwargs, 'cpp_library', {
             'srcs',  # Handled above
             'deps',  # Handled above
-            'external_deps',  # Handled in build/CMakeLists.txt
+            'external_deps',  # Handled in cmake/setup.cmake
             'headers',  # CMake handles headers automatically
         })
 
@@ -195,5 +195,5 @@ def make_cmake_dir_recursive(root_path):
 
 make_cmake_dir_recursive(sys.argv[1]).write_cmake_lists(
     setup_lines='cmake_minimum_required(VERSION 2.8)\n'
-        'include("build/setup.cmake")\n',
+        'include("cmake/setup.cmake")\n',
 )

bistro/build/fbinclude/common/fb303/cpp/FacebookBase2.h bistro/cmake/fbinclude/common/fb303/cpp/FacebookBase2.h

@@ -30,9 +30,9 @@ if [[ -x "$BISTRO_BINARY" ]] ; then
 else
   PHABRICATOR_DOMAIN="[YOUR PHABRICATOR DOMAIN]"
   # Try the release binary, then default to the debug binary
-  BISTRO_BINARY="$D/../build/Release/bistro/bistro/server/bistro_scheduler"
+  BISTRO_BINARY="$D/../cmake/Release/bistro/bistro/server/bistro_scheduler"
   if [[ ! -x "$BISTRO_BINARY" ]] ; then
-    BISTRO_BINARY="$D/../build/Debug/bistro/bistro/server/bistro_scheduler"
+    BISTRO_BINARY="$D/../cmake/Debug/bistro/bistro/server/bistro_scheduler"
   fi
 fi
 

bistro/build/fbinclude/common/fb303/if/fb303.thrift bistro/cmake/fbinclude/common/fb303/if/fb303.thrift

@@ -18,9 +18,11 @@
 import os
 import tempfile
 
-from .fbcode_builder import FBCodeBuilder
-from .shell_quoting import raw_shell, shell_comment, shell_join, ShellQuoted
-from .utils import recursively_flatten_list, run_command
+from fbcode_builder import FBCodeBuilder
+from shell_quoting import (
+    raw_shell, shell_comment, shell_join, ShellQuoted
+)
+from utils import recursively_flatten_list, run_command
 
 
 class DockerFBCodeBuilder(FBCodeBuilder):

bistro/build/run-cmake.sh bistro/cmake/run-cmake.sh

@@ -37,17 +37,14 @@
    shell script that runs outside of a container. Or maybe even
    synchronously executes the shell commands, `make`-style.
 
- - Support for dependencies. This was not built since FB projects'
-   dependency chains are simple, so a linear sequence of steps was enough.
-
  - A "Makefile" generator. That might make iterating on builds even quicker
    than what you can currently get with Docker build caching.
 
 '''
 
 import os
 
-from .shell_quoting import path_join, shell_join, ShellQuoted
+from shell_quoting import path_join, shell_join, ShellQuoted
 
 
 class FBCodeBuilder(object):
@@ -99,7 +96,7 @@ def render(self, steps):
             raise RuntimeError(
                 'Unused options: {0} -- please check if you made a typo '
                 'in any of them. Those that are truly not useful should '
-                'be removed so that this typo detection can be useful.'
+                'be not be set so that this typo detection can be useful.'
                 .format(unused_options)
             )
         return res
@@ -260,14 +257,6 @@ def make_and_install(self, make_vars=None):
             )),
         ]
 
-    def zstd_make_and_install(self):
-        'Zstd is a special snowflake, so make it easy to built it correctly'
-        return self.step('Build and install zstd', [
-            self.make_and_install(make_vars={
-                'PREFIX': self.option('prefix'),
-            })
-        ])
-
     def autoconf_install(self, name):
         return self.step('Build and install {0}'.format(name), [
             self.run(ShellQuoted('autoreconf -ivf')),

bistro/build/setup.cmake bistro/cmake/setup.cmake

@@ -0,0 +1,14 @@
+#!/usr/bin/env python
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+'Demo config, so that `make_docker_context.py --help` works in this directory.'
+
+config = {
+    'fbcode_builder_spec': lambda _builder: {
+        'depends_on': [],
+        'steps': [],
+    },
+    'github_project': 'demo/project',
+}