tooling: Make cloe a super-build of all packages

New:
- Add `cloe-meta` package to replace previous role of `cloe`.

- Add `cloe-plugins-core` package recipe (but do not build by default).

- Add editable builds to GitHub workflow `build-cloe` matrix.

Changed:
- Package `cloe` provides all Cloe packages compiled in one go.
  This is a boon to development, as we make `cloe` editable and
  only have to work with a single package.
  It also massively speeds up compilation:
  - Conan and CMake configuration is only performed once.
  - All cores can now be utilized much more effectively during the build
    process.

- Do not aggressively lint everything
  The developer can do this themselves by setting a cmake define:
  https://cmake.org/cmake/help/latest/variable/CMAKE_LANG_CLANG_TIDY.html

- Renamed top-level Make targets:
  - `status` is now `status-all`
  - `export` is now `export-all`
  - `deploy` is now `deploy-all`
  - `clean` is now `clean-all`
  - `purge` is now `purge-all`

- The following top-level Make targets just refer to `cloe` super-build package:
  - `package`
  - `smoketest`
  - `smoketest-deps`
  - `status`
  - `export`

Fixed:
- Plugin Conan configurations do not export correct library path.

Author: cassava

.github/workflows/build-cloe.yaml

@@ -35,7 +35,16 @@ jobs:
           - "cloe-normal"
         package_target:
           # 1. Build each test configuration in Conan cache and run all tests
-          - "export export-vendor smoketest-deps smoketest"
+          - "export-vendor export-all smoketest-deps smoketest"
+
+          # 2. Build cloe super-package in editable mode and run tests
+          - "export-vendor editable all smoketest TEST_CONANFILES=tests/conanfile_superbuild.py"
+
+          # 3. TODO: Build individual packages in editable mode and run tests
+          #    This cannot be currently enabled because of a Conan deficiency in v1.
+          #    Once all build tooling is based on Conan v2, we can re-enable this use-case.
+          #    Until then, use the previous target for this use-case.
+          # - "export-vendor editable-select build-all smoketest TEST_CONANFILES=tests/conanfile_deployment.py"
     env:
       CONAN_NON_INTERACTIVE: "yes"
       DEBIAN_FRONTEND: noninteractive

.gitignore

@@ -30,6 +30,7 @@ CMakeFiles/
 compile_commands.json
 
 # Files generated by Clang:
+.cache/
 .clangd/
 .cache/
 

CMakeLists.txt

@@ -0,0 +1,40 @@
+# This CMakeLists.txt configures a super-build containing everything
+# from this repo.
+#
+# It is currently experimental.
+#
+
+cmake_minimum_required(VERSION 3.15...3.27 FATAL_ERROR)
+
+project(cloe LANGUAGES CXX)
+
+option(CLOE_WITH_ESMINI "Build simulator_esmini plugin?" ON)
+option(CLOE_WITH_VTD "Build simulator_vtd plugin?" OFF)
+
+set(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/runtime/cmake")
+
+# Since a super-build does not export packages individually via Conan,
+# we cannot depend on Conan-generated CMake config files, instead we
+# use the CMake targets directly as if they were already found.
+set(CLOE_FIND_PACKAGES OFF CACHE BOOL "Call find_package() for cloe packages" FORCE)
+
+# Ensure output goes to one place so cloe-launch can find the plugins
+set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/lib)
+set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/bin)
+
+# Ensure we can test from this level.
+set(CMAKE_CTEST_ARGUMENTS "--output-on-failure")
+include(CTest)
+
+# Order matters:
+add_subdirectory(fable)
+add_subdirectory(runtime)
+add_subdirectory(models)
+add_subdirectory(osi)
+add_subdirectory(oak)
+add_subdirectory(engine)
+add_subdirectory(plugins)
+
+if(CLOE_WITH_VTD)
+    add_subdirectory(optional/vtd)
+endif()

Makefile

@@ -3,6 +3,11 @@
 # This file contains Makefile targets for the cloe project.
 #
 
+# Make configuration:
+SHELL := /bin/bash
+GNUMAKEFLAGS := --no-print-directory
+SUBMAKEFLAGS :=
+
 CLOE_ROOT   := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
 CLOE_LAUNCH := PYTHONPATH="${CLOE_ROOT}/cli" python3 -m cloe_launch
 
@@ -20,78 +25,98 @@ AG := $(or \
 
 # Build configuration:
 BUILD_DIR       := build
-LOCKFILE_SOURCE := conanfile.py
-BUILD_LOCKFILE  := ${BUILD_DIR}/conan.lock
-LOCKFILE_OPTION := --lockfile="${CLOE_ROOT}/${BUILD_LOCKFILE}"
 INSTALL_DIR     := /usr/local
+DEPLOY_DIR      := deploy
 CONAN_OPTIONS   :=
 
+# Lockfile for cloe-deployment:
+DEPLOY_LOCKFILE_SOURCE := tests/conanfile_deployment.py
+DEPLOY_BUILD_LOCKFILE  := ${DEPLOY_DIR}/conan.lock
+DEPLOY_LOCKFILE_OPTION := --lockfile="${CLOE_ROOT}/${DEPLOY_BUILD_LOCKFILE}"
+
 .DEFAULT_GOAL := help
 .PHONY: help
 .SILENT: help
 help::
 	$(call print_help_usage)
 	echo
+	$(call print_help_section, "Default target")
+	$(call print_help_target, help, "show this help on available targets")
+	echo
 
 # Setup targets ---------------------------------------------------------------
 include Makefile.setup
 
+${DEPLOY_BUILD_LOCKFILE}:
+	mkdir -p "${DEPLOY_DIR}"
+	conan lock create --lockfile-out "${DEPLOY_BUILD_LOCKFILE}" --build -- "${DEPLOY_LOCKFILE_SOURCE}"
+
+.PHONY: lockfile
+lockfile: ${DEPLOY_BUILD_LOCKFILE}
+
 # Workspace targets -----------------------------------------------------------
 help::
 	$(call print_help_section, "Available workspace targets")
-	$(call print_help_target, status, "show status of each of the Conan packages")
-	$(call print_help_target, smoketest-deps, "build system test pre-requisites")
-	$(call print_help_target, smoketest, "run system tests")
-	$(call print_help_target, docs, "generate documentation")
-	$(call print_help_target, deploy, "deploy Cloe to INSTALL_DIR [=${INSTALL_DIR}]")
-	$(call print_help_target, deploy-cli, "install ${_yel}cloe-launch${_rst} with ${_dim}${PIPX}${_rst}")
-	$(call print_help_target, export-cli, "export ${_yel}cloe-launch-profile${_rst} Conan recipe")
+	$(call print_help_target, docs, "generate Doxygen and Sphinx documentation")
 	echo
 
-${BUILD_LOCKFILE}:
-	${MAKE} -f Makefile.package SOURCE_CONANFILE=/dev/null LOCKFILE_SOURCE=${LOCKFILE_SOURCE} ${BUILD_LOCKFILE}
-
-.PHONY: lockfile
-lockfile: ${BUILD_LOCKFILE}
+.PHONY: docs
+docs:
+	$(call print_header, "Generating Doxygen documentation...")
+	${MAKE} -C docs doxygen
+	$(call print_header, "Generating Sphinx documentation...")
+	${MAKE} -C docs html
 
-.PHONY: status
-status: ${BUILD_LOCKFILE}
-	@for pkg in ${ALL_PKGS}; do \
-		[ -d $${pkg} ] || continue; \
-		${MAKE} LOCKFILE_SOURCE="" LOCKFILE_OPTION=${LOCKFILE_OPTION} -C $${pkg} status || true; \
-	done
+help::
+	$(call print_help_target, export-cli, "export ${_yel}cloe-launch-profile${_rst} Conan recipe")
+	$(call print_help_target, deploy-cli, "install ${_yel}cloe-launch${_rst} with ${_dim}${PIPX}${_rst}")
+	echo
 
-.PHONY: deploy
-deploy:
-	$(call print_header, "Deploying binaries to ${INSTALL_DIR}...")
-	conan install ${CONAN_OPTIONS} --install-folder ${BUILD_DIR}/deploy -g deploy .
-	mkdir -p ${INSTALL_DIR}
-	cp -r ${BUILD_DIR}/deploy/cloe-*/* ${INSTALL_DIR}/
+.PHONY: export-cli
+export-cli:
+	${MAKE} -C cli export
 
 .PHONY: deploy-cli
 deploy-cli:
 	$(call print_header, "Deploying cloe-launch binary with pip...")
 	${MAKE} -C cli install
 
-.PHONY: export-cli
-export-cli:
-	${MAKE} -C cli export
+help::
+	$(call print_help_target, lockfile, "create a lockfile for cloe deployment packages")
+	$(call print_help_target, package-all, "package all cloe deployment packages")
+	$(call print_help_target, status-all, "show status of each of the Conan packages")
+	$(call print_help_target, export-all, "export all package sources to Conan cache")
+	$(call print_help_target, build-all, "build individual packages locally in-source")
+	$(call print_help_target, deploy-all, "deploy Cloe to INSTALL_DIR [=${INSTALL_DIR}]")
+	$(call print_help_target, clean-all, "clean entire repository of temporary files")
+	$(call print_help_target, purge-all, "remove all cloe packages (in any version) from Conan cache")
+	echo
 
-export: export-cli
-package: export-cli
+.PHONY: build-all
+build-all: lockfile
+	${MAKE} all-select CONAN_OPTIONS="${CONAN_OPTIONS} ${DEPLOY_LOCKFILE_OPTION}"
 
-.PHONY: docs
-docs:
-	$(call print_header, "Generating Doxygen documentation...")
-	${MAKE} -C docs doxygen
-	$(call print_header, "Generating Sphinx documentation...")
-	${MAKE} -C docs html
+.PHONY: status-all
+status-all: ${DEPLOY_BUILD_LOCKFILE}
+	@for pkg in ${ALL_PKGS}; do \
+		${MAKE} LOCKFILE_SOURCE="" LOCKFILE_OPTION=${DEPLOY_LOCKFILE_OPTION} -C $${pkg} status || true; \
+	done
 
-.PHONY: smoketest-deps
-smoketest-deps: export-cli smoketest-deps-select
+.PHONY: export-all
+export-all:
+	$(call print_header, "Exporting all cloe Conan packages...")
+	${MAKE} export-select export-cli export
+
+.PHONY: deploy-all
+deploy-all:
+	$(call print_header, "Deploying binaries to ${INSTALL_DIR}...")
+	conan install ${CONAN_OPTIONS} --install-folder ${DEPLOY_DIR} -g deploy .
+	mkdir -p ${INSTALL_DIR}
+	cp -r ${DEPLOY_DIR}/cloe-*/* ${INSTALL_DIR}/
 
-.PHONY: smoketest
-smoketest: smoketest-select
+.PHONY: clean-all
+clean-all:
+	${MAKE} clean clean-select
 
 .PHONY: purge-all
 purge-all:
@@ -100,6 +125,10 @@ purge-all:
 	conan remove -f 'cloe'
 	conan remove -f 'fable'
 
+.PHONY: package-all
+package-all:
+	conan install ${CONAN_OPTIONS} --install-folder ${DEPLOY_DIR} --build=missing --build=outdated ${DEPLOY_LOCKFILE_SOURCE}
+
 # Development targets ---------------------------------------------------------
 help::
 	$(call print_help_section, "Available development targets")
@@ -122,10 +151,15 @@ todos:
 	${AG} FIXME
 	${AG} XXX
 
+# Hidden development targets --------------------------------------------------
+
 .PHONY: grep-uuids
 grep-uuids:
 	${AG} "\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b"
 
+grep-conan-requires:
+	@rg -t py '^.*requires\(f?["](.+/[0-9]+\.[^)]+)["].*\).*$$' -r '$$1' -I --no-heading --no-line-number | sort | uniq
+
 .PHONY: find-missing-eol
 find-missing-eol:
 	find . -type f -size +0 -exec gawk 'ENDFILE{if ($0 == "") print FILENAME}' {} \;
@@ -134,5 +168,12 @@ find-missing-eol:
 sanitize-files:
 	git grep --cached -Ilz '' | while IFS= read -rd '' f; do tail -c1 < "$$f" | read -r _ || echo >> "$$f"; done
 
-# Build targets ---------------------------------------------------------------
+# Micro-packages build targets ------------------------------------------------
 include Makefile.all
+
+# Mono-package build targets --------------------------------------------------
+DISABLE_HELP_PREAMBLE := true
+help::
+	@printf "Available $(_yel)cloe$(_rst) package targets:\n"
+
+include Makefile.package

Makefile.all

@@ -35,9 +35,25 @@ SHELL := /bin/bash
 GNUMAKEFLAGS := --no-print-directory
 SUBMAKEFLAGS :=
 
-META_PKG := cloe
-PLUGIN_PKGS := $(wildcard plugins/*)
-ALL_PKGS := fable runtime models oak osi engine ${PLUGIN_PKGS} ${META_PKG}
+PLUGIN_PKGS := \
+	plugins/basic \
+	plugins/clothoid_fit \
+	plugins/esmini \
+	plugins/frustum_culling \
+	plugins/gndtruth_extractor \
+	plugins/minimator \
+	plugins/mocks \
+	plugins/noisy_sensor \
+	plugins/speedometer \
+	plugins/virtue
+ALL_PKGS := \
+	fable \
+	runtime \
+	models \
+	osi \
+	oak \
+	engine \
+	${PLUGIN_PKGS}
 WITHOUT_PKGS :=
 UNSELECT_PKGS := ${WITHOUT_PKGS}
 WITH_PKGS :=
@@ -59,11 +75,11 @@ SELECT_VENDOR := $(call uniq, $(filter-out ${UNSELECT_VENDOR}, ${ALL_VENDOR}) ${
 fable:
 runtime: fable
 models: runtime
+osi: runtime models
 oak: runtime
 engine: models oak
-osi: runtime models vendor/open-simulation-interface
 ${PLUGIN_PKGS}: runtime models
-plugins/esmini: vendor/open-simulation-interface vendor/esmini
+plugins/esmini: osi
 
 vendor/esmini: vendor/open-simulation-interface
 vendor/esmini-data:
@@ -112,18 +128,9 @@ ${1}-each: ${4}
 endef
 
 REGEX_TARGET := 's/(-vendor|-select)?-each//'
-$(filter-out ${META_PKG}, ${ALL_PKGS} ${ALL_VENDOR}):
+${ALL_PKGS} ${ALL_VENDOR}:
 	${MAKE} -C $@ $(shell echo ${MAKECMDGOALS} | sed -re ${REGEX_TARGET})
 
-# Re-define ${META_PKG} target to use Makefile.package, and only run for targets
-# where it makes sense, since "${META_PKG}" is a Conan meta-package.
-${META_PKG}:
-	for case in export package package-outdated list purge clean smoketest smoketest-deps; do \
-		if [ "$$(echo '${MAKECMDGOALS}' | sed -re ${REGEX_TARGET})" == "$${case}" ]; then \
-			${MAKE} -f Makefile.package CONAN_OPTIONS="${CONAN_OPTIONS}" $${case} || exit 1; \
-		fi \
-	done
-
 # Usage: $(call make_vendor_target, TARGET-NAME, HELP-DESCRIPTION, HELP-CATEGORY)
 define make_vendor_target
 $(eval $(call _make_target_rules,${1},${2},${3},${SELECT_VENDOR}))
@@ -145,19 +152,16 @@ endef
 .PHONY: help
 .SILENT: help
 help::
-	$(call print_help_section, "Available build targets")
+	$(call print_help_section, "Available multi-package targets")
 
+ifneq "${ALL_VENDOR}" ""
+help::
 $(call make_vendor_target, export-vendor, "export all vendor packages", "[conan-cache]")
 $(call make_vendor_target, package-vendor, "create all vendor packages", "[conan-cache]")
 $(call make_vendor_target, download-vendor, "download or build vendor packages", "[conan-cache]")
-
 help::
 	echo
-
-$(call make_every_target, export, "export all package recipes", "[conan-cache]")
-help::
-	$(call print_help_target, package, "create ${META_PKG} package and plugins", "[conan-cache]")
-	echo
+endif
 
 $(call make_select_target, export-select, "export selected packages", "[conan-cache]")
 $(call make_select_target, package-select, "create selected packages with policy", "[conan-cache]")
@@ -185,34 +189,19 @@ $(call make_select_target, clean-select, "remove build artifacts", "[in-source]"
 help::
 	echo
 	$(call print_help_subsection, "Options")
+ifneq "${ALL_VENDOR}" ""
 	$(call print_help_option, WITH_VENDOR, "", "include optional vendor packages from ${_grn}UNSELECT_VENDOR${_rst}")
+endif
 	$(call print_help_option, WITH_PKGS, "", "include optional packages from ${_grn}UNSELECT_PKGS${_rst}")
 	$(call print_help_option, LOCKFILE_SOURCE, "", "use specified conanfile as lockfile source for build")
 	echo
 	$(call print_help_subsection, "Defines")
 	$(call print_help_option, BUILD_POLICY, ${BUILD_POLICY})
 	$(call print_help_define, CONAN_OPTIONS, ${CONAN_OPTIONS})
+ifneq "${ALL_VENDOR}" ""
 	$(call print_help_define_lines, UNSELECT_VENDOR, ${UNSELECT_VENDOR})
 	$(call print_help_define_lines, SELECT_VENDOR, ${SELECT_VENDOR})
+endif
 	$(call print_help_define_lines, UNSELECT_PKGS, ${UNSELECT_PKGS})
 	$(call print_help_define_lines, SELECT_PKGS, ${SELECT_PKGS})
 	echo
-
-.PHONY: package
-package: export-select
-	# Build cloe with all targets and options together.
-	#
-	#   This is different from the package target in that it always builds the
-	#   packages from this workspace, the ones in SELECT_PKGS.
-	#   This is different from the package-select target in that it builds them
-	#   all together and thereby uses the correct dependency resolution with
-	#   overrides and options.
-	TARGETS=$$( \
-		for pkg in ${SELECT_PKGS}; do \
-			if [ ! -d $${pkg} ]; then \
-				continue; \
-			fi; \
-			echo -n "--build=$$(make --no-print-directory -C $${pkg} info-name) "; \
-		done; \
-	) && \
-	${MAKE} -f Makefile.package CONAN_OPTIONS="${CONAN_OPTIONS} $$TARGETS" package