vtd: Move vtd with dependencies into optional/vtd directory

Author: cassava

.dockerignore

@@ -2,4 +2,4 @@
 build/
 **/build/
 dist/docker/
-vendor/vtd/dl/
+optional/

Makefile.all

@@ -10,7 +10,7 @@
 # Use WITHOUT_PKGS variable on the command line to opt-out of specific
 # packages:
 #
-#     make WITHOUT_PKGS="plugins/vtd plugins/demo_printer" package
+#     make WITHOUT_PKGS="plugins/demo_printer" package
 #
 # This extends the UNSELECT_PKGS variable. Set this variable directly
 # to override default omitted packages. Use make help to see which
@@ -21,7 +21,7 @@
 # Use WITH_PKGS variable on the command line to include specific by-default
 # unselected packages:
 #
-#     make WITH_PKGS="plugins/vtd" package
+#     make PLUGIN_PKGS= WITH_PKGS="plugins/minimator" package
 #
 # This extends the SELECT_PKGS variable. Set this variable directly
 # to explicitely specify exactly which packages are considered. Use make help

docs/develop/debugging-cloe.rst

@@ -0,0 +1,4 @@
+Dockerfile.ubuntu
+Makefile.docker
+vendor/vtd/dl
+setup.sh

docs/reference/plugins/gndtruth_extractor.rst

@@ -0,0 +1,2 @@
+build/
+setup.sh

docs/reference/plugins/noisy_lane_sensor.rst

@@ -0,0 +1,78 @@
+# syntax = docker/dockerfile:1.4.3
+# Dockerfile
+#
+# This file acts as a Docker recipe for building cloe-plugin-vtd on Ubuntu.
+# It uses the base image from Cloe to reduce overhead.
+ARG IMAGE_BASE=cloe/cloe-engine
+ARG PROJECT_VERSION
+ARG UBUNTU_VERSION
+ARG IMAGE_VERSION=${PROJECT_VERSION}-ubuntu-${UBUNTU_VERSION}
+
+# This is a work-around to not being able to use variables in RUN --mount=from:
+# If you want to use VTD in this image, you need to specify the Docker image
+# containing the distribution that can be mounted at /root/.conan/data/
+ARG VTD_IMAGE=scratch
+FROM ${VTD_IMAGE} as vtd
+WORKDIR /vtd
+
+FROM ${IMAGE_BASE}:${IMAGE_VERSION} as stage-vendor
+WORKDIR /cloe-vtd
+ARG KEEP_SOURCES=0
+
+# Download vendor packages:
+COPY vendor /cloe-vtd/vendor
+COPY Makefile /cloe-vtd
+COPY conanfile.py /cloe-vtd
+ARG VENDOR_TARGET="export-vendor download-vendor"
+RUN --mount=type=cache,target=/ccache \
+    --mount=type=secret,target=/root/setup.sh,id=setup,mode=0400 \
+    --mount=type=bind,target=/root/.conan/data/vtd,source=/vtd,from=vtd,rw \
+    if [ -r /root/setup.sh ]; then . /root/setup.sh; fi && \
+    conan search && \
+    cat /root/.conan/data/vtd/2.2.0/cloe-restricted/stable/metadata.json && \
+    make PROJECT_ROOT=/cloe ${VENDOR_TARGET} && \
+    # Clean up:
+    conan user --clean && \
+    if [ ${KEEP_SOURCES} -eq 0 ]; then \
+        find /root/.conan/data -name dl -type d -maxdepth 5 -exec rm -r {} + && \
+        conan remove \* -s -b -f; \
+    else \
+        conan remove \* -b -f; \
+    fi
+
+# Build cloe-plugin-vtd package:
+FROM stage-vendor as stage-package
+COPY . /cloe-vtd
+ARG PROJECT_VERSION=unknown
+ARG PACKAGE_TARGET="package"
+RUN --mount=type=cache,target=/ccache \
+    --mount=type=secret,target=/root/setup.sh,id=setup,mode=0400 \
+    --mount=type=bind,target=/root/.conan/data/vtd,source=/vtd,from=vtd,rw \
+    if [ -r /root/setup.sh ]; then . /root/setup.sh; fi && \
+    conan search && \
+    echo "${PROJECT_VERSION}" > /cloe-vtd/VERSION && \
+    make PROJECT_ROOT=/cloe ${PACKAGE_TARGET} && \
+    # Clean up:
+    conan user --clean && \
+    if [ ${KEEP_SOURCES} -eq 0 ]; then \
+        find /root/.conan/data -name dl -type d -maxdepth 5 -exec rm -r {} + && \
+        conan remove \* -s -b -f; \
+    else \
+        conan remove \* -b -f; \
+    fi
+
+# Build smoketest dependencies (in case they are different):
+FROM stage-package as stage-smoketest-deps
+RUN --mount=type=cache,target=/ccache \
+    --mount=type=secret,target=/root/setup.sh,id=setup,mode=0400 \
+    --mount=type=bind,target=/root/.conan/data/vtd,source=/vtd,from=vtd,rw \
+    if [ -r /root/setup.sh ]; then . /root/setup.sh; fi && \
+    make PROJECT_ROOT=/cloe smoketest-deps && \
+    # Clean up:
+    conan user --clean && \
+    if [ ${KEEP_SOURCES} -eq 0 ]; then \
+        find /root/.conan/data -name dl -type d -maxdepth 5 -exec rm -r {} + && \
+        conan remove \* -s -b -f; \
+    else \
+        conan remove \* -b -f; \
+    fi

docs/reference/plugins/noisy_object_sensor.rst

@@ -0,0 +1,47 @@
+PROJECT_ROOT := ../..
+include ${PROJECT_ROOT}/Makefile.package
+
+smoketest: smoketest-req
+
+smoketest-req:
+	@echo "VI_LIC_SERVER = $$VI_LIC_SERVER"
+	test -n $$VI_LIC_SERVER
+
+# -------------------------------------------------
+
+SELECT_VENDOR := $(wildcard vendor/*)
+.PHONY: ${SELECT_VENDOR}
+vendor/osi-sensor-1.0.0-vtd2.2: vendor/vtd vendor/open-simulation-interface-3.0.1
+vendor/open-simulation-interface-3.0.1: vendor/protobuf-2.6.1
+vendor/open-simulation-interface-3.2.0: vendor/protobuf-2.6.1
+
+REGEX_TARGET := 's/(-vendor|-select)?-each//'
+${SELECT_VENDOR}:
+	${MAKE} -C $@ $(shell echo ${MAKECMDGOALS} | sed -re ${REGEX_TARGET})
+
+# Usage: $(call _make_target_rule, TARGET-NAME, MAKE-TARGET, HELP-DESCRIPTION, MAKE-ARGUMENTS)
+define _make_target_rule
+${1}:
+	$(call print_header, "Proceeding to $(call unquote, ${3})")
+	${MAKE} ${SUBMAKEFLAGS} PROJECT_ROOT=$(realpath ${PROJECT_ROOT}) ${4} ${2}
+endef
+
+# Usage: $(call _make_target_rules, TARGET-NAME, HELP-DESCRIPTION, HELP-CATEGORY, PACKAGE-DIRS)
+define _make_target_rules
+help::
+	$(call print_help_target, ${1}, ${2}, ${3})
+$(call _make_target_rule,${1},${1}-each,${2})
+${1}-each: ${4}
+endef
+
+# Usage: $(call make_vendor_target, TARGET-NAME, HELP-DESCRIPTION, HELP-CATEGORY)
+define make_vendor_target
+$(eval $(call _make_target_rules,${1},${2},${3},${SELECT_VENDOR}))
+endef
+
+help::
+	$(call print_help_section, "Available vendor targets")
+
+$(call make_vendor_target, export-vendor, "export all vendor packages", "[conan-cache]")
+$(call make_vendor_target, package-vendor, "create all vendor packages", "[conan-cache]")
+$(call make_vendor_target, download-vendor, "download or build vendor packages", "[conan-cache]")

optional/vtd/.dockerignore

@@ -0,0 +1,116 @@
+# Usage: make [options] <target>
+#
+
+PROJECT_ROOT := ../..
+PROJECT_VERSION := $(shell make --no-print-directory -C ${PROJECT_ROOT} -f Makefile.package info-version)
+
+include ${PROJECT_ROOT}/Makefile.help
+
+DOCKER := DOCKER_BUILDKIT=1 docker
+
+DOCKER_NETWORK := \
+	$(shell \
+		if [ -n $$https_proxy ]; then \
+			echo " --network=host"; \
+			echo " --build-arg https_proxy=\"$$https_proxy\""; \
+			echo " --build-arg http_proxy=\"$$http_proxy\""; \
+			echo " --build-arg no_proxy=\"$$no_proxy\""; \
+		fi \
+	)
+
+VTD_IMAGE := cloe/vtd-conan-package:2.2.0
+IMAGE_BASE := cloe/cloe-engine-with-vtd
+IMAGE_VERSION := ${PROJECT_VERSION}
+DOCKER_IMAGE := ${IMAGE_BASE}:${IMAGE_VERSION}
+DOCKER_CONTEXT := .
+
+# Default build arguments
+VENDOR_TARGET :=
+PACKAGE_TARGET :=
+CONAN_PROFILE :=
+KEEP_SOURCES :=
+
+DOCKER_BUILD_ARGS += ${DOCKER_NETWORK}
+DOCKER_BUILD_ARGS += --build-arg PROJECT_VERSION=${PROJECT_VERSION} --build-arg VTD_IMAGE=${VTD_IMAGE}
+
+ifeq ($(shell [ -f setup.sh ] && echo "true"),true)
+DOCKER_BUILD_ARGS += --secret id=setup,src=setup.sh
+endif
+
+ifneq (${CONAN_PROFILE},)
+DOCKER_BUILD_ARGS += --build-arg CONAN_PROFILE=${CONAN_PROFILE}
+endif
+
+ifneq (${VENDOR_TARGET},)
+DOCKER_BUILD_ARGS += --build-arg VENDOR_TARGET=${VENDOR_TARGET}
+endif
+
+ifneq (${PACKAGE_TARGET},)
+DOCKER_BUILD_ARGS += --build-arg PACKAGE_TARGET=${PACKAGE_TARGET}
+endif
+
+ifeq (${KEEP_SOURCES},1)
+DOCKER_IMAGE := ${DOCKER_IMAGE}-with-src
+DOCKER_BUILD_ARGS += --build-arg KEEP_SOURCES=1
+endif
+
+DOCKER_USER_ARGS :=
+DOCKER_ARGS := ${DOCKER_BUILD_ARGS} ${DOCKER_USER_ARGS}
+
+DOCKER_RUN_ARGS=-it --rm --network=host -v $$(pwd)/setup.sh:/root/setup.sh -v conan-data-vtd:/root/.conan/data/vtd
+
+.PHONY: help
+.SILENT: help
+.DEFAULT: help
+help:
+	$(call print_help_usage)
+	echo
+	$(call print_help_section, "Available build targets")
+	$(call print_help_target, all, "build all stable docker images: ubuntu-18.04 ubuntu-20.04 ubuntu-22.04")
+	$(call print_help_target, ubuntu-VERSION, "build and test the Ubuntu VERSION (e.g. 20.04) image")
+	$(call print_help_target, build-ubuntu-VERSION, "build the Ubuntu VERSION image")
+	$(call print_help_target, test-ubuntu-VERSION, "test the Ubuntu VERSION image")
+	$(call print_help_target, run-ubuntu-VERSION, "run the Ubuntu VERSION image")
+	echo
+	$(call print_help_section, "Available system targets")
+	$(call print_help_target, release, "push Conan packages from current images to remote")
+	$(call print_help_target, purge-current, "purge and prune current docker versions")
+	$(call print_help_target, purge-all, "purge and prune all docker versions")
+	echo
+	$(call print_help_section, "Configuration")
+	$(call print_help_define, IMAGE_BASE, ${IMAGE_BASE})
+	$(call print_help_define, IMAGE_VERSION, ${IMAGE_VERSION})
+	$(call print_help_define, DOCKER_CONTEXT, ${DOCKER_CONTEXT})
+	$(call print_help_define_lines, DOCKER_ARGS, ${DOCKER_ARGS})
+
+.PHONY: all ubuntu-%
+all: ubuntu-18.04 ubuntu-20.04 ubuntu-22.04
+ubuntu-%: | build-$@ test-$@
+
+build-ubuntu-%: Dockerfile.ubuntu
+	${DOCKER} build -f Dockerfile.ubuntu --build-arg UBUNTU_VERSION=$(subst build-ubuntu-,,$@) ${DOCKER_ARGS} -t ${DOCKER_IMAGE}-$(subst build-,,$@) ${DOCKER_CONTEXT}
+
+.PHONY: test-ubuntu-%
+test-ubuntu-%:
+	docker run ${DOCKER_RUN_ARGS} ${IMAGE_BASE}:${IMAGE_VERSION}-$(subst test-,,$@) bash -c "source /root/setup.sh && make PROJECT_ROOT=/cloe smoketest"
+
+.PHONY: run-ubuntu-%
+run-ubuntu-%:
+	docker run ${DOCKER_RUN_ARGS} ${IMAGE_BASE}:${IMAGE_VERSION}-$(subst run-,,$@) bash
+
+release:
+	@test -f setup.sh || echo 'Error: require setup.sh for user authentication'
+	for dist in ubuntu-18.04 ubuntu-20.04 ubuntu-22.04; do \
+		${DOCKER} run --rm \
+			$(subst --build-arg,-e,${DOCKER_NETWORK}) \
+			--mount "type=bind,source=$(CURDIR)/setup.sh,target=/root/setup.sh" \
+			${DOCKER_IMAGE}-$${dist} \
+			bash -c '. /root/setup.sh && conan upload --force --all -c "*"'; \
+	done
+
+.PHONY: purge-current purge-all
+purge-current:
+	docker rmi $$(docker images --format '{{.Repository}}:{{.Tag}}' | grep '${DOCKER_IMAGE}')
+
+purge-all:
+	docker rmi $$(docker images --format '{{.Repository}}:{{.Tag}}' | grep '${IMAGE_BASE}:')

optional/vtd/.gitignore

@@ -179,6 +179,7 @@ def verify_runtime_contents(runtime_dir: Path) -> None:
     "-S",
     "--vtd-setup-cloe",
     type=click.Choice(
+        # FIXME: Support .tgz AND directories!
         [Path(s).stem for s in os.listdir(os.getenv("VTD_SETUP_DIR"))],
         case_sensitive=False,
     ),

plugins/vtd/.lvimrc renamed to optional/vtd/.lvimrc

@@ -37,17 +37,26 @@ class CloeSimulatorVTD(ConanFile):
     _cmake = None
 
     def set_version(self):
-        version_file = Path(self.recipe_folder) / "../../VERSION"
-        if version_file.exists():
-            self.version = tools.load(version_file).strip()
-        else:
-            git = tools.Git(folder=self.recipe_folder)
-            self.version = git.run("describe --dirty=-dirty")[1:]
+        for version_path in ["VERSION", "../../VERSION"]:
+            version_file = Path(self.recipe_folder) / version_path
+            if version_file.exists():
+                self.version = tools.load(version_file).strip()
+                return
+        git = tools.Git(folder=self.recipe_folder)
+        self.version = git.run("describe --dirty=-dirty")[1:]
 
     def requirements(self):
         self.requires(f"cloe-runtime/{self.version}@cloe/develop")
         self.requires(f"cloe-models/{self.version}@cloe/develop")
 
+        # Overrides, same as in the cloe conanfile.py:
+        self.requires("protobuf/[>=3.9.1]", override=True)
+        self.requires("zlib/1.2.12", override=True)
+        self.requires("fmt/[~=8.1.1]", override=True)
+        self.requires("inja/[~=3.3.0]", override=True)
+        self.requires("nlohmann_json/[~=3.10.5]", override=True)
+        self.requires("incbin/[~=0.88.0]@cloe/stable", override=True),
+
     def build_requirements(self):
         if self.options.test:
             self.build_requires("gtest/[~1.10]")