vtd: Change compression method to avoid revision change

Author: scpa1055

optional/vtd/Makefile

@@ -11,7 +11,7 @@ smoketest-req:
 
 SELECT_VENDOR := $(wildcard vendor/*)
 .PHONY: ${SELECT_VENDOR}
-vendor/osi-sensor-1.0.0-vtd2.2: vendor/vtd vendor/open-simulation-interface-3.0.1
+vendor/osi-sensor-1.0.0-vtd2.2: vendor/vtd-2.2.0 vendor/open-simulation-interface-3.0.1
 vendor/open-simulation-interface-3.0.1: vendor/protobuf-2.6.1
 vendor/open-simulation-interface-3.2.0: vendor/protobuf-2.6.1
 

optional/vtd/bin/vtd-launch

@@ -254,9 +254,12 @@ def start(
         shutil.copytree(vtd_setup, dst, symlinks=True)
 
     elif vtd_setup_cloe is not None:
-        vtd_setup_cloe = Path(os.getenv("VTD_SETUP_DIR")) / f"{vtd_setup_cloe}.tgz"
+        # Compressing will add a timestamp to the package and therefore 
+        # leads to different package_hashes everytime we export with conan.
+        # To avoid that, changing from .tgz to .tar was necessary
+        vtd_setup_cloe = Path(os.getenv("VTD_SETUP_DIR")) / f"{vtd_setup_cloe}.tar"
         logging.info(f"Use setup: {vtd_setup_cloe}")
-        with tarfile.open(vtd_setup_cloe, "r:gz") as tar:
+        with tarfile.open(vtd_setup_cloe, "r:tar") as tar:
             dst = opt.runtime_dir / "Data" / "Setups" / "Current"
             if dst.exists():
                 os.remove(dst)

optional/vtd/conanfile.py

@@ -63,48 +63,53 @@ def build_requirements(self):
         self.test_requires("gtest/[~1.10]")
 
     def _compress_and_remove(self, dir):
+        # reset() will remove the packages metadata
+        def reset(tarinfo):
+            tarinfo.uid = tarinfo.gid = 0
+            tarinfo.uname = ""
+            tarinfo.gname = ""
+            tarinfo.mtime = 1
+            return tarinfo
+
         if not dir.is_dir():
             return
-        with tarfile.open(f"{dir.name}.tgz", "w:gz") as tar:
-            tar.add(dir.path, arcname=os.path.basename(dir.path))
+        # Compressing will add a timestamp to the package and therefore 
+        # leads to different package_hashes everytime we export with conan.
+        # To avoid that, changing from .tgz to .tar was necessary
+        with tarfile.open(f"{dir.name}.tar", "w:") as tar:
+            tar.add(dir.path, arcname=os.path.basename(dir.path), filter=reset)
             rmtree(dir.path)
 
     def _compress_setups(self):
         cwd = os.getcwd()
-        os.chdir(f"{self.export_sources_folder}/{self._setup_folder}")
-        with os.scandir() as scan:
-            for dir in scan:
-                self._compress_and_remove(dir)
+        setup_dir = f"{self.export_sources_folder}/{self._setup_folder}"
+        os.chdir(setup_dir)
+        versions = [name for name in os.listdir(setup_dir) if os.path.isdir(os.path.join(setup_dir, name))]
+        for version in versions:
+            os.chdir(os.path.join(setup_dir, version))
+            with os.scandir() as scan:
+                for dir in scan:
+                    self._compress_and_remove(dir)
         os.chdir(cwd)
 
     def export_sources(self):
         self.copy("*", dst=self._setup_folder, src=self._setup_folder, symlinks=True)
         self._compress_setups()
 
-    def _configure_cmake(self):
-        vtd_api_version = str(self.requires.get("vtd-api"))
-        if "vtd-api/2.2.0" in vtd_api_version:
-            self.vtd_api_version = "2.2.0"
-        else:
-            self.vtd_api_version = "2022.3"
-
-        if self._cmake:
-            return self._cmake
-        self._cmake = CMake(self)
-        self._cmake.definitions["VTD_API_VERSION"] = self.vtd_api_version
-        self._cmake.definitions["CMAKE_EXPORT_COMPILE_COMMANDS"] = True
-        self._cmake.definitions["BuildTests"] = self.options.test
-        self._cmake.definitions["TargetLintingExtended"] = self.options.pedantic
-        self._cmake.configure()
-        return self._cmake
-
     def configure(self):
         self.options["open-simulation-interface"].shared = False
         self.options["open-simulation-interface"].fPIC = True
 
     def generate(self):
+        vtd_api_version = str(self.requires.get("vtd-api"))
+        if "vtd-api/2.2.0" in vtd_api_version:
+            self.vtd_api_version = "2.2.0"
+        else:
+            self.vtd_api_version = "2022.3"
         tc = cmake.CMakeToolchain(self)
         tc.cache_variables["CMAKE_EXPORT_COMPILE_COMMANDS"] = True
+        tc.preprocessor_definitions["VTD_API_VERSION"] = self.vtd_api_version
+        tc.cache_variables["VTD_API_VERSION"] = self.vtd_api_version
         tc.cache_variables["CLOE_PROJECT_VERSION"] = self.version
         tc.cache_variables["TargetLintingExtended"] = self.options.pedantic
         tc.generate()
@@ -120,9 +125,9 @@ def package(self):
         cm.install()
         self.copy("vtd-launch", dst="bin", src=f"{self.source_folder}/bin")
         self.copy(
-            "*.tgz",
+            "*.tar",
             dst=self._setup_folder,
-            src=f"{self.source_folder}/{self._setup_folder}",
+            src=f"{self.source_folder}/{self._setup_folder}/{self.vtd_api_version}"
         )
 
     def package_id(self):

optional/vtd/tests/profile_with_vtd-2.2.0.py

@@ -35,7 +35,7 @@ def requirements(self):
         self.requires(f"cloe-plugin-speedometer/{self.version}@cloe/develop")
         self.requires(f"cloe-plugin-virtue/{self.version}@cloe/develop")
         self.requires(f"cloe-plugin-vtd/{self.version}@cloe/develop")
-
+        self.requires("incbin/cci.20211107", override=True)
         # Runtime requirements for VTD.
         self.requires("osi-sensor/1.0.0-vtd2.2@cloe/stable")
         self.requires("vtd/2.2.0@cloe-restricted/stable")

optional/vtd/tests/profile_with_vtd-2022.3.py

@@ -34,11 +34,10 @@ def requirements(self):
         self.requires(f"cloe-plugin-noisy-sensor/{self.version}@cloe/develop")
         self.requires(f"cloe-plugin-speedometer/{self.version}@cloe/develop")
         self.requires(f"cloe-plugin-virtue/{self.version}@cloe/develop")
-
+        self.requires("incbin/cci.20211107", override=True)
         self.requires(f"cloe-plugin-vtd/{self.version}@cloe/develop")
         self.requires("vtd-api/2022.3@cloe/stable", override=True)
         # Runtime requirements for VTD.
-        self.requires("osi-sensor/1.0.0-vtd2.2@cloe/stable")
         self.requires("vtd/2022.3@cloe-restricted/stable")
 
         # Overrides:

optional/vtd/vendor/vtd-2022.3/Dockerfile

@@ -6,22 +6,22 @@
 #
 # Build the image like this:
 #
-#   docker build -t cloe/vtd-conan-package:2.2.0 Dockerfile vires
+#   docker build -t cloe/vtd-conan-package:2022.3 Dockerfile vires
 #
 # And use it in another Dockerfile (with buildkit) like this:
 #
-#   RUN --mount=type=bind,target=/root/.conan/vtd/2.2.0/cloe-restricted/stable,from=cloe/vtd-conan-package:2.2.0
+#   RUN --mount=type=bind,target=/root/.conan/vtd/2022.3/cloe-restricted/stable,from=cloe/vtd-conan-package:2022.3
 #
 # Note, you can create this image even if you don't have anything in the vires
 # directory. It will result in an empty image, but it will allow the other
 # Docker builds to continue.
 #
-FROM ubuntu:18.04 AS build
+FROM ubuntu:20.04 AS build
 
 ENV DEBIAN_FRONTEND=noninteractive
 
-RUN --mount=type=cache,id=bionic-cache,target=/var/cache/apt \
-    --mount=type=cache,id=bionic-lib,target=/var/lib/apt \
+RUN --mount=type=cache,id=ubuntu-20.04-cache,target=/var/cache/apt \
+    --mount=type=cache,id=ubuntu-20.04-lib,target=/var/lib/apt \
     apt-get update && \
     apt-get install --no-install-recommends -y \
         build-essential \
@@ -145,7 +145,7 @@ RUN --mount=type=cache,id=bionic-cache,target=/var/cache/apt \
     rm -rf /var/lib/apt/lists/*
 
 RUN pip3 install --upgrade pip && \
-    pip3 install conan
+    pip3 install "conan<2.0.0"
 
 RUN conan profile new --detect default && \
     conan profile update settings.compiler.libcxx=libstdc++11 default
@@ -163,7 +163,7 @@ RUN conan create conanfile.py ${PACKAGE_FQN} && \
     conan remove -b -s -f vtd && \
     find /root/.conan/data -type d -name "dl" -exec rm -rf {} +
 
-FROM ubuntu:18.04
+FROM ubuntu:20.04
 COPY --from=build /root/.conan/data/vtd /vtd
 RUN ln -s /vtd/*/*/*/package/* /VTD
 COPY --from=build /vtd/entrypoint.sh ./

optional/vtd/vendor/vtd-2022.3/conanfile.py

@@ -1,3 +1,6 @@
+# mypy: ignore-errors
+# pylint: skip-file
+
 import os
 import os.path
 import shutil
@@ -72,7 +75,7 @@ class VtdConan(ConanFile):
     options = {
         "with_osi": [True, False],
         "with_road_designer": [True, False],
-        "with_image_generator":  [True, False], 
+        "with_image_generator":  [True, False],
     }
     default_options = {
         "with_osi": True,
@@ -95,7 +98,8 @@ def export_sources(self):
 
     def configure(self):
         if self.settings.os == "Windows":
-            raise ConanInvalidConfiguration("VTD binaries do not exist for Windows")
+            raise ConanInvalidConfiguration(
+                "VTD binaries do not exist for Windows")
 
     def build(self):
         src = Path(self.source_folder)
@@ -106,7 +110,7 @@ def build(self):
         def extract_archive(archive):
             print(f"Extracting: {archive}")
             tools.untargz(src / archive, dst)
-        
+
         extract_archive(self._archive_base)
         libdir.mkdir()
         if self.options.with_osi:
@@ -118,73 +122,61 @@ def extract_archive(archive):
             shutil.rmtree(path)
             Path(vtddir / "Runtime/Core/ImageGenerator").unlink()
             remove_broken_symlinks()
-            
+
         # Patch RPATH of several critical binaries.
         patch_rpath(
             vtddir / "Runtime/Core/ModuleManager/moduleManager.2022.3.40_flexlm",
             ["$ORIGIN/../Lib", "$ORIGIN/lib"],
         )
-
         patch_rpath(
             vtddir / "Runtime/Core/ModuleManager/lib/libVTDModulePlugin.so.2022",
             ["$ORIGIN/../../Lib"]
         )
-
         patch_rpath(
             vtddir / "Runtime/Core/ModuleManager/lib/libprotobuf.so.9",
             ["$ORIGIN/../../Lib"]
         )
-
         patch_rpath(
             vtddir / "Runtime/Core/ModuleManager/lib/libopen_simulation_interface.so",
             ["$ORIGIN/../../Lib", "$ORIGIN"],
         )
-
         patch_rpath(
             vtddir / "Runtime/Core/ModuleManager.cxx98/moduleManager.2022.3.40_flexlm",
             ["$ORIGIN/../Lib", "$ORIGIN/lib"],
         )
-
         patch_rpath(
             vtddir / "Runtime/Core/ModuleManager.cxx98/lib/libopen_simulation_interface.so",
             ["$ORIGIN/../../Lib", "$ORIGIN"],
         )
-
         patch_rpath(
             vtddir / "Runtime/Core/ModuleManager.cxx98/lib/libprotobuf.so.9",
             ["$ORIGIN/../../Lib"],
         )
-
         patch_rpath(
             vtddir / "Runtime/Core/ModuleManager.cxx98/lib/libVTDModulePlugin.so",
             ["$ORIGIN/../../Lib"],
         )
-
         patch_rpath(
             vtddir / "Runtime/Core/ModuleManager.cxx11/moduleManager.2022.3.40_flexlm",
             ["$ORIGIN/../Lib", "$ORIGIN/lib"],
         )
-
-
         patch_rpath(
             vtddir / "Runtime/Core/ModuleManager.cxx11/lib/libVTDModulePlugin.so.2022",
             ["$ORIGIN/../../Lib"],
         )
-
         patch_rpath(
             vtddir / "Runtime/Core/ModuleManager.cxx11/lib/libopen_simulation_interface.so",
             ["$ORIGIN/../../Lib", "$ORIGIN"],
         )
-
         patch_rpath(
             vtddir / "Runtime/Core/ModuleManager.cxx11/lib/libprotobuf.so.9",
             ["$ORIGIN/../../Lib"],
         )
-
         patch_rpath(
-            vtddir / "Runtime/Core/ParamServer/paramServer.2022.3.40", ["$ORIGIN/../Lib"]
+            vtddir /
+            "Runtime/Core/ParamServer/paramServer.2022.3.40", [
+                "$ORIGIN/../Lib"]
         )
-        
         patch_rpath(
             vtddir / "Runtime/Core/Traffic/ghostdriver.2022.3.40_flexlm",
             ["$ORIGIN/../Lib"],
@@ -194,7 +186,8 @@ def extract_archive(archive):
         for file in find_binary_files():
             try:
                 patch_rpath(
-                    file, [f"$ORIGIN/{os.path.relpath(libdir, (dst / file).parent)}"]
+                    file, [
+                        f"$ORIGIN/{os.path.relpath(libdir, (dst / file).parent)}"]
                 )
             except:
                 # Not all files can be set, but even if this happens it doesn't appear

optional/vtd/vendor/vtd-2022.3/entrypoint.sh

@@ -1,22 +1,25 @@
 #!/bin/bash
 set -e
-
-# This entrypoint script wraps VTD like it would be running in the foreground
-# VTD's startscript actually terminates itself and the main simulator proceeses
-# are regularly a bunch of orphans.
-# In order to compensate for this misbehavior, this script monitor's the state
+# This entrypoint script wraps VTD so it is as if it were running in the foreground.
+#
+# VTD's start script terminates itself, so that the main simulator processes
+# are regularly a bunch of orphans (zombies).
+# In order to compensate for this misbehavior, this script monitors the state
 # of the most important process, the simServer. Its termination leads to the
 # container's termination.
+#
 # All arguments are passed unchanged to the vtdStart.sh script.
-cd /vtd_setups && cp -r Cloe* /VTD/Data/Setups
-cd /VTD/Data/Setups && \
-    rm -f Current && \
-    ln -s Cloe.noGUInoIG Current
-
-cd /VTD && bin/vtdStart.sh $@
+# Make Cloe setups available to VTD, since it expects everything
+# in its own directory.
+cd /vtd_setups
+cp -r Cloe* /VTD/Data/Setups
+# Within a container, use the noGUInoIG setup, since
+# we are not running in a desktop environment.
+cd /VTD/Data/Setups
+ln -sf Cloe.noGUInoIG Current
+# Start VTD
+cd /VTD
+bin/vtdStart.sh $@
 
-# workaround for vtd start script starting vtd in the background
-while pidof simServer > /dev/null
-do
-  sleep 1
-done
+pid=$(pidof simServer)
+wait $pid