Merge branch 'master' of https://github.com/testcontainers/testcontainers-java into issues-2876

Author: silaev

.github/workflows/ci-rootless.yml

@@ -0,0 +1,44 @@
+name: CI-Docker-Rootless
+
+on:
+  pull_request: {}
+  push: { branches: [ master ] }
+
+jobs:
+  test:
+    runs-on: ubuntu-18.04
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - { XDG_RUNTIME_DIR: "" }
+          - { XDG_RUNTIME_DIR: "/tmp/docker-testcontainers/" }
+    env:
+      XDG_RUNTIME_DIR: ${{ matrix.XDG_RUNTIME_DIR }}
+    steps:
+      - uses: actions/checkout@v2
+      - name: debug
+        run: id -u; whoami
+      - name: uninstall rootful Docker
+        run: sudo apt-get -q -y --purge remove moby-engine moby-buildx && sudo rm -rf /var/run/docker.sock
+      - name: install rootless Docker
+        run: |
+          mkdir -p $XDG_RUNTIME_DIR || true
+          curl -fsSL https://get.docker.com/rootless | sh > init.sh
+          cat init.sh
+          source <(grep '^export' init.sh)
+          PATH=$HOME/bin:$PATH dockerd-rootless.sh --experimental --storage-driver vfs &
+          sleep 1
+          DOCKER_HOST=unix://$XDG_RUNTIME_DIR/docker.sock docker info || ls -la $XDG_RUNTIME_DIR
+      - name: Build with Gradle
+        run: ./gradlew --no-daemon --scan testcontainers:test
+      - name: aggregate test reports with ciMate
+        if: always()
+        continue-on-error: true
+        env:
+          CIMATE_PROJECT_ID: 2348n4vl
+          CIMATE_CI_KEY: "CI / rootless Docker (${{ matrix.XDG_RUNTIME_DIR }})"
+        run: |
+          wget -q https://get.cimate.io/release/linux/cimate
+          chmod +x cimate
+          ./cimate "**/TEST-*.xml"

.github/workflows/ci.yml

@@ -56,7 +56,7 @@ jobs:
         run: docker image prune -af
       - name: Build and test with Gradle (${{matrix.gradle_args}})
         run: |
-          ./gradlew --no-daemon --continue --scan --info ${{matrix.gradle_args}}
+          ./gradlew --no-daemon --continue --scan ${{matrix.gradle_args}}
       - name: Aggregate test reports with ciMate
         if: always()
         continue-on-error: true

core/src/main/java/org/testcontainers/DockerClientFactory.java

@@ -18,8 +18,10 @@
 import lombok.SneakyThrows;
 import lombok.Synchronized;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang.StringUtils;
 import org.testcontainers.dockerclient.DockerClientProviderStrategy;
 import org.testcontainers.dockerclient.DockerMachineClientProviderStrategy;
+import org.testcontainers.dockerclient.TransportConfig;
 import org.testcontainers.images.TimeLimitedLoggedPullImageResultCallback;
 import org.testcontainers.utility.ComparableVersion;
 import org.testcontainers.utility.MountableFile;
@@ -29,6 +31,7 @@
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.URI;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -57,7 +60,7 @@ public class DockerClientFactory {
             TESTCONTAINERS_SESSION_ID_LABEL, SESSION_ID
     );
 
-    private static final String TINY_IMAGE = TestcontainersConfiguration.getInstance().getTinyImage();
+    private static final String TINY_IMAGE = TestcontainersConfiguration.getInstance().getTinyDockerImageName().asCanonicalNameString();
     private static DockerClientFactory instance;
 
     // Cached client configuration
@@ -129,6 +132,24 @@ private DockerClientProviderStrategy getOrInitializeStrategy() {
         return strategy;
     }
 
+    @UnstableAPI
+    public TransportConfig getTransportConfig() {
+        return getOrInitializeStrategy().getTransportConfig();
+    }
+
+    @UnstableAPI
+    public String getRemoteDockerUnixSocketPath() {
+        String dockerSocketOverride = System.getenv("TESTCONTAINERS_DOCKER_SOCKET_OVERRIDE");
+        if (!StringUtils.isBlank(dockerSocketOverride)) {
+            return dockerSocketOverride;
+        }
+
+        URI dockerHost = getTransportConfig().getDockerHost();
+        return "unix".equals(dockerHost.getScheme())
+            ? dockerHost.getRawPath()
+            : "/var/run/docker.sock";
+    }
+
     /**
      *
      * @return a new initialized Docker client

core/src/main/java/org/testcontainers/containers/DockerComposeContainer.java

@@ -2,6 +2,8 @@
 
 import com.github.dockerjava.api.DockerClient;
 import com.github.dockerjava.api.model.Container;
+import com.github.dockerjava.core.LocalDirectorySSLConfig;
+import com.github.dockerjava.transport.SSLConfig;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Joiner;
 import com.google.common.base.Splitter;
@@ -22,6 +24,7 @@
 import org.testcontainers.containers.wait.strategy.Wait;
 import org.testcontainers.containers.wait.strategy.WaitAllStrategy;
 import org.testcontainers.containers.wait.strategy.WaitStrategy;
+import org.testcontainers.dockerclient.TransportConfig;
 import org.testcontainers.lifecycle.Startable;
 import org.testcontainers.utility.AuditLogger;
 import org.testcontainers.utility.Base58;
@@ -59,7 +62,6 @@
 import static com.google.common.base.Strings.isNullOrEmpty;
 import static java.util.stream.Collectors.joining;
 import static java.util.stream.Collectors.toList;
-import static org.testcontainers.containers.BindMode.READ_ONLY;
 import static org.testcontainers.containers.BindMode.READ_WRITE;
 
 /**
@@ -580,12 +582,11 @@ interface DockerCompose {
  */
 class ContainerisedDockerCompose extends GenericContainer<ContainerisedDockerCompose> implements DockerCompose {
 
-    private static final String DOCKER_SOCKET_PATH = "/var/run/docker.sock";
     public static final char UNIX_PATH_SEPERATOR = ':';
 
     public ContainerisedDockerCompose(List<File> composeFiles, String identifier) {
 
-        super(TestcontainersConfiguration.getInstance().getDockerComposeContainerImage());
+        super(TestcontainersConfiguration.getInstance().getDockerComposeDockerImageName());
         addEnv(ENV_PROJECT_NAME, identifier);
 
         // Map the docker compose file into the container
@@ -601,24 +602,18 @@ public ContainerisedDockerCompose(List<File> composeFiles, String identifier) {
         final String composeFileEnvVariableValue = Joiner.on(UNIX_PATH_SEPERATOR).join(absoluteDockerComposeFiles); // we always need the UNIX path separator
         logger().debug("Set env COMPOSE_FILE={}", composeFileEnvVariableValue);
         addEnv(ENV_COMPOSE_FILE, composeFileEnvVariableValue);
-        addFileSystemBind(pwd, containerPwd, READ_ONLY);
+        addFileSystemBind(pwd, containerPwd, READ_WRITE);
 
         // Ensure that compose can access docker. Since the container is assumed to be running on the same machine
         //  as the docker daemon, just mapping the docker control socket is OK.
         // As there seems to be a problem with mapping to the /var/run directory in certain environments (e.g. CircleCI)
         //  we map the socket file outside of /var/run, as just /docker.sock
-        addFileSystemBind(getDockerSocketHostPath(), "/docker.sock", READ_WRITE);
+        addFileSystemBind("/" + DockerClientFactory.instance().getRemoteDockerUnixSocketPath(), "/docker.sock", READ_WRITE);
         addEnv("DOCKER_HOST", "unix:///docker.sock");
         setStartupCheckStrategy(new IndefiniteWaitOneShotStartupCheckStrategy());
         setWorkingDirectory(containerPwd);
     }
 
-    private String getDockerSocketHostPath() {
-        return SystemUtils.IS_OS_WINDOWS
-            ? "/" + DOCKER_SOCKET_PATH
-            : DOCKER_SOCKET_PATH;
-    }
-
     @Override
     public void invoke() {
         super.start();
@@ -681,16 +676,36 @@ public DockerCompose withEnv(Map<String, String> env) {
         return this;
     }
 
+    @VisibleForTesting
+    static boolean executableExists() {
+        return CommandLine.executableExists(COMPOSE_EXECUTABLE);
+    }
+
     @Override
     public void invoke() {
         // bail out early
-        if (!CommandLine.executableExists(COMPOSE_EXECUTABLE)) {
+        if (!executableExists()) {
             throw new ContainerLaunchException("Local Docker Compose not found. Is " + COMPOSE_EXECUTABLE + " on the PATH?");
         }
 
         final Map<String, String> environment = Maps.newHashMap(env);
         environment.put(ENV_PROJECT_NAME, identifier);
 
+        String dockerHost = System.getenv("DOCKER_HOST");
+        if (dockerHost == null) {
+            TransportConfig transportConfig = DockerClientFactory.instance().getTransportConfig();
+            SSLConfig sslConfig = transportConfig.getSslConfig();
+            if (sslConfig != null) {
+                if (sslConfig instanceof LocalDirectorySSLConfig) {
+                    environment.put("DOCKER_CERT_PATH", ((LocalDirectorySSLConfig) sslConfig).getDockerCertPath());
+                    environment.put("DOCKER_TLS_VERIFY", "true");
+                } else {
+                    logger().warn("Couldn't set DOCKER_CERT_PATH. `sslConfig` is present but it's not LocalDirectorySSLConfig.");
+                }
+            }
+            dockerHost = transportConfig.getDockerHost().toString();
+        }
+        environment.put("DOCKER_HOST", dockerHost);
 
         final Stream<String> absoluteDockerComposeFilePaths = composeFiles.stream()
             .map(File::getAbsolutePath)

core/src/main/java/org/testcontainers/containers/FixedHostPortGenericContainer.java

@@ -12,6 +12,11 @@
  * not available - which could manifest as flaky or unstable tests.</p>
  */
 public class FixedHostPortGenericContainer<SELF extends FixedHostPortGenericContainer<SELF>> extends GenericContainer<SELF> {
+
+    /**
+     * @deprecated it is highly recommended that {@link FixedHostPortGenericContainer} not be used, as it risks port conflicts.
+     */
+    @Deprecated
     public FixedHostPortGenericContainer(@NotNull String dockerImageName) {
         super(dockerImageName);
     }

core/src/main/java/org/testcontainers/containers/GenericContainer.java

@@ -12,7 +12,6 @@
 import com.github.dockerjava.api.model.ContainerNetwork;
 import com.github.dockerjava.api.model.ExposedPort;
 import com.github.dockerjava.api.model.HostConfig;
-import com.github.dockerjava.api.model.Info;
 import com.github.dockerjava.api.model.Link;
 import com.github.dockerjava.api.model.PortBinding;
 import com.github.dockerjava.api.model.Volume;
@@ -55,6 +54,7 @@
 import org.testcontainers.lifecycle.TestDescription;
 import org.testcontainers.lifecycle.TestLifecycleAware;
 import org.testcontainers.utility.Base58;
+import org.testcontainers.utility.DockerImageName;
 import org.testcontainers.utility.DockerLoggerFactory;
 import org.testcontainers.utility.DockerMachineClient;
 import org.testcontainers.utility.MountableFile;
@@ -79,6 +79,7 @@
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
@@ -180,7 +181,8 @@ public class GenericContainer<SELF extends GenericContainer<SELF>>
     @Nullable
     private Long shmSize;
 
-    private Map<MountableFile, String> copyToFileContainerPathMap = new HashMap<>();
+    // Maintain order in which entries are added, as earlier target location may be a prefix of a later location.
+    private Map<MountableFile, String> copyToFileContainerPathMap = new LinkedHashMap<>();
 
     protected final Set<Startable> dependencies = new HashSet<>();
 
@@ -225,10 +227,27 @@ public class GenericContainer<SELF extends GenericContainer<SELF>>
     @Setter(AccessLevel.NONE)
     private boolean shouldBeReused = false;
 
+
+    public GenericContainer(@NonNull final DockerImageName dockerImageName) {
+        this.image = new RemoteDockerImage(dockerImageName);
+    }
+
+    public GenericContainer(@NonNull final RemoteDockerImage image) {
+        this.image = image;
+    }
+
+    /**
+     * @deprecated use {@link GenericContainer(DockerImageName)} instead
+     */
+    @Deprecated
     public GenericContainer() {
         this(TestcontainersConfiguration.getInstance().getTinyImage());
     }
 
+    /**
+     * @deprecated use {@link GenericContainer(DockerImageName)} instead
+     */
+    @Deprecated
     public GenericContainer(@NonNull final String dockerImageName) {
         this.setDockerImageName(dockerImageName);
     }

core/src/main/java/org/testcontainers/containers/PortForwardingContainer.java

@@ -19,7 +19,7 @@
 public enum PortForwardingContainer {
     INSTANCE;
 
-    private GenericContainer container;
+    private GenericContainer<?> container;
 
     private final Set<Entry<Integer, Integer>> exposedPorts = Collections.newSetFromMap(new ConcurrentHashMap<>());
 
@@ -29,7 +29,7 @@ public enum PortForwardingContainer {
     @SneakyThrows
     private Connection createSSHSession() {
         String password = UUID.randomUUID().toString();
-        container = new GenericContainer<>(TestcontainersConfiguration.getInstance().getSSHdImage())
+        container = new GenericContainer<>(TestcontainersConfiguration.getInstance().getSSHdDockerImageName())
             .withExposedPorts(22)
             .withEnv("PASSWORD", password)
             .withCommand(

core/src/main/java/org/testcontainers/containers/SocatContainer.java

@@ -1,6 +1,7 @@
 package org.testcontainers.containers;
 
 import org.testcontainers.utility.Base58;
+import org.testcontainers.utility.DockerImageName;
 import org.testcontainers.utility.TestcontainersConfiguration;
 
 import java.util.HashMap;
@@ -16,7 +17,11 @@ public class SocatContainer extends GenericContainer<SocatContainer> {
     private final Map<Integer, String> targets = new HashMap<>();
 
     public SocatContainer() {
-        super(TestcontainersConfiguration.getInstance().getSocatContainerImage());
+        this(TestcontainersConfiguration.getInstance().getSocatDockerImageName());
+    }
+
+    public SocatContainer(final DockerImageName dockerImageName) {
+        super(dockerImageName);
         withCreateContainerCmdModifier(it -> it.withEntrypoint("/bin/sh"));
         withCreateContainerCmdModifier(it -> it.withName("testcontainers-socat-" + Base58.randomString(8)));
     }
@@ -39,4 +44,4 @@ protected void configure() {
                         .collect(Collectors.joining(" & "))
         );
     }
-}
+}

core/src/main/java/org/testcontainers/containers/VncRecordingContainer.java

@@ -52,7 +52,7 @@ public VncRecordingContainer(@NonNull GenericContainer<?> targetContainer) {
      * Create a sidekick container and attach it to another container. The VNC output of that container will be recorded.
      */
     public VncRecordingContainer(@NonNull Network network, @NonNull String targetNetworkAlias) throws IllegalStateException {
-        super(TestcontainersConfiguration.getInstance().getVncRecordedContainerImage());
+        super(TestcontainersConfiguration.getInstance().getVncDockerImageName());
 
         this.targetNetworkAlias = targetNetworkAlias;
         withNetwork(network);

core/src/main/java/org/testcontainers/dockerclient/AuthDelegatingDockerClientConfig.java

@@ -40,7 +40,7 @@ public AuthConfig effectiveAuthConfig(String imageName) {
         }
 
         // try and obtain more accurate auth config using our resolution
-        final DockerImageName parsed = new DockerImageName(imageName);
+        final DockerImageName parsed = DockerImageName.parse(imageName);
         final AuthConfig effectiveAuthConfig = RegistryAuthLocator.instance()
             .lookupAuthConfig(parsed, fallbackAuthConfig);