Image substitution (#3102)

* Refactor Testcontainers configuration to allow config by env var

* Add Image substitution mechanism

Builds upon #3021 and #3411:

* adds a pluggable image substitution mechanism using ServiceLoader, enabling users to perform custom substitution/auditing of images being used by their tests

* provides a default implementation that behaves similarly to legacy `TestcontainersConfiguration` approach (`testcontainers.properties`)

Notes:

* behaviour is similar but not quite identical to `TestcontainersConfiguration`: use of a configured custom image for, e.g. Kafka/Pulsar that does not have a tag specified causes the substitution to take effect for all usages. It seems very unlikely that people would be using a mix of the config file image overrides in some places _and_ specific images specified in code in others.

* Duplication of default image names in modules vs `TestcontainersConfiguration` class is intentional: specifying image overrides in `testcontainers.properties` should be removed in the future.

* ~Add log deprecation warnings when `testcontainers.properties` image overrides are used.~ Defer to a future release?

* Remove extraneous change

* Un-ignore docs example test by implementing a 'reversing' image name substitutor

* Use configuration, not service loader, to select an ImageNameSubstitutor

* Add check for order of config setting precedence

* Extract classpath scanner and support finding of multiple resources

* Introduce deterministic merging of classpath properties files

* Update docs

* Update docs

* Remove service loader reference

* Chain substitution through default and configured implementations

* Small tweaks following review

* Fix test compile error

* Add UnstableAPI annotation

* Move TestSpecificImageNameSubstitutor back to original package and remove duplicate use of default substitutor

Author: rnorth

core/src/main/java/org/testcontainers/DockerClientFactory.java

@@ -25,6 +25,8 @@
 import org.testcontainers.dockerclient.TransportConfig;
 import org.testcontainers.images.TimeLimitedLoggedPullImageResultCallback;
 import org.testcontainers.utility.ComparableVersion;
+import org.testcontainers.utility.DockerImageName;
+import org.testcontainers.utility.ImageNameSubstitutor;
 import org.testcontainers.utility.MountableFile;
 import org.testcontainers.utility.ResourceReaper;
 import org.testcontainers.utility.TestcontainersConfiguration;
@@ -61,7 +63,7 @@ public class DockerClientFactory {
             TESTCONTAINERS_SESSION_ID_LABEL, SESSION_ID
     );
 
-    private static final String TINY_IMAGE = TestcontainersConfiguration.getInstance().getTinyDockerImageName().asCanonicalNameString();
+    private static final DockerImageName TINY_IMAGE = DockerImageName.parse("alpine:3.5");
     private static DockerClientFactory instance;
 
     // Cached client configuration
@@ -343,8 +345,11 @@ public <T> T runInsideDocker(Consumer<CreateContainerCmd> createContainerCmdCons
     }
 
     private <T> T runInsideDocker(DockerClient client, Consumer<CreateContainerCmd> createContainerCmdConsumer, BiFunction<DockerClient, String, T> block) {
-        checkAndPullImage(client, TINY_IMAGE);
-        CreateContainerCmd createContainerCmd = client.createContainerCmd(TINY_IMAGE)
+
+        final String tinyImage = ImageNameSubstitutor.instance().apply(TINY_IMAGE).asCanonicalNameString();
+
+        checkAndPullImage(client, tinyImage);
+        CreateContainerCmd createContainerCmd = client.createContainerCmd(tinyImage)
                 .withLabels(DEFAULT_LABELS);
         createContainerCmdConsumer.accept(createContainerCmd);
         String id = createContainerCmd.exec().getId();

core/src/main/java/org/testcontainers/containers/DockerComposeContainer.java

@@ -29,11 +29,11 @@
 import org.testcontainers.utility.AuditLogger;
 import org.testcontainers.utility.Base58;
 import org.testcontainers.utility.CommandLine;
+import org.testcontainers.utility.DockerImageName;
 import org.testcontainers.utility.DockerLoggerFactory;
 import org.testcontainers.utility.LogUtils;
 import org.testcontainers.utility.MountableFile;
 import org.testcontainers.utility.ResourceReaper;
-import org.testcontainers.utility.TestcontainersConfiguration;
 import org.zeroturnaround.exec.InvalidExitValueException;
 import org.zeroturnaround.exec.ProcessExecutor;
 import org.zeroturnaround.exec.stream.slf4j.Slf4jStream;
@@ -608,10 +608,11 @@ interface DockerCompose {
 class ContainerisedDockerCompose extends GenericContainer<ContainerisedDockerCompose> implements DockerCompose {
 
     public static final char UNIX_PATH_SEPERATOR = ':';
+    public static final DockerImageName DEFAULT_IMAGE_NAME = DockerImageName.parse("docker/compose:1.24.1");
 
     public ContainerisedDockerCompose(List<File> composeFiles, String identifier) {
 
-        super(TestcontainersConfiguration.getInstance().getDockerComposeDockerImageName());
+        super(DEFAULT_IMAGE_NAME);
         addEnv(ENV_PROJECT_NAME, identifier);
 
         // Map the docker compose file into the container

core/src/main/java/org/testcontainers/containers/GenericContainer.java

@@ -239,13 +239,9 @@ public GenericContainer(@NonNull final RemoteDockerImage image) {
      */
     @Deprecated
     public GenericContainer() {
-        this(TestcontainersConfiguration.getInstance().getTinyDockerImageName().asCanonicalNameString());
+        this(TestcontainersConfiguration.getInstance().getTinyImage());
     }
 
-    /**
-     * @deprecated use {@link GenericContainer(DockerImageName)} instead
-     */
-    @Deprecated
     public GenericContainer(@NonNull final String dockerImageName) {
         this.setDockerImageName(dockerImageName);
     }

core/src/main/java/org/testcontainers/containers/PortForwardingContainer.java

@@ -5,15 +5,15 @@
 import lombok.AccessLevel;
 import lombok.Getter;
 import lombok.SneakyThrows;
-import org.testcontainers.utility.TestcontainersConfiguration;
+import org.testcontainers.utility.DockerImageName;
 
 import java.time.Duration;
 import java.util.AbstractMap;
 import java.util.Collections;
+import java.util.Map.Entry;
 import java.util.Optional;
 import java.util.Set;
 import java.util.UUID;
-import java.util.Map.Entry;
 import java.util.concurrent.ConcurrentHashMap;
 
 public enum PortForwardingContainer {
@@ -29,7 +29,7 @@ public enum PortForwardingContainer {
     @SneakyThrows
     private Connection createSSHSession() {
         String password = UUID.randomUUID().toString();
-        container = new GenericContainer<>(TestcontainersConfiguration.getInstance().getSSHdDockerImageName())
+        container = new GenericContainer<>(DockerImageName.parse("testcontainers/sshd:1.0.0"))
             .withExposedPorts(22)
             .withEnv("PASSWORD", password)
             .withCommand(

core/src/main/java/org/testcontainers/containers/SocatContainer.java

@@ -1,12 +1,10 @@
 package org.testcontainers.containers;
 
-import org.testcontainers.utility.Base58;
-import org.testcontainers.utility.DockerImageName;
-import org.testcontainers.utility.TestcontainersConfiguration;
-
 import java.util.HashMap;
 import java.util.Map;
 import java.util.stream.Collectors;
+import org.testcontainers.utility.Base58;
+import org.testcontainers.utility.DockerImageName;
 
 /**
  * A socat container is used as a TCP proxy, enabling any TCP port of another container to be exposed
@@ -17,7 +15,7 @@ public class SocatContainer extends GenericContainer<SocatContainer> {
     private final Map<Integer, String> targets = new HashMap<>();
 
     public SocatContainer() {
-        this(TestcontainersConfiguration.getInstance().getSocatDockerImageName());
+        this(DockerImageName.parse("alpine/socat:1.7.3.4-r0"));
     }
 
     public SocatContainer(final DockerImageName dockerImageName) {

core/src/main/java/org/testcontainers/containers/VncRecordingContainer.java

@@ -6,7 +6,7 @@
 import lombok.ToString;
 import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
 import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy;
-import org.testcontainers.utility.TestcontainersConfiguration;
+import org.testcontainers.utility.DockerImageName;
 
 import java.io.File;
 import java.io.InputStream;
@@ -52,7 +52,7 @@ public VncRecordingContainer(@NonNull GenericContainer<?> targetContainer) {
      * Create a sidekick container and attach it to another container. The VNC output of that container will be recorded.
      */
     public VncRecordingContainer(@NonNull Network network, @NonNull String targetNetworkAlias) throws IllegalStateException {
-        super(TestcontainersConfiguration.getInstance().getVncDockerImageName());
+        super(DockerImageName.parse("testcontainers/vnc-recorder:1.1.0"));
 
         this.targetNetworkAlias = targetNetworkAlias;
         withNetwork(network);

core/src/main/java/org/testcontainers/dockerclient/DockerClientProviderStrategy.java

@@ -171,7 +171,7 @@ public static DockerClientProviderStrategy getFirstValidStrategy(List<DockerClie
                         }
 
                         if (strategy.isPersistable()) {
-                            TestcontainersConfiguration.getInstance().updateGlobalConfig("docker.client.strategy", strategy.getClass().getName());
+                            TestcontainersConfiguration.getInstance().updateUserConfig("docker.client.strategy", strategy.getClass().getName());
                         }
 
                         return Stream.of(strategy);

core/src/main/java/org/testcontainers/images/RemoteDockerImage.java

@@ -15,6 +15,7 @@
 import org.testcontainers.containers.ContainerFetchException;
 import org.testcontainers.utility.DockerImageName;
 import org.testcontainers.utility.DockerLoggerFactory;
+import org.testcontainers.utility.ImageNameSubstitutor;
 import org.testcontainers.utility.LazyFuture;
 
 import java.time.Duration;
@@ -44,12 +45,12 @@ public RemoteDockerImage(DockerImageName dockerImageName) {
 
     @Deprecated
     public RemoteDockerImage(String dockerImageName) {
-        this.imageNameFuture = CompletableFuture.completedFuture(DockerImageName.parse(dockerImageName));
+        this(DockerImageName.parse(dockerImageName));
     }
 
     @Deprecated
     public RemoteDockerImage(@NonNull String repository, @NonNull String tag) {
-        this.imageNameFuture = CompletableFuture.completedFuture(DockerImageName.parse(repository).withTag(tag));
+        this(DockerImageName.parse(repository).withTag(tag));
     }
 
     public RemoteDockerImage(@NonNull Future<String> imageFuture) {
@@ -100,7 +101,10 @@ protected final String resolve() {
     }
 
     private DockerImageName getImageName() throws InterruptedException, ExecutionException {
-        return imageNameFuture.get();
+        final DockerImageName specifiedImageName = imageNameFuture.get();
+
+        // Allow the image name to be substituted
+        return ImageNameSubstitutor.instance().apply(specifiedImageName);
     }
 
     @ToString.Include(name = "imageName", rank = 1)

core/src/main/java/org/testcontainers/utility/ClasspathScanner.java

@@ -0,0 +1,59 @@
+package org.testcontainers.utility;
+
+import lombok.extern.slf4j.Slf4j;
+import org.jetbrains.annotations.Nullable;
+import org.jetbrains.annotations.VisibleForTesting;
+
+import java.net.URL;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Objects;
+import java.util.stream.Stream;
+
+/**
+ * Utility for identifying resource files on classloaders.
+ */
+@Slf4j
+class ClasspathScanner {
+
+    @VisibleForTesting
+    static Stream<URL> scanFor(final String name, ClassLoader... classLoaders) {
+        return Stream
+            .of(classLoaders)
+            .flatMap(classLoader -> getAllPropertyFilesOnClassloader(classLoader, name))
+            .filter(Objects::nonNull)
+            .sorted(
+                Comparator
+                    .comparing(ClasspathScanner::filesFileSchemeFirst) // resolve 'local' files first
+                    .thenComparing(URL::toString) // sort alphabetically for the sake of determinism
+            )
+            .distinct();
+    }
+
+    private static Integer filesFileSchemeFirst(final URL t) {
+        return t.getProtocol().equals("file") ? 0 : 1;
+    }
+
+    /**
+     * @param name the resource name to search for
+     * @return distinct, ordered stream of resources found by searching this class' classloader and the current thread's
+     * context classloader. Results are currently alphabetically sorted.
+     */
+    static Stream<URL> scanFor(final String name) {
+        return scanFor(
+            name,
+            ClasspathScanner.class.getClassLoader(),
+            Thread.currentThread().getContextClassLoader()
+        );
+    }
+
+    @Nullable
+    private static Stream<URL> getAllPropertyFilesOnClassloader(final ClassLoader it, final String s) {
+        try {
+            return Collections.list(it.getResources(s)).stream();
+        } catch (Exception e) {
+            log.error("Unable to read configuration from classloader {} - this is probably a bug", it, e);
+            return Stream.empty();
+        }
+    }
+}

core/src/main/java/org/testcontainers/utility/ConfigurationFileImageNameSubstitutor.java

@@ -0,0 +1,44 @@
+package org.testcontainers.utility;
+
+import com.google.common.annotations.VisibleForTesting;
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * {@link ImageNameSubstitutor} which takes replacement image names from configuration.
+ * See {@link TestcontainersConfiguration} for the subset of image names which can be substituted using this mechanism.
+ */
+@Slf4j
+final class ConfigurationFileImageNameSubstitutor extends ImageNameSubstitutor {
+
+    private final TestcontainersConfiguration configuration;
+
+    public ConfigurationFileImageNameSubstitutor() {
+        this(TestcontainersConfiguration.getInstance());
+    }
+
+    @VisibleForTesting
+    ConfigurationFileImageNameSubstitutor(TestcontainersConfiguration configuration) {
+        this.configuration = configuration;
+    }
+
+    @Override
+    public DockerImageName apply(final DockerImageName original) {
+        final DockerImageName result = configuration
+            .getConfiguredSubstituteImage(original)
+            .asCompatibleSubstituteFor(original);
+
+        if (!result.equals(original)) {
+            log.warn("Image name {} was substituted by configuration to {}. This approach is deprecated and will be removed in the future",
+                original,
+                result
+            );
+        }
+
+        return result;
+    }
+
+    @Override
+    protected String getDescription() {
+        return getClass().getSimpleName();
+    }
+}

core/src/main/java/org/testcontainers/utility/DefaultImageNameSubstitutor.java

@@ -0,0 +1,35 @@
+package org.testcontainers.utility;
+
+import com.google.common.annotations.VisibleForTesting;
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * Testcontainers' default implementation of {@link ImageNameSubstitutor}.
+ * Delegates to {@link ConfigurationFileImageNameSubstitutor}.
+ */
+@Slf4j
+final class DefaultImageNameSubstitutor extends ImageNameSubstitutor {
+
+    private final ConfigurationFileImageNameSubstitutor configurationFileImageNameSubstitutor;
+
+    public DefaultImageNameSubstitutor() {
+        configurationFileImageNameSubstitutor = new ConfigurationFileImageNameSubstitutor();
+    }
+
+    @VisibleForTesting
+    DefaultImageNameSubstitutor(
+        final ConfigurationFileImageNameSubstitutor configurationFileImageNameSubstitutor
+    ) {
+        this.configurationFileImageNameSubstitutor = configurationFileImageNameSubstitutor;
+    }
+
+    @Override
+    public DockerImageName apply(final DockerImageName original) {
+        return configurationFileImageNameSubstitutor.apply(original);
+    }
+
+    @Override
+    protected String getDescription() {
+        return "DefaultImageNameSubstitutor (" + configurationFileImageNameSubstitutor.getDescription() + ")";
+    }
+}