Run CI on windows

.github/workflows/gradle.yml

@@ -1,13 +1,17 @@
 name: Airbyte CI
 
+
 on:
   schedule:
     - cron: '0 */6 * * *'
   push:
 
 jobs:
   launch_integration_tests:
-    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        runner: [ windows-latest, ubuntu-latest ]
+    runs-on: ${{ matrix.runner }}
     if: github.ref == 'refs/heads/master'
     steps:
       - name: Checkout Airbyte
@@ -17,7 +21,10 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.SLASH_COMMAND_PAT }}
   build:
-    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        runner: [ windows-latest, ubuntu-latest ]
+    runs-on: ${{ matrix.runner }}
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v2

airbyte-commons/build.gradle

@@ -3,5 +3,5 @@ plugins {
 }
 
 dependencies {
-    testImplementation 'org.apache.commons:commons-lang3:3.11'
+    implementation 'org.apache.commons:commons-lang3:3.11'
 }

airbyte-commons/src/main/java/io/airbyte/commons/os/OsSupport.java

@@ -0,0 +1,66 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2020 Airbyte
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package io.airbyte.commons.os;
+
+import com.google.common.collect.Lists;
+import java.util.ArrayList;
+import java.util.List;
+
+public class OsSupport {
+
+  /**
+   * Formats the input command
+   *
+   * This helper is typically only necessary when calling external bash scripts.
+   *
+   * @param cmdParts
+   * @return
+   */
+  public static String[] formatCmd(String... cmdParts) {
+    return formatCmd(new OsUtils(), cmdParts);
+  }
+
+  public static String[] formatCmd(OsUtils osUtils, String... cmdParts) {
+    if (cmdParts.length == 0){
+      return cmdParts;
+    }
+
+    if (!cmdParts[0].endsWith(".sh")){
+      // We need to apply windows formatting only for shell scripts since Windows doesn't support .sh files by default
+      // Other commands like `docker` should work fine
+      return cmdParts;
+    }
+
+    List<String> formattedParts = new ArrayList<>();
+    if (osUtils.isWindows()) {
+      formattedParts.addAll(Lists.newArrayList("cmd", "/c"));
+    }
+
+    formattedParts.addAll(List.of(cmdParts));
+
+    return formattedParts.toArray(new String[] {});
+  }
+
+}

airbyte-commons/src/main/java/io/airbyte/commons/os/OsUtils.java

@@ -0,0 +1,35 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2020 Airbyte
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package io.airbyte.commons.os;
+
+import org.apache.commons.lang3.SystemUtils;
+
+public class OsUtils {
+
+  public boolean isWindows() {
+    return SystemUtils.IS_OS_WINDOWS;
+  }
+
+}

airbyte-commons/src/main/java/io/airbyte/commons/resources/MoreResources.java

@@ -75,12 +75,12 @@ public static Stream<Path> listResources(Class<?> klass, String name) throws IOE
 
   @SuppressWarnings("UnstableApiUsage")
   public static void writeResource(String filename, String contents) {
-    final Path source = Paths.get(Resources.getResource("").getPath());
     try {
+      final Path source = Paths.get(Resources.getResource("").toURI());
       Files.deleteIfExists(source.resolve(filename));
       Files.createFile(source.resolve(filename));
-      IOs.writeFile(Path.of(Resources.getResource(filename).getPath()), contents);
-    } catch (IOException e) {
+      IOs.writeFile(Paths.get(Resources.getResource(filename).toURI()), contents);
+    } catch (Exception e) {
       throw new RuntimeException(e);
     }
   }

airbyte-commons/src/test/java/io/airbyte/commons/json/JsonsTest.java

@@ -234,8 +234,8 @@ void testKeys() {
   void testToPrettyString() {
     final JsonNode jsonNode = Jsons.jsonNode(ImmutableMap.of("test", "abc"));
     final String expectedOutput = ""
-        + "{\n"
-        + "  \"test\": \"abc\"\n"
+        + "{" + System.lineSeparator()
+        + "  \"test\": \"abc\"" + System.lineSeparator()
         + "}\n";
     assertEquals(expectedOutput, Jsons.toPrettyString(jsonNode));
   }

airbyte-commons/src/test/java/io/airbyte/commons/resources/MoreResourcesTest.java

@@ -36,8 +36,8 @@ class MoreResourcesTest {
 
   @Test
   void testResourceRead() throws IOException {
-    assertEquals("content1\n", MoreResources.readResource("resource_test"));
-    assertEquals("content2\n", MoreResources.readResource("subdir/resource_test_sub"));
+    assertEquals("content1" + System.lineSeparator(), MoreResources.readResource("resource_test"));
+    assertEquals("content2" + System.lineSeparator(), MoreResources.readResource("subdir/resource_test_sub"));
 
     assertThrows(IllegalArgumentException.class, () -> MoreResources.readResource("invalid"));
   }

airbyte-config/models/src/test/java/io/airbyte/config/EnvConfigsTest.java

@@ -26,6 +26,7 @@
 
 import static org.mockito.Mockito.when;
 
+import java.io.File;
 import java.nio.file.Paths;
 import java.util.function.Function;
 import org.junit.jupiter.api.Assertions;
@@ -126,7 +127,8 @@ void testGetDatabaseUrl() {
   void testGetWorkspaceDockerMount() {
     when(function.apply(EnvConfigs.WORKSPACE_DOCKER_MOUNT)).thenReturn(null);
     when(function.apply(EnvConfigs.WORKSPACE_ROOT)).thenReturn("abc/def");
-    Assertions.assertEquals("abc/def", config.getWorkspaceDockerMount());
+
+    Assertions.assertEquals("abc" + File.separator + "def", config.getWorkspaceDockerMount());
 
     when(function.apply(EnvConfigs.WORKSPACE_DOCKER_MOUNT)).thenReturn("root");
     when(function.apply(EnvConfigs.WORKSPACE_ROOT)).thenReturn(null);
@@ -141,7 +143,7 @@ void testGetWorkspaceDockerMount() {
   void testGetLocalDockerMount() {
     when(function.apply(EnvConfigs.LOCAL_DOCKER_MOUNT)).thenReturn(null);
     when(function.apply(EnvConfigs.LOCAL_ROOT)).thenReturn("abc/def");
-    Assertions.assertEquals("abc/def", config.getLocalDockerMount());
+    Assertions.assertEquals("abc" + File.separator + "def", config.getLocalDockerMount());
 
     when(function.apply(EnvConfigs.LOCAL_DOCKER_MOUNT)).thenReturn("root");
     when(function.apply(EnvConfigs.LOCAL_ROOT)).thenReturn(null);

airbyte-integrations/bases/airbyte-protocol/build.gradle

@@ -9,7 +9,7 @@ airbytePython {
 
 task generateProtocolClassFilesWithoutLicense(type: Exec) {
     environment 'ROOT_DIR', rootDir.absolutePath
-    commandLine 'bin/generate-protocol-files.sh'
+    commandLine CrossPlatformSupport.formatCmd('bin/generate-protocol-files.sh')
     dependsOn ':tools:code-generator:airbyteDocker'
 }
 

airbyte-scheduler/src/test/java/io/airbyte/scheduler/JobLogsTest.java

@@ -26,14 +26,15 @@
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
+import java.nio.file.Path;
 import org.junit.jupiter.api.Test;
 
 class JobLogsTest {
 
   @Test
   public void testGetLogDirectory() {
     final String actual = JobLogs.getLogDirectory("blah");
-    final String expected = "logs/jobs/blah";
+    final String expected = Path.of("logs/jobs/blah").toString();
     assertEquals(expected, actual);
   }
 

airbyte-server/src/main/java/io/airbyte/server/handlers/DebugInfoHandler.java

@@ -29,6 +29,7 @@
 import com.google.common.collect.Lists;
 import io.airbyte.api.model.DebugRead;
 import io.airbyte.commons.docker.DockerUtils;
+import io.airbyte.commons.os.OsSupport;
 import io.airbyte.config.persistence.ConfigRepository;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
@@ -74,22 +75,24 @@ private static List<Map<String, String>> getRunningCoreImages() {
           "inspect",
           "--format='{{.Image}} {{.Config.Image}}'");
 
-      inspectCommand.addAll(Lists.newArrayList(runningAirbyteContainers.split("\n")));
+      inspectCommand.addAll(Lists.newArrayList(runningAirbyteContainers.split(System.lineSeparator())));
 
       final String output = runAndGetOutput(inspectCommand).replaceAll("'", "");
 
-      final List<String> coreOutput = Lists.newArrayList(output.split("\n"));
+      final List<String> coreOutput = Lists.newArrayList(output.split(System.lineSeparator()));
 
-      return coreOutput.stream().map(entry -> {
-        final String[] elements = entry.split(" ");
-        final String shortHash = getShortHash(elements[0]);
-        final String taggedImage = elements[1];
+      return coreOutput.stream()
+          .filter(s -> !s.equals(""))
+          .map(entry -> {
+            final String[] elements = entry.split("");
+            final String shortHash = getShortHash(elements[0]);
+            final String taggedImage = elements[1];
 
-        final Map<String, String> result = new HashMap<>();
-        result.put("hash", shortHash);
-        result.put("image", taggedImage);
-        return result;
-      }).collect(toList());
+            final Map<String, String> result = new HashMap<>();
+            result.put("hash", shortHash);
+            result.put("image", taggedImage);
+            return result;
+          }).collect(toList());
     } catch (Exception e) {
       throw new RuntimeException(e);
     }
@@ -122,7 +125,11 @@ private List<Map<String, String>> getIntegrationImages() {
   }
 
   protected static String runAndGetOutput(List<String> cmd) throws IOException, InterruptedException {
-    final ProcessBuilder processBuilder = new ProcessBuilder(cmd);
+    return runAndGetOutput(cmd.toArray(new String[] {}));
+  }
+
+  protected static String runAndGetOutput(String... cmd) throws IOException, InterruptedException {
+    final ProcessBuilder processBuilder = new ProcessBuilder(OsSupport.formatCmd(cmd));
     final Process process = processBuilder.start();
     process.waitFor();
 

airbyte-server/src/test/java/io/airbyte/server/handlers/DebugInfoHandlerTest.java

@@ -56,8 +56,8 @@ public void testNoFailures() throws ConfigNotFoundException, IOException, JsonVa
 
   @Test
   public void testRunAndGetOutput() throws IOException, InterruptedException {
-    final String expected = "hi";
-    final String actual = DebugInfoHandler.runAndGetOutput(Lists.newArrayList("echo", "-n", "hi"));
+    final String expected = "hi" + System.lineSeparator();
+    final String actual = DebugInfoHandler.runAndGetOutput("echo", "hi");
     assertEquals(expected, actual);
   }
 

airbyte-workers/src/main/java/io/airbyte/workers/process/DockerProcessBuilderFactory.java

@@ -29,6 +29,7 @@
 import com.google.common.collect.Lists;
 import io.airbyte.commons.io.IOs;
 import io.airbyte.commons.io.LineGobbler;
+import io.airbyte.commons.os.OsSupport;
 import io.airbyte.commons.resources.MoreResources;
 import io.airbyte.workers.WorkerException;
 import java.io.IOException;
@@ -113,7 +114,7 @@ private Path rebasePath(final Path jobRoot) {
   @VisibleForTesting
   boolean checkImageExists(String imageName) {
     try {
-      final Process process = new ProcessBuilder(imageExistsScriptPath.toString(), imageName).start();
+      final Process process = new ProcessBuilder(OsSupport.formatCmd(imageExistsScriptPath.toString(), imageName)).start();
       LineGobbler.gobble(process.getErrorStream(), LOGGER::error);
       LineGobbler.gobble(process.getInputStream(), LOGGER::info);
 
@@ -127,3 +128,5 @@ boolean checkImageExists(String imageName) {
   }
 
 }
+
+

build.gradle

@@ -1,7 +1,13 @@
+import org.apache.tools.ant.taskdefs.condition.Os
+
+import java.util.stream.Stream
+import java.util.stream.StreamSupport
+
 plugins {
     id 'base'
     id 'pmd'
     id 'com.diffplug.spotless' version '5.7.0'
+    id "com.github.jlouns.cpe" version "0.5.0"
 }
 
 repositories {
@@ -191,13 +197,9 @@ subprojects {
     }
 }
 
-task composeBuild {
-    doFirst {
-        exec {
-            workingDir rootDir
-            commandLine 'docker-compose', '-f', 'docker-compose.build.yaml', 'build', '--parallel', '--quiet'
-        }
-    }
+task composeBuild(type: CrossPlatformExec) {
+    workingDir rootDir
+    commandLine 'docker-compose', '-f', 'docker-compose.build.yaml', 'build', '--parallel', '--quiet'
 }
 build.dependsOn(composeBuild)
 

buildSrc/src/main/groovy/CrossPlatformSupport.groovy

@@ -0,0 +1,19 @@
+import org.apache.tools.ant.taskdefs.condition.Os
+
+class CrossPlatformSupport {
+    /**
+     *
+     * @param args The full command to be executed split into array. This is typically the input to `commandLine` in
+     * the exec task.
+     */
+    static formatCmd(Object... cmdParts){
+        List<Object> formattedParts = []
+        if (Os.isFamily(Os.FAMILY_WINDOWS)) {
+            formattedParts.addAll(['cmd', '/c'])
+        }
+
+        formattedParts.addAll(cmdParts)
+
+        return formattedParts.toArray()
+    }
+}