Merge pull request #76 from aozarov/temp

Replacing gcd.sh as a local resource with a remote version

Author: ludoch

gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java

@@ -20,24 +20,33 @@
 
 import com.google.common.base.Strings;
 
+import java.io.BufferedInputStream;
 import java.io.BufferedOutputStream;
 import java.io.BufferedReader;
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.FileReader;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.OutputStream;
+import java.math.BigInteger;
 import java.net.HttpURLConnection;
+import java.net.MalformedURLException;
 import java.net.URL;
+import java.nio.channels.Channels;
+import java.nio.channels.ReadableByteChannel;
 import java.nio.file.FileVisitResult;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.SimpleFileVisitor;
 import java.nio.file.attribute.BasicFileAttributes;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.Locale;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
 
@@ -52,8 +61,18 @@ public class LocalGcdHelper {
 
   public static final String DEFAULT_PROJECT_ID = "projectid1";
   public static final int PORT = 8080;
-  private static final String GCD = "gcd-head";
-  private static final String GCD_LOC = '/' + GCD + ".zip";
+  private static final String GCD = "gcd-v1beta2-rev1-2.1.2b";
+  private static final String GCD_FILENAME = GCD + ".zip";
+  private static final String MD5_CHECKSUM = "d84384cdfa8658e1204f4f8be51300e8";
+  private static final URL GCD_URL;
+
+  static {
+    try {
+      GCD_URL = new URL("http://storage.googleapis.com/gcd/tools/" + GCD_FILENAME);
+    } catch (MalformedURLException e) {
+      throw new RuntimeException(e);
+    }
+  }
 
   private static class ProcessStreamReader extends Thread {
 
@@ -101,13 +120,32 @@ public LocalGcdHelper(String projectId) {
     this.projectId = projectId;
   }
 
+  /**
+   * Starts the local datastore for the specific project.
+   *
+   * This will unzip the gcd tool, create the project and start it.
+   * All content is written to a temporary directory that will be deleted when
+   * {@link #stop()} is called or when the program terminates) to make sure that no left-over
+   * data from prior runs is used.
+   */
   public void start() throws IOException, InterruptedException {
+    // send a quick request in case we have a hanging process from a previous run
     sendQuitRequest();
+    // Each run is associated with its own folder that is deleted once test completes.
     gcdPath = Files.createTempDirectory("gcd");
     File gcdFolder = gcdPath.toFile();
     gcdFolder.deleteOnExit();
 
-    try (ZipInputStream zipIn = new ZipInputStream(getClass().getResourceAsStream(GCD_LOC))) {
+    // check if we already have a local copy of the gcd utility and download it if not.
+    File gcdZipFile = new File(System.getProperty("java.io.tmpdir"), GCD_FILENAME);
+    if (!gcdZipFile.exists() || !MD5_CHECKSUM.equals(md5(gcdZipFile))) {
+      ReadableByteChannel rbc = Channels.newChannel(GCD_URL.openStream());
+      FileOutputStream fos = new FileOutputStream(gcdZipFile);
+      fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
+      fos.close();
+    }
+    // unzip the gcd
+    try (ZipInputStream zipIn = new ZipInputStream(new FileInputStream(gcdZipFile))) {
       ZipEntry entry = zipIn.getNextEntry();
       while (entry != null) {
         File filePath = new File(gcdFolder, entry.getName());
@@ -120,27 +158,60 @@ public void start() throws IOException, InterruptedException {
         entry = zipIn.getNextEntry();
       }
     }
-
+    // cleanup any possible data for the same project
     File datasetFolder = new File(gcdFolder, GCD + '/' + projectId);
     deleteRecurse(datasetFolder.toPath());
 
-    // TODO: if System.getProperty("os.name").startsWith("Windows") use cmd.exe /c and gcd.cmd
-    Process temp = new ProcessBuilder()
-        .redirectErrorStream(true)
-        .directory(new File(gcdFolder, GCD))
-        .redirectOutput(new File("/dev/null"))
-        .command("bash", "gcd.sh", "create", "-d", projectId, projectId)
-        .start();
+    // create the datastore for the project
+    ProcessBuilder processBuilder = new ProcessBuilder()
+        .redirectError(ProcessBuilder.Redirect.INHERIT)
+        .directory(new File(gcdFolder, GCD));
+    if (isWindows()) {
+      processBuilder.command("cmd", "/C", "gcd.cmd", "create", "-p", projectId, projectId);
+      processBuilder.redirectOutput(new File("NULL:"));
+    } else {
+      processBuilder.redirectOutput(new File("/dev/null"));
+      processBuilder.command("bash", "gcd.sh", "create", "-p", projectId, projectId);
+    }
+
+    Process temp = processBuilder.start();
     temp.waitFor();
 
-    temp = new ProcessBuilder()
+    // start the datastore for the project
+    processBuilder = new ProcessBuilder()
         .directory(new File(gcdFolder, GCD))
-        .redirectErrorStream(true)
-        .command("bash", "gcd.sh", "start", "--testing", "--allow_remote_shutdown", projectId)
-        .start();
+        .redirectErrorStream(true);
+    if (isWindows()) {
+      processBuilder.command("cmd", "/C", "gcd.cmd", "start", "--testing",
+          "--allow_remote_shutdown", projectId);
+    } else {
+      processBuilder.command("bash", "gcd.sh", "start", "--testing", "--allow_remote_shutdown",
+          projectId);
+    }
+    temp = processBuilder.start();
     processReader = ProcessStreamReader.start(temp, "Dev App Server is now running");
   }
 
+  private static String md5(File gcdZipFile) throws IOException {
+    try {
+      MessageDigest md5 = MessageDigest.getInstance("MD5");
+      try (InputStream is = new BufferedInputStream(new FileInputStream(gcdZipFile))) {
+        byte[] bytes = new byte[4 * 1024 * 1024];
+        int len;
+        while ((len = is.read(bytes)) >= 0) {
+          md5.update(bytes, 0, len);
+        }
+      }
+      return String.format("%032x",new BigInteger(1, md5.digest()));
+    } catch (NoSuchAlgorithmException e) {
+      throw new IOException(e);
+    }
+  }
+
+  private static boolean isWindows() {
+    return System.getProperty("os.name").toLowerCase(Locale.ENGLISH).indexOf("windows") > -1;
+  }
+
   private static void extractFile(ZipInputStream zipIn, File filePath) throws IOException {
     try (BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(filePath))) {
       byte[] bytesIn = new byte[1024];