Reorganize $DENO_DIR/deps folder and split by protocol

js/io.ts

@@ -101,7 +101,7 @@ export interface ReadWriteSeeker extends Reader, Writer, Seeker {}
 // https://golang.org/pkg/io/#Copy
 export async function copy(dst: Writer, src: Reader): Promise<number> {
   let n = 0;
-  const b = new Uint8Array(32*1024);
+  const b = new Uint8Array(32 * 1024);
   let gotEOF = false;
   while (gotEOF === false) {
     const result = await src.read(b);

src/deno_dir.rs

@@ -28,6 +28,9 @@ pub struct DenoDir {
   // This is where we cache compilation outputs. Example:
   // /Users/rld/.deno/gen/f39a473452321cacd7c346a870efb0e3e1264b43.js
   pub deps: PathBuf,
+  // This splits to http and https deps
+  pub deps_http: PathBuf,
+  pub deps_https: PathBuf,
   // If remote resources should be reloaded.
   reload: bool,
 }
@@ -49,19 +52,27 @@ impl DenoDir {
     };
     let gen = root.as_path().join("gen");
     let deps = root.as_path().join("deps");
+    let deps_http = deps.join("http");
+    let deps_https = deps.join("https");
 
     let deno_dir = DenoDir {
       root,
       gen,
       deps,
+      deps_http,
+      deps_https,
       reload,
     };
     deno_fs::mkdir(deno_dir.gen.as_ref(), 0o755)?;
     deno_fs::mkdir(deno_dir.deps.as_ref(), 0o755)?;
+    deno_fs::mkdir(deno_dir.deps_http.as_ref(), 0o755)?;
+    deno_fs::mkdir(deno_dir.deps_https.as_ref(), 0o755)?;
 
     debug!("root {}", deno_dir.root.display());
     debug!("gen {}", deno_dir.gen.display());
     debug!("deps {}", deno_dir.deps.display());
+    debug!("deps_http {}", deno_dir.deps_http.display());
+    debug!("deps_https {}", deno_dir.deps_https.display());
 
     Ok(deno_dir)
   }
@@ -225,13 +236,22 @@ impl DenoDir {
   fn src_file_to_url(self: &DenoDir, filename: &str) -> String {
     let filename_path = Path::new(filename);
     if filename_path.starts_with(&self.deps) {
-      let rest = filename_path.strip_prefix(&self.deps).unwrap();
+      let (rest, prefix) = if filename_path.starts_with(&self.deps_https) {
+        let rest = filename_path.strip_prefix(&self.deps_https).unwrap();
+        let prefix = "https://".to_string();
+        (rest, prefix)
+      } else {
+        // TODO(kevinkassimo): change this to support other protocols than http
+        let rest = filename_path.strip_prefix(&self.deps_http).unwrap();
+        let prefix = "http://".to_string();
+        (rest, prefix)
+      };
       // Windows doesn't support ":" in filenames, so we represent port using a
       // special string.
       // TODO(ry) This current implementation will break on a URL that has
       // the default port but contains "_PORT" in the path.
       let rest = rest.to_str().unwrap().replacen("_PORT", ":", 1);
-      "http://".to_string() + &rest
+      prefix + &rest
     } else {
       String::from(filename)
     }
@@ -279,10 +299,17 @@ impl DenoDir {
         module_name = p.clone();
         filename = p;
       }
+      "https" => {
+        module_name = j.to_string();
+        filename = deno_fs::normalize_path(
+          get_cache_filename(self.deps_https.as_path(), j).as_ref(),
+        )
+      }
+      // TODO(kevinkassimo): change this to support other protocols than http
       _ => {
         module_name = j.to_string();
         filename = deno_fs::normalize_path(
-          get_cache_filename(self.deps.as_path(), j).as_ref(),
+          get_cache_filename(self.deps_http.as_path(), j).as_ref(),
         )
       }
     }
@@ -480,7 +507,7 @@ fn test_src_file_to_url_1() {
   let (_temp_dir, deno_dir) = test_setup();
   assert_eq!("hello", deno_dir.src_file_to_url("hello"));
   assert_eq!("/hello", deno_dir.src_file_to_url("/hello"));
-  let x = deno_dir.deps.join("hello/world.txt");
+  let x = deno_dir.deps_http.join("hello/world.txt");
   assert_eq!(
     "http://hello/world.txt",
     deno_dir.src_file_to_url(x.to_str().unwrap())
@@ -490,13 +517,35 @@ fn test_src_file_to_url_1() {
 #[test]
 fn test_src_file_to_url_2() {
   let (_temp_dir, deno_dir) = test_setup();
-  let x = deno_dir.deps.join("localhost_PORT4545/world.txt");
+  assert_eq!("hello", deno_dir.src_file_to_url("hello"));
+  assert_eq!("/hello", deno_dir.src_file_to_url("/hello"));
+  let x = deno_dir.deps_https.join("hello/world.txt");
+  assert_eq!(
+    "https://hello/world.txt",
+    deno_dir.src_file_to_url(x.to_str().unwrap())
+  );
+}
+
+#[test]
+fn test_src_file_to_url_3() {
+  let (_temp_dir, deno_dir) = test_setup();
+  let x = deno_dir.deps_http.join("localhost_PORT4545/world.txt");
   assert_eq!(
     "http://localhost:4545/world.txt",
     deno_dir.src_file_to_url(x.to_str().unwrap())
   );
 }
 
+#[test]
+fn test_src_file_to_url_4() {
+  let (_temp_dir, deno_dir) = test_setup();
+  let x = deno_dir.deps_https.join("localhost_PORT4545/world.txt");
+  assert_eq!(
+    "https://localhost:4545/world.txt",
+    deno_dir.src_file_to_url(x.to_str().unwrap())
+  );
+}
+
 // https://github.com/denoland/deno/blob/golang/os_test.go#L16-L87
 #[test]
 fn test_resolve_module_1() {
@@ -556,7 +605,7 @@ fn test_resolve_module_2() {
     "http://localhost:4545/testdata/subdir/print_hello.ts";
   let expected_filename = deno_fs::normalize_path(
     deno_dir
-      .deps
+      .deps_http
       .join("localhost_PORT4545/testdata/subdir/print_hello.ts")
       .as_ref(),
   );
@@ -573,14 +622,14 @@ fn test_resolve_module_3() {
   let (_temp_dir, deno_dir) = test_setup();
 
   let module_specifier_ =
-    deno_dir.deps.join("unpkg.com/[email protected]/index.ts");
+    deno_dir.deps_http.join("unpkg.com/[email protected]/index.ts");
   let module_specifier = module_specifier_.to_str().unwrap();
   let containing_file = ".";
 
   let expected_module_name = "http://unpkg.com/[email protected]/index.ts";
   let expected_filename = deno_fs::normalize_path(
     deno_dir
-      .deps
+      .deps_http
       .join("unpkg.com/[email protected]/index.ts")
       .as_ref(),
   );
@@ -597,12 +646,17 @@ fn test_resolve_module_4() {
   let (_temp_dir, deno_dir) = test_setup();
 
   let module_specifier = "./util";
-  let containing_file_ = deno_dir.deps.join("unpkg.com/[email protected]/index.ts");
+  let containing_file_ =
+    deno_dir.deps_http.join("unpkg.com/[email protected]/index.ts");
   let containing_file = containing_file_.to_str().unwrap();
 
+  // http containing files -> load relative import with http
   let expected_module_name = "http://unpkg.com/[email protected]/util";
   let expected_filename = deno_fs::normalize_path(
-    deno_dir.deps.join("unpkg.com/[email protected]/util").as_ref(),
+    deno_dir
+      .deps_http
+      .join("unpkg.com/[email protected]/util")
+      .as_ref(),
   );
 
   let (module_name, filename) = deno_dir
@@ -616,12 +670,37 @@ fn test_resolve_module_4() {
 fn test_resolve_module_5() {
   let (_temp_dir, deno_dir) = test_setup();
 
+  let module_specifier = "./util";
+  let containing_file_ =
+    deno_dir.deps_https.join("unpkg.com/[email protected]/index.ts");
+  let containing_file = containing_file_.to_str().unwrap();
+
+  // https containing files -> load relative import with https
+  let expected_module_name = "https://unpkg.com/[email protected]/util";
+  let expected_filename = deno_fs::normalize_path(
+    deno_dir
+      .deps_https
+      .join("unpkg.com/[email protected]/util")
+      .as_ref(),
+  );
+
+  let (module_name, filename) = deno_dir
+    .resolve_module(module_specifier, containing_file)
+    .unwrap();
+  assert_eq!(module_name, expected_module_name);
+  assert_eq!(filename, expected_filename);
+}
+
+#[test]
+fn test_resolve_module_6() {
+  let (_temp_dir, deno_dir) = test_setup();
+
   let module_specifier = "http://localhost:4545/tests/subdir/mod2.ts";
   let containing_file = add_root!("/deno/tests/006_url_imports.ts");
   let expected_module_name = "http://localhost:4545/tests/subdir/mod2.ts";
   let expected_filename = deno_fs::normalize_path(
     deno_dir
-      .deps
+      .deps_http
       .join("localhost_PORT4545/tests/subdir/mod2.ts")
       .as_ref(),
   );

tools/unit_tests.py

@@ -4,6 +4,7 @@
 import subprocess
 import re
 
+
 def run_unit_test2(cmd):
     process = subprocess.Popen(
         cmd,
@@ -26,6 +27,7 @@ def run_unit_test2(cmd):
     if errcode != 0:
         sys.exit(errcode)
 
+
 def run_unit_test(deno_exe, permStr, flags=[]):
     cmd = [deno_exe, "--reload", "js/unit_tests.ts", permStr] + flags
     run_unit_test2(cmd)
@@ -48,9 +50,7 @@ def unit_tests(deno_exe):
     # These are not strictly unit tests for Deno, but for ts_library_builder.
     # They run under Node, but use the same //js/testing/ library.
     run_unit_test2([
-        "node",
-        "./node_modules/.bin/ts-node",
-        "--project",
+        "node", "./node_modules/.bin/ts-node", "--project",
         "tools/ts_library_builder/tsconfig.json",
         "tools/ts_library_builder/test.ts"
     ])