refactor: snapshotting of runtime/ and cli/ (#21430)

This commit removes some of the technical debt related 
to snapshotting JS code:
- "cli/ops/mod.rs" and "cli/build.rs" no longer define "cli" extension
which was not required anymore
- Cargo features for "deno_runtime" crate have been unified in
"cli/Cargo.toml"
- "cli/build.rs" uses "deno_runtime::snapshot::create_runtime_snapshot"
API
instead of copy-pasting the code
- "cli/js/99_main.js" was completely removed as it's not necessary
anymore

Towards #21137

Author: bartlomieju

cli/Cargo.toml

@@ -39,7 +39,10 @@ __runtime_js_sources = ["deno_runtime/__runtime_js_sources"]
 __vendored_zlib_ng = ["flate2/zlib-ng-compat", "libz-sys/zlib-ng"]
 
 [build-dependencies]
-deno_runtime = { workspace = true, features = ["exclude_runtime_main_js", "include_js_files_for_snapshotting"] }
+# TODO(bartlomieju): should we not include `dont_create_runtime_snapshot`
+# feature here and actually create the snapshot in `deno_runtime` build script?
+# How do we pass options?
+deno_runtime = { workspace = true, features = ["dont_create_runtime_snapshot", "include_js_files_for_snapshotting"] }
 deno_core = { workspace = true, features = ["include_js_files_for_snapshotting"] }
 lazy-regex.workspace = true
 serde.workspace = true
@@ -63,7 +66,7 @@ deno_graph = "=0.61.5"
 deno_lint = { version = "=0.52.2", features = ["docs"] }
 deno_lockfile.workspace = true
 deno_npm = "0.15.2"
-deno_runtime = { workspace = true, features = ["dont_create_runtime_snapshot", "exclude_runtime_main_js", "include_js_files_for_snapshotting"] }
+deno_runtime = { workspace = true, features = ["dont_create_runtime_snapshot", "include_js_files_for_snapshotting"] }
 deno_semver = "0.5.1"
 deno_task_shell = "=0.14.0"
 eszip = "=0.55.5"

cli/build.rs

@@ -4,8 +4,6 @@ use std::env;
 use std::path::PathBuf;
 
 use deno_core::snapshot_util::*;
-use deno_core::ExtensionFileSource;
-use deno_core::ExtensionFileSourceCode;
 use deno_runtime::*;
 
 mod ts {
@@ -318,36 +316,9 @@ mod ts {
   }
 }
 
-// Duplicated in `ops/mod.rs`. Keep in sync!
-deno_core::extension!(
-  cli,
-  deps = [runtime],
-  esm_entry_point = "ext:cli/99_main.js",
-  esm = [
-    dir "js",
-    "99_main.js"
-  ],
-  customizer = |ext: &mut deno_core::Extension| {
-    ext.esm_files.to_mut().push(ExtensionFileSource {
-      specifier: "ext:cli/runtime/js/99_main.js",
-      code: ExtensionFileSourceCode::LoadedFromFsDuringSnapshot(
-        deno_runtime::js::PATH_FOR_99_MAIN_JS,
-      ),
-    });
-  }
-);
-
 #[cfg(not(feature = "__runtime_js_sources"))]
-#[must_use = "The files listed by create_cli_snapshot should be printed as 'cargo:rerun-if-changed' lines"]
-fn create_cli_snapshot(snapshot_path: PathBuf) -> CreateSnapshotOutput {
-  use deno_core::Extension;
-  use deno_runtime::deno_cache::SqliteBackedCache;
-  use deno_runtime::deno_cron::local::LocalCronHandler;
-  use deno_runtime::deno_http::DefaultHttpPropertyExtractor;
-  use deno_runtime::deno_kv::sqlite::SqliteDbHandler;
+fn create_cli_snapshot(snapshot_path: PathBuf) {
   use deno_runtime::ops::bootstrap::SnapshotOptions;
-  use deno_runtime::permissions::PermissionsContainer;
-  use std::sync::Arc;
 
   // NOTE(bartlomieju): keep in sync with `cli/version.rs`.
   // Ideally we could deduplicate that code.
@@ -359,76 +330,17 @@ fn create_cli_snapshot(snapshot_path: PathBuf) -> CreateSnapshotOutput {
     }
   }
 
-  // NOTE(bartlomieju): ordering is important here, keep it in sync with
-  // `runtime/worker.rs`, `runtime/web_worker.rs` and `runtime/build.rs`!
-  let fs = Arc::new(deno_fs::RealFs);
-  let extensions: Vec<Extension> = vec![
-    deno_webidl::deno_webidl::init_ops(),
-    deno_console::deno_console::init_ops(),
-    deno_url::deno_url::init_ops(),
-    deno_web::deno_web::init_ops::<PermissionsContainer>(
-      Default::default(),
-      Default::default(),
-    ),
-    deno_fetch::deno_fetch::init_ops::<PermissionsContainer>(Default::default()),
-    deno_cache::deno_cache::init_ops::<SqliteBackedCache>(None),
-    deno_websocket::deno_websocket::init_ops::<PermissionsContainer>(
-      "".to_owned(),
-      None,
-      None,
-    ),
-    deno_webstorage::deno_webstorage::init_ops(None),
-    deno_crypto::deno_crypto::init_ops(None),
-    deno_broadcast_channel::deno_broadcast_channel::init_ops(
-      deno_broadcast_channel::InMemoryBroadcastChannel::default(),
-    ),
-    deno_ffi::deno_ffi::init_ops::<PermissionsContainer>(),
-    deno_net::deno_net::init_ops::<PermissionsContainer>(None, None),
-    deno_tls::deno_tls::init_ops(),
-    deno_kv::deno_kv::init_ops(SqliteDbHandler::<PermissionsContainer>::new(
-      None, None,
-    )),
-    deno_cron::deno_cron::init_ops(LocalCronHandler::new()),
-    deno_napi::deno_napi::init_ops::<PermissionsContainer>(),
-    deno_http::deno_http::init_ops::<DefaultHttpPropertyExtractor>(),
-    deno_io::deno_io::init_ops(Default::default()),
-    deno_fs::deno_fs::init_ops::<PermissionsContainer>(fs.clone()),
-    deno_node::deno_node::init_ops::<PermissionsContainer>(None, fs),
-    deno_runtime::runtime::init_ops(),
-    deno_runtime::ops::runtime::deno_runtime::init_ops(
-      "deno:runtime".parse().unwrap(),
-    ),
-    deno_runtime::ops::worker_host::deno_worker_host::init_ops(
-      Arc::new(|_| unreachable!("not used in snapshot.")),
-      None,
-    ),
-    deno_runtime::ops::fs_events::deno_fs_events::init_ops(),
-    deno_runtime::ops::os::deno_os::init_ops(Default::default()),
-    deno_runtime::ops::permissions::deno_permissions::init_ops(),
-    deno_runtime::ops::process::deno_process::init_ops(),
-    deno_runtime::ops::signal::deno_signal::init_ops(),
-    deno_runtime::ops::tty::deno_tty::init_ops(),
-    deno_runtime::ops::http::deno_http_runtime::init_ops(),
-    deno_runtime::ops::bootstrap::deno_bootstrap::init_ops(Some(
-      SnapshotOptions {
-        deno_version: deno_version(),
-        ts_version: ts::version(),
-        v8_version: deno_core::v8_version(),
-        target: std::env::var("TARGET").unwrap(),
-      },
-    )),
-    cli::init_ops_and_esm(), // NOTE: This needs to be init_ops_and_esm!
-  ];
-
-  create_snapshot(CreateSnapshotOptions {
-    cargo_manifest_dir: env!("CARGO_MANIFEST_DIR"),
+  let snapshot_options = SnapshotOptions {
+    deno_version: deno_version(),
+    ts_version: ts::version(),
+    v8_version: deno_core::v8_version(),
+    target: std::env::var("TARGET").unwrap(),
+  };
+
+  deno_runtime::snapshot::create_runtime_snapshot(
     snapshot_path,
-    startup_snapshot: deno_runtime::js::deno_isolate_init(),
-    extensions,
-    compression_cb: None,
-    with_runtime_cb: None,
-    skip_op_registration: false,
-  })
+    snapshot_options,
+  );
 }
 
 fn git_commit_hash() -> String {
@@ -539,10 +451,7 @@ fn main() {
   #[cfg(not(feature = "__runtime_js_sources"))]
   {
     let cli_snapshot_path = o.join("CLI_SNAPSHOT.bin");
-    let output = create_cli_snapshot(cli_snapshot_path);
-    for path in output.files_loaded_during_snapshot {
-      println!("cargo:rerun-if-changed={}", path.display())
-    }
+    create_cli_snapshot(cli_snapshot_path);
   }
 
   #[cfg(target_os = "windows")]

cli/js/99_main.js

@@ -1,35 +1,5 @@
 // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
 
-use deno_core::Extension;
-
 pub mod bench;
 pub mod jupyter;
 pub mod testing;
-
-pub fn cli_exts() -> Vec<Extension> {
-  vec![
-    #[cfg(not(feature = "__runtime_js_sources"))]
-    cli::init_ops(),
-    #[cfg(feature = "__runtime_js_sources")]
-    cli::init_ops_and_esm(),
-  ]
-}
-
-// ESM parts duplicated in `../build.rs`. Keep in sync!
-deno_core::extension!(cli,
-  deps = [runtime],
-  esm_entry_point = "ext:cli/99_main.js",
-  esm = [
-    dir "js",
-    "40_testing.js",
-    "99_main.js"
-  ],
-  customizer = |ext: &mut deno_core::Extension| {
-    ext.esm_files.to_mut().push(deno_core::ExtensionFileSource {
-      specifier: "ext:cli/runtime/js/99_main.js",
-      code: deno_core::ExtensionFileSourceCode::LoadedFromFsDuringSnapshot(
-        deno_runtime::js::PATH_FOR_99_MAIN_JS,
-      ),
-    });
-  },
-);

cli/ops/mod.rs

@@ -52,7 +52,6 @@ use crate::args::StorageKeyResolver;
 use crate::emit::Emitter;
 use crate::errors;
 use crate::npm::CliNpmResolver;
-use crate::ops;
 use crate::tools;
 use crate::tools::coverage::CoverageCollector;
 use crate::tools::run::hmr::HmrRunner;
@@ -459,7 +458,7 @@ impl CliMainWorkerFactory {
     &self,
     main_module: ModuleSpecifier,
     permissions: PermissionsContainer,
-    mut custom_extensions: Vec<Extension>,
+    custom_extensions: Vec<Extension>,
     stdio: deno_runtime::deno_io::Stdio,
   ) -> Result<CliMainWorker, AnyError> {
     let shared = &self.shared;
@@ -564,9 +563,6 @@ impl CliMainWorkerFactory {
         .join(checksum::gen(&[key.as_bytes()]))
     });
 
-    let mut extensions = ops::cli_exts();
-    extensions.append(&mut custom_extensions);
-
     // TODO(bartlomieju): this is cruft, update FeatureChecker to spit out
     // list of enabled features.
     let feature_checker = shared.feature_checker.clone();
@@ -601,7 +597,7 @@ impl CliMainWorkerFactory {
           .maybe_binary_npm_command_name
           .clone(),
       },
-      extensions,
+      extensions: custom_extensions,
       startup_snapshot: crate::js::deno_isolate_init(),
       create_params: None,
       unsafely_ignore_certificate_errors: shared
@@ -753,8 +749,6 @@ fn create_web_worker_callback(
     let create_web_worker_cb =
       create_web_worker_callback(shared.clone(), stdio.clone());
 
-    let extensions = ops::cli_exts();
-
     let maybe_storage_key = shared
       .storage_key_resolver
       .resolve_storage_key(&args.main_module);
@@ -800,7 +794,7 @@ fn create_web_worker_callback(
           .maybe_binary_npm_command_name
           .clone(),
       },
-      extensions,
+      extensions: vec![],
       startup_snapshot: crate::js::deno_isolate_init(),
       unsafely_ignore_certificate_errors: shared
         .options

cli/worker.rs

@@ -214,7 +214,7 @@ mod startup_snapshot {
 
   pub fn create_runtime_snapshot(snapshot_path: PathBuf) {
     // NOTE(bartlomieju): ordering is important here, keep it in sync with
-    // `runtime/worker.rs`, `runtime/web_worker.rs` and `cli/build.rs`!
+    // `runtime/worker.rs`, `runtime/web_worker.rs` and `runtime/snapshot.rs`!
     let fs = std::sync::Arc::new(deno_fs::RealFs);
     let mut extensions: Vec<Extension> = vec![
       deno_webidl::deno_webidl::init_ops_and_esm(),

runtime/build.rs

@@ -1,6 +1,7 @@
 // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
 
 use crate::ops;
+use crate::ops::bootstrap::SnapshotOptions;
 use crate::shared::maybe_transpile_source;
 use crate::shared::runtime;
 use deno_cache::SqliteBackedCache;
@@ -183,9 +184,12 @@ impl deno_kv::sqlite::SqliteDbHandlerPermissions for Permissions {
   }
 }
 
-pub fn create_runtime_snapshot(snapshot_path: PathBuf) {
+pub fn create_runtime_snapshot(
+  snapshot_path: PathBuf,
+  snapshot_options: SnapshotOptions,
+) {
   // NOTE(bartlomieju): ordering is important here, keep it in sync with
-  // `runtime/worker.rs`, `runtime/web_worker.rs` and `cli/build.rs`!
+  // `runtime/worker.rs`, `runtime/web_worker.rs` and `runtime/snapshot.rs`!
   let fs = std::sync::Arc::new(deno_fs::RealFs);
   let mut extensions: Vec<Extension> = vec![
     deno_webidl::deno_webidl::init_ops_and_esm(),
@@ -234,7 +238,7 @@ pub fn create_runtime_snapshot(snapshot_path: PathBuf) {
     ops::signal::deno_signal::init_ops(),
     ops::tty::deno_tty::init_ops(),
     ops::http::deno_http_runtime::init_ops(),
-    ops::bootstrap::deno_bootstrap::init_ops(None),
+    ops::bootstrap::deno_bootstrap::init_ops(Some(snapshot_options)),
   ];
 
   for extension in &mut extensions {

runtime/snapshot.rs

@@ -397,7 +397,7 @@ impl WebWorker {
     });
 
     // NOTE(bartlomieju): ordering is important here, keep it in sync with
-    // `runtime/build.rs`, `runtime/worker.rs` and `cli/build.rs`!
+    // `runtime/build.rs`, `runtime/worker.rs` and `runtime/snapshot.rs`!
 
     let mut extensions = vec![
       // Web APIs

runtime/web_worker.rs

@@ -297,7 +297,7 @@ impl MainWorker {
     });
 
     // NOTE(bartlomieju): ordering is important here, keep it in sync with
-    // `runtime/build.rs`, `runtime/web_worker.rs` and `cli/build.rs`!
+    // `runtime/build.rs`, `runtime/web_worker.rs` and `runtime/snapshot.rs`!
     let mut extensions = vec![
       // Web APIs
       deno_webidl::deno_webidl::init_ops_and_esm(),