From 9a6536114ce7207ae0a28cf4ff0662605a494fe7 Mon Sep 17 00:00:00 2001 From: HasanAlrimawi Date: Wed, 18 Dec 2024 14:46:07 +0200 Subject: [PATCH 1/7] feat: show remote modules size on deno compile --- cli/standalone/binary.rs | 23 +++++++++++++++++++ .../compile/env_vars_support/compile.out | 2 ++ .../compile.out | 2 ++ .../compile/include/symlink_twice/compile.out | 2 ++ tests/specs/compile/npm_fs/compile.out | 2 ++ .../main_compile_folder.out | 2 ++ 6 files changed, 33 insertions(+) diff --git a/cli/standalone/binary.rs b/cli/standalone/binary.rs index 2ed52010fb156c..a357babd40b756 100644 --- a/cli/standalone/binary.rs +++ b/cli/standalone/binary.rs @@ -575,11 +575,29 @@ impl<'a> DenoCompileBinaryWriter<'a> { None => None, }; let mut vfs = VfsBuilder::new(); + let mut remote_modules_size = 0.0; let npm_snapshot = match self.npm_resolver.as_inner() { InnerCliNpmResolverRef::Managed(managed) => { let snapshot = managed.serialized_valid_snapshot_for_system(&self.npm_system_info); if !snapshot.as_serialized().packages.is_empty() { + let npm_snapshot = managed.snapshot(); + // make sure that graph.npm_packages.is_empty() isn't empty + if !graph.npm_packages.is_empty() { + for module in graph.modules() { + if let deno_graph::Module::Npm(module) = module { + let nv = module.nv_reference.nv(); + if let Ok(package) = + npm_snapshot.resolve_package_from_deno_module(nv) + { + // package.id.clone() is found now + if let Ok(size) = managed.package_size(&package.id) { + remote_modules_size += size as f64; + } + } + } + } + } self.fill_npm_vfs(&mut vfs).context("Building npm vfs.")?; Some(snapshot) } else { @@ -738,6 +756,11 @@ impl<'a> DenoCompileBinaryWriter<'a> { }; output_vfs(&vfs, display_output_filename); + log::info!( + "{} {}\n", + crate::colors::bold("Remote modules size:"), + crate::util::display::human_size(remote_modules_size) + ); let metadata = Metadata { argv: compile_flags.args.clone(), diff --git a/tests/specs/compile/env_vars_support/compile.out b/tests/specs/compile/env_vars_support/compile.out index cba114b1f7c55e..06eace1c192228 100644 --- a/tests/specs/compile/env_vars_support/compile.out +++ b/tests/specs/compile/env_vars_support/compile.out @@ -10,3 +10,5 @@ out[WILDLINE] Size: [WILDLINE] +Remote modules size: [WILDLINE] + diff --git a/tests/specs/compile/global_npm_cache_implicit_read_permission/compile.out b/tests/specs/compile/global_npm_cache_implicit_read_permission/compile.out index fa1dd2bf9fb459..2cfa6b71266657 100644 --- a/tests/specs/compile/global_npm_cache_implicit_read_permission/compile.out +++ b/tests/specs/compile/global_npm_cache_implicit_read_permission/compile.out @@ -47,3 +47,5 @@ main[WILDLINE] Size: [WILDLINE] +Remote modules size: [WILDLINE] + diff --git a/tests/specs/compile/include/symlink_twice/compile.out b/tests/specs/compile/include/symlink_twice/compile.out index 6ae93bf1cb6b92..43c94a89c74cf6 100644 --- a/tests/specs/compile/include/symlink_twice/compile.out +++ b/tests/specs/compile/include/symlink_twice/compile.out @@ -9,3 +9,5 @@ main[WILDLINE] Size: [WILDLINE] +Remote modules size: [WILDLINE] + diff --git a/tests/specs/compile/npm_fs/compile.out b/tests/specs/compile/npm_fs/compile.out index c2ecef40155e20..d8b09b6be7e091 100644 --- a/tests/specs/compile/npm_fs/compile.out +++ b/tests/specs/compile/npm_fs/compile.out @@ -8,3 +8,5 @@ main[WILDLINE] Size: [WILDLINE] +Remote modules size: [WILDLINE] + diff --git a/tests/testdata/compile/node_modules_symlink_outside/main_compile_folder.out b/tests/testdata/compile/node_modules_symlink_outside/main_compile_folder.out index 538aaa414c6904..5cb964c8e1a67d 100644 --- a/tests/testdata/compile/node_modules_symlink_outside/main_compile_folder.out +++ b/tests/testdata/compile/node_modules_symlink_outside/main_compile_folder.out @@ -15,3 +15,5 @@ bin[WILDLINE] Size: [WILDLINE] +Remote modules size: [WILDLINE] + From 79bb8068171cd0049ca74258f46696c0009da4df Mon Sep 17 00:00:00 2001 From: HasanAlrimawi Date: Wed, 18 Dec 2024 15:07:25 +0200 Subject: [PATCH 2/7] trigger checks From 9417786309f6f0c478b079ea2af0a1d986ad8245 Mon Sep 17 00:00:00 2001 From: HasanAlrimawi Date: Mon, 23 Dec 2024 12:10:59 +0200 Subject: [PATCH 3/7] update: get remote size from remote modules store --- cli/standalone/binary.rs | 4 +++- cli/standalone/serialization.rs | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/cli/standalone/binary.rs b/cli/standalone/binary.rs index 0c1d06740c5e3b..92873c6f464854 100644 --- a/cli/standalone/binary.rs +++ b/cli/standalone/binary.rs @@ -814,7 +814,9 @@ impl<'a> DenoCompileBinaryWriter<'a> { log::info!( "{} {}\n", crate::colors::bold("Remote modules size:"), - crate::util::display::human_size(remote_modules_size) + crate::util::display::human_size( + remote_modules_store.get_data_byte_len() as f64 + ) ); let metadata = Metadata { diff --git a/cli/standalone/serialization.rs b/cli/standalone/serialization.rs index 30802aa08197dd..95216bc6b61931 100644 --- a/cli/standalone/serialization.rs +++ b/cli/standalone/serialization.rs @@ -271,6 +271,10 @@ impl RemoteModulesStoreBuilder { } } } + + pub fn get_data_byte_len(&self) -> u64 { + self.data_byte_len + } } pub enum DenoCompileModuleSource { From 941e3149a9c93182ced04a3aae7ea440fa6608d7 Mon Sep 17 00:00:00 2001 From: HasanAlrimawi Date: Mon, 23 Dec 2024 14:37:05 +0200 Subject: [PATCH 4/7] remove unneeded variable --- cli/standalone/binary.rs | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/cli/standalone/binary.rs b/cli/standalone/binary.rs index 92873c6f464854..86cd480352bea2 100644 --- a/cli/standalone/binary.rs +++ b/cli/standalone/binary.rs @@ -598,29 +598,11 @@ impl<'a> DenoCompileBinaryWriter<'a> { None => None, }; let mut vfs = VfsBuilder::new(); - let mut remote_modules_size = 0.0; let npm_snapshot = match self.npm_resolver.as_inner() { InnerCliNpmResolverRef::Managed(managed) => { let snapshot = managed.serialized_valid_snapshot_for_system(&self.npm_system_info); if !snapshot.as_serialized().packages.is_empty() { - let npm_snapshot = managed.snapshot(); - // make sure that graph.npm_packages.is_empty() isn't empty - if !graph.npm_packages.is_empty() { - for module in graph.modules() { - if let deno_graph::Module::Npm(module) = module { - let nv = module.nv_reference.nv(); - if let Ok(package) = - npm_snapshot.resolve_package_from_deno_module(nv) - { - // package.id.clone() is found now - if let Ok(size) = managed.package_size(&package.id) { - remote_modules_size += size as f64; - } - } - } - } - } self.fill_npm_vfs(&mut vfs).context("Building npm vfs.")?; Some(snapshot) } else { From 6ae9e5a0b3e9e1b8dd73fef27fef952f990b22d8 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Thu, 23 Jan 2025 11:18:37 -0500 Subject: [PATCH 5/7] also show metadata --- cli/standalone/binary.rs | 47 +++++++++++++++---- cli/standalone/virtual_fs.rs | 6 +-- tests/integration/compile_tests.rs | 1 - .../case_insensitive_building/compile.out | 4 +- .../compile/env_vars_support/compile.out | 6 +-- .../compile.out | 6 +-- .../compile/include/symlink_twice/compile.out | 6 +-- tests/specs/compile/npm_fs/compile.out | 6 +-- .../main_compile_folder.out | 6 +-- 9 files changed, 56 insertions(+), 32 deletions(-) diff --git a/cli/standalone/binary.rs b/cli/standalone/binary.rs index d8a9db3922cae4..357a186ab7bd4f 100644 --- a/cli/standalone/binary.rs +++ b/cli/standalone/binary.rs @@ -1,6 +1,7 @@ // Copyright 2018-2025 the Deno authors. MIT license. use std::borrow::Cow; +use std::cell::Cell; use std::collections::HashMap; use std::collections::VecDeque; use std::env; @@ -633,13 +634,6 @@ impl<'a> DenoCompileBinaryWriter<'a> { }; output_vfs(&vfs, display_output_filename); - log::info!( - "{} {}\n", - crate::colors::bold("Remote modules size:"), - crate::util::display::human_size( - remote_modules_store.get_data_byte_len() as f64 - ) - ); let metadata = Metadata { argv: compile_flags.args.clone(), @@ -706,7 +700,7 @@ impl<'a> DenoCompileBinaryWriter<'a> { vfs_case_sensitivity: vfs.case_sensitivity, }; - let data_section_bytes = serialize_binary_data_section( + let (data_section_bytes, section_sizes) = serialize_binary_data_section( &metadata, npm_snapshot.map(|s| s.into_serialized()), &specifier_store.for_serialization(&root_dir_url), @@ -716,6 +710,22 @@ impl<'a> DenoCompileBinaryWriter<'a> { ) .context("Serializing binary data section.")?; + log::info!( + "\n{} {}", + crate::colors::bold("Files:"), + crate::util::display::human_size(section_sizes.vfs as f64) + ); + log::info!( + "{} {}", + crate::colors::bold("Metadata:"), + crate::util::display::human_size(section_sizes.metadata as f64) + ); + log::info!( + "{} {}\n", + crate::colors::bold("Remote modules:"), + crate::util::display::human_size(section_sizes.remote_modules as f64) + ); + write_binary_bytes(writer, original_bin, data_section_bytes, compile_flags) .context("Writing binary bytes") } @@ -919,6 +929,12 @@ fn write_binary_bytes( Ok(()) } +struct BinaryDataSectionSizes { + metadata: usize, + remote_modules: usize, + vfs: usize, +} + /// Binary format: /// * d3n0l4nd /// * @@ -937,12 +953,16 @@ fn serialize_binary_data_section( redirects: &SpecifierDataStore, remote_modules: &SpecifierDataStore>, vfs: &BuiltVfs, -) -> Result, AnyError> { +) -> Result<(Vec, BinaryDataSectionSizes), AnyError> { let metadata = serde_json::to_string(metadata)?; let npm_snapshot = npm_snapshot.map(serialize_npm_snapshot).unwrap_or_default(); let serialized_vfs = serde_json::to_string(&vfs.entries)?; + let remote_modules_len = Cell::new(0); + let metadata_len = Cell::new(0); + let vfs_len = Cell::new(0); + let bytes = capacity_builder::BytesBuilder::build(|builder| { builder.append(MAGIC_BYTES); // 1. Metadata @@ -955,12 +975,14 @@ fn serialize_binary_data_section( builder.append_le(npm_snapshot.len() as u64); builder.append(&npm_snapshot); } + metadata_len.set(builder.len()); // 3. Specifiers builder.append(specifiers); // 4. Redirects redirects.serialize(builder); // 5. Remote modules remote_modules.serialize(builder); + remote_modules_len.set(builder.len() - metadata_len.get()); // 6. VFS { builder.append_le(serialized_vfs.len() as u64); @@ -971,13 +993,18 @@ fn serialize_binary_data_section( builder.append(file); } } + vfs_len.set(builder.len() - remote_modules_len.get()); // write the magic bytes at the end so we can use it // to make sure we've deserialized correctly builder.append(MAGIC_BYTES); })?; - Ok(bytes) + Ok((bytes, BinaryDataSectionSizes { + metadata: metadata_len.get(), + remote_modules: remote_modules_len.get(), + vfs: vfs_len.get(), + })) } fn serialize_npm_snapshot( diff --git a/cli/standalone/virtual_fs.rs b/cli/standalone/virtual_fs.rs index fa79b784dcab33..a3e023620f98dc 100644 --- a/cli/standalone/virtual_fs.rs +++ b/cli/standalone/virtual_fs.rs @@ -30,11 +30,7 @@ pub fn output_vfs(vfs: &BuiltVfs, executable_name: &str) { let display_tree = vfs_as_display_tree(vfs, executable_name); display_tree.print(&mut text).unwrap(); // unwrap ok because it's writing to a string log::info!("\n{}\n", deno_terminal::colors::bold("Embedded Files")); - log::info!("{}\n", text.trim()); - log::info!( - "Size: {}\n", - human_size(vfs.files.iter().map(|f| f.len() as f64).sum()) - ); + log::info!("{}", text.trim()); } fn vfs_as_display_tree( diff --git a/tests/integration/compile_tests.rs b/tests/integration/compile_tests.rs index 7519ca4e86b078..aa4f576a7c9e9d 100644 --- a/tests/integration/compile_tests.rs +++ b/tests/integration/compile_tests.rs @@ -1020,7 +1020,6 @@ Warning Failed resolving symlink. Ignoring. Embedded Files [WILDCARD] - "#, ); diff --git a/tests/specs/compile/case_insensitive_building/compile.out b/tests/specs/compile/case_insensitive_building/compile.out index 895c2f1d46f094..2f63dabf702128 100644 --- a/tests/specs/compile/case_insensitive_building/compile.out +++ b/tests/specs/compile/case_insensitive_building/compile.out @@ -6,5 +6,7 @@ main[WILDLINE] ├── file.txt ([WILDLINE]) └── main.js ([WILDLINE]) -Size: [WILDLINE] +Files: [WILDLINE] +Metadata: [WILDLINE] +Remote modules: [WILDLINE] diff --git a/tests/specs/compile/env_vars_support/compile.out b/tests/specs/compile/env_vars_support/compile.out index 06eace1c192228..c8646d49c0afb4 100644 --- a/tests/specs/compile/env_vars_support/compile.out +++ b/tests/specs/compile/env_vars_support/compile.out @@ -8,7 +8,7 @@ Embedded Files out[WILDLINE] └── main.ts ([WILDLINE]) -Size: [WILDLINE] - -Remote modules size: [WILDLINE] +Files: [WILDLINE] +Metadata: [WILDLINE] +Remote modules: [WILDLINE] diff --git a/tests/specs/compile/global_npm_cache_implicit_read_permission/compile.out b/tests/specs/compile/global_npm_cache_implicit_read_permission/compile.out index 2cfa6b71266657..0bbc1ab3affc4a 100644 --- a/tests/specs/compile/global_npm_cache_implicit_read_permission/compile.out +++ b/tests/specs/compile/global_npm_cache_implicit_read_permission/compile.out @@ -45,7 +45,7 @@ main[WILDLINE] │ └── yargs-parser/18.1.3/* ([WILDLINE]) └── main.ts ([WILDLINE]) -Size: [WILDLINE] - -Remote modules size: [WILDLINE] +Files: [WILDLINE] +Metadata: [WILDLINE] +Remote modules: [WILDLINE] diff --git a/tests/specs/compile/include/symlink_twice/compile.out b/tests/specs/compile/include/symlink_twice/compile.out index 43c94a89c74cf6..a862d5d3264c2e 100644 --- a/tests/specs/compile/include/symlink_twice/compile.out +++ b/tests/specs/compile/include/symlink_twice/compile.out @@ -7,7 +7,7 @@ main[WILDLINE] ├── link.js --> index.js └── setup.js ([WILDLINE]) -Size: [WILDLINE] - -Remote modules size: [WILDLINE] +Files: [WILDLINE] +Metadata: [WILDLINE] +Remote modules: [WILDLINE] diff --git a/tests/specs/compile/npm_fs/compile.out b/tests/specs/compile/npm_fs/compile.out index d8b09b6be7e091..71067090a65df3 100644 --- a/tests/specs/compile/npm_fs/compile.out +++ b/tests/specs/compile/npm_fs/compile.out @@ -6,7 +6,7 @@ main[WILDLINE] ├── main.ts ([WILDLINE]) └── node_modules/* ([WILDLINE]) -Size: [WILDLINE] - -Remote modules size: [WILDLINE] +Files: [WILDLINE] +Metadata: [WILDLINE] +Remote modules: [WILDLINE] diff --git a/tests/testdata/compile/node_modules_symlink_outside/main_compile_folder.out b/tests/testdata/compile/node_modules_symlink_outside/main_compile_folder.out index 5cb964c8e1a67d..b6083bdd6dc4db 100644 --- a/tests/testdata/compile/node_modules_symlink_outside/main_compile_folder.out +++ b/tests/testdata/compile/node_modules_symlink_outside/main_compile_folder.out @@ -13,7 +13,7 @@ bin[WILDLINE] │ └── node_modules/* ([WILDLINE]) └── some_folder/* ([WILDLINE]) -Size: [WILDLINE] - -Remote modules size: [WILDLINE] +Files: [WILDLINE] +Metadata: [WILDLINE] +Remote modules: [WILDLINE] From 1f95c4b2b01fd6a696350a76a30ba78ac9125d6e Mon Sep 17 00:00:00 2001 From: David Sherret Date: Thu, 23 Jan 2025 11:20:16 -0500 Subject: [PATCH 6/7] revert change i did --- tests/integration/compile_tests.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/compile_tests.rs b/tests/integration/compile_tests.rs index aa4f576a7c9e9d..7519ca4e86b078 100644 --- a/tests/integration/compile_tests.rs +++ b/tests/integration/compile_tests.rs @@ -1020,6 +1020,7 @@ Warning Failed resolving symlink. Ignoring. Embedded Files [WILDCARD] + "#, ); From 9443f29c9d6f79650dfe53ddf276fe26acafb4a4 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Thu, 23 Jan 2025 11:22:10 -0500 Subject: [PATCH 7/7] Format --- cli/standalone/binary.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/cli/standalone/binary.rs b/cli/standalone/binary.rs index 357a186ab7bd4f..59744a0c72509e 100644 --- a/cli/standalone/binary.rs +++ b/cli/standalone/binary.rs @@ -1000,11 +1000,14 @@ fn serialize_binary_data_section( builder.append(MAGIC_BYTES); })?; - Ok((bytes, BinaryDataSectionSizes { - metadata: metadata_len.get(), - remote_modules: remote_modules_len.get(), - vfs: vfs_len.get(), - })) + Ok(( + bytes, + BinaryDataSectionSizes { + metadata: metadata_len.get(), + remote_modules: remote_modules_len.get(), + vfs: vfs_len.get(), + }, + )) } fn serialize_npm_snapshot(