Unify progress data (#972)

* Unify progress data code

* Do not increment in threads values every time when finding file/folder(works quite good, may works bad with folders with ~several thousands of files)

* Partial changes

* Simplify creating tree_view

* Allow setting thread number in CLI

* Simplified code and add tests with help of copilot

Author: qarmin

czkawka_cli/src/commands.rs

@@ -85,6 +85,8 @@ pub enum Commands {
 
 #[derive(Debug, clap::Args)]
 pub struct DuplicatesArgs {
+    #[clap(flatten)]
+    pub thread_number: ThreadNumber,
     #[clap(flatten)]
     pub directories: Directories,
     #[clap(flatten)]
@@ -163,6 +165,8 @@ pub struct DuplicatesArgs {
 
 #[derive(Debug, clap::Args)]
 pub struct EmptyFoldersArgs {
+    #[clap(flatten)]
+    pub thread_number: ThreadNumber,
     #[clap(flatten)]
     pub directories: Directories,
     #[clap(flatten)]
@@ -180,6 +184,8 @@ pub struct EmptyFoldersArgs {
 
 #[derive(Debug, clap::Args)]
 pub struct BiggestFilesArgs {
+    #[clap(flatten)]
+    pub thread_number: ThreadNumber,
     #[clap(flatten)]
     pub directories: Directories,
     #[clap(flatten)]
@@ -205,6 +211,8 @@ pub struct BiggestFilesArgs {
 
 #[derive(Debug, clap::Args)]
 pub struct EmptyFilesArgs {
+    #[clap(flatten)]
+    pub thread_number: ThreadNumber,
     #[clap(flatten)]
     pub directories: Directories,
     #[clap(flatten)]
@@ -226,6 +234,8 @@ pub struct EmptyFilesArgs {
 
 #[derive(Debug, clap::Args)]
 pub struct TemporaryArgs {
+    #[clap(flatten)]
+    pub thread_number: ThreadNumber,
     #[clap(flatten)]
     pub directories: Directories,
     #[clap(flatten)]
@@ -245,6 +255,8 @@ pub struct TemporaryArgs {
 
 #[derive(Debug, clap::Args)]
 pub struct SimilarImagesArgs {
+    #[clap(flatten)]
+    pub thread_number: ThreadNumber,
     #[clap(flatten)]
     pub directories: Directories,
     #[clap(flatten)]
@@ -313,6 +325,8 @@ pub struct SimilarImagesArgs {
 
 #[derive(Debug, clap::Args)]
 pub struct SameMusicArgs {
+    #[clap(flatten)]
+    pub thread_number: ThreadNumber,
     #[clap(flatten)]
     pub directories: Directories,
     #[clap(flatten)]
@@ -359,6 +373,8 @@ pub struct SameMusicArgs {
 
 #[derive(Debug, clap::Args)]
 pub struct InvalidSymlinksArgs {
+    #[clap(flatten)]
+    pub thread_number: ThreadNumber,
     #[clap(flatten)]
     pub directories: Directories,
     #[clap(flatten)]
@@ -380,6 +396,8 @@ pub struct InvalidSymlinksArgs {
 
 #[derive(Debug, clap::Args)]
 pub struct BrokenFilesArgs {
+    #[clap(flatten)]
+    pub thread_number: ThreadNumber,
     #[clap(flatten)]
     pub directories: Directories,
     #[clap(flatten)]
@@ -401,6 +419,8 @@ pub struct BrokenFilesArgs {
 
 #[derive(Debug, clap::Args)]
 pub struct SimilarVideosArgs {
+    #[clap(flatten)]
+    pub thread_number: ThreadNumber,
     #[clap(flatten)]
     pub directories: Directories,
     #[clap(flatten)]
@@ -449,6 +469,8 @@ pub struct SimilarVideosArgs {
 
 #[derive(Debug, clap::Args)]
 pub struct BadExtensionsArgs {
+    #[clap(flatten)]
+    pub thread_number: ThreadNumber,
     #[clap(flatten)]
     pub directories: Directories,
     #[clap(flatten)]
@@ -517,6 +539,12 @@ pub struct NotRecursive {
     pub not_recursive: bool,
 }
 
+#[derive(Debug, clap::Args)]
+pub struct ThreadNumber {
+    #[clap(short = 'T', long, default_value = "0", help = "Limits thread number, 0(default) will use all available threads")]
+    pub thread_number: usize,
+}
+
 #[cfg(target_family = "unix")]
 #[derive(Debug, clap::Args)]
 pub struct ExcludeOtherFilesystems {

czkawka_cli/src/main.rs

@@ -6,7 +6,7 @@ use clap::Parser;
 
 use commands::Commands;
 use czkawka_core::big_file::SearchMode;
-use czkawka_core::common::{get_number_of_threads, set_default_number_of_threads};
+use czkawka_core::common::set_number_of_threads;
 #[allow(unused_imports)] // It is used in release for print_results().
 use czkawka_core::common_traits::*;
 use czkawka_core::similar_images::test_image_conversion_speed;
@@ -34,8 +34,6 @@ mod commands;
 fn main() {
     let command = Args::parse().command;
 
-    set_default_number_of_threads();
-    println!("Set thread number to {}", get_number_of_threads());
     #[cfg(debug_assertions)]
     println!("{command:?}");
 
@@ -59,6 +57,7 @@ fn main() {
 
 fn duplicates(duplicates: DuplicatesArgs) {
     let DuplicatesArgs {
+        thread_number,
         directories,
         excluded_directories,
         excluded_items,
@@ -78,6 +77,8 @@ fn duplicates(duplicates: DuplicatesArgs) {
         case_sensitive_name_comparison,
     } = duplicates;
 
+    set_number_of_threads(thread_number.thread_number);
+
     let mut df = DuplicateFinder::new();
 
     df.set_included_directory(directories.directories);
@@ -113,6 +114,7 @@ fn duplicates(duplicates: DuplicatesArgs) {
 
 fn empty_folders(empty_folders: EmptyFoldersArgs) {
     let EmptyFoldersArgs {
+        thread_number,
         directories,
         delete_folders,
         file_to_save,
@@ -122,6 +124,8 @@ fn empty_folders(empty_folders: EmptyFoldersArgs) {
         exclude_other_filesystems,
     } = empty_folders;
 
+    set_number_of_threads(thread_number.thread_number);
+
     let mut ef = EmptyFolder::new();
 
     ef.set_included_directory(directories.directories);
@@ -147,6 +151,7 @@ fn empty_folders(empty_folders: EmptyFoldersArgs) {
 
 fn biggest_files(biggest_files: BiggestFilesArgs) {
     let BiggestFilesArgs {
+        thread_number,
         directories,
         excluded_directories,
         excluded_items,
@@ -160,6 +165,8 @@ fn biggest_files(biggest_files: BiggestFilesArgs) {
         smallest_mode,
     } = biggest_files;
 
+    set_number_of_threads(thread_number.thread_number);
+
     let mut bf = BigFile::new();
 
     bf.set_included_directory(directories.directories);
@@ -193,6 +200,7 @@ fn biggest_files(biggest_files: BiggestFilesArgs) {
 
 fn empty_files(empty_files: EmptyFilesArgs) {
     let EmptyFilesArgs {
+        thread_number,
         directories,
         excluded_directories,
         excluded_items,
@@ -204,6 +212,8 @@ fn empty_files(empty_files: EmptyFilesArgs) {
         exclude_other_filesystems,
     } = empty_files;
 
+    set_number_of_threads(thread_number.thread_number);
+
     let mut ef = EmptyFiles::new();
 
     ef.set_included_directory(directories.directories);
@@ -234,6 +244,7 @@ fn empty_files(empty_files: EmptyFilesArgs) {
 
 fn temporary(temporary: TemporaryArgs) {
     let TemporaryArgs {
+        thread_number,
         directories,
         excluded_directories,
         excluded_items,
@@ -244,6 +255,8 @@ fn temporary(temporary: TemporaryArgs) {
         not_recursive,
     } = temporary;
 
+    set_number_of_threads(thread_number.thread_number);
+
     let mut tf = Temporary::new();
 
     tf.set_included_directory(directories.directories);
@@ -273,6 +286,7 @@ fn temporary(temporary: TemporaryArgs) {
 
 fn similar_images(similar_images: SimilarImagesArgs) {
     let SimilarImagesArgs {
+        thread_number,
         directories,
         excluded_directories,
         excluded_items,
@@ -288,6 +302,8 @@ fn similar_images(similar_images: SimilarImagesArgs) {
         hash_size,
     } = similar_images;
 
+    set_number_of_threads(thread_number.thread_number);
+
     let mut sf = SimilarImages::new();
 
     sf.set_included_directory(directories.directories);
@@ -320,6 +336,7 @@ fn similar_images(similar_images: SimilarImagesArgs) {
 
 fn same_music(same_music: SameMusicArgs) {
     let SameMusicArgs {
+        thread_number,
         directories,
         excluded_directories,
         excluded_items,
@@ -333,6 +350,8 @@ fn same_music(same_music: SameMusicArgs) {
         music_similarity,
     } = same_music;
 
+    set_number_of_threads(thread_number.thread_number);
+
     let mut mf = SameMusic::new();
 
     mf.set_included_directory(directories.directories);
@@ -365,6 +384,7 @@ fn same_music(same_music: SameMusicArgs) {
 
 fn invalid_symlinks(invalid_symlinks: InvalidSymlinksArgs) {
     let InvalidSymlinksArgs {
+        thread_number,
         directories,
         excluded_directories,
         excluded_items,
@@ -376,6 +396,8 @@ fn invalid_symlinks(invalid_symlinks: InvalidSymlinksArgs) {
         delete_files,
     } = invalid_symlinks;
 
+    set_number_of_threads(thread_number.thread_number);
+
     let mut ifs = InvalidSymlinks::new();
 
     ifs.set_included_directory(directories.directories);
@@ -405,6 +427,7 @@ fn invalid_symlinks(invalid_symlinks: InvalidSymlinksArgs) {
 
 fn broken_files(broken_files: BrokenFilesArgs) {
     let BrokenFilesArgs {
+        thread_number,
         directories,
         excluded_directories,
         excluded_items,
@@ -416,6 +439,8 @@ fn broken_files(broken_files: BrokenFilesArgs) {
         exclude_other_filesystems,
     } = broken_files;
 
+    set_number_of_threads(thread_number.thread_number);
+
     let mut br = BrokenFiles::new();
 
     br.set_included_directory(directories.directories);
@@ -446,6 +471,7 @@ fn broken_files(broken_files: BrokenFilesArgs) {
 
 fn similar_videos(similar_videos: SimilarVideosArgs) {
     let SimilarVideosArgs {
+        thread_number,
         directories,
         excluded_directories,
         excluded_items,
@@ -459,6 +485,8 @@ fn similar_videos(similar_videos: SimilarVideosArgs) {
         allowed_extensions,
     } = similar_videos;
 
+    set_number_of_threads(thread_number.thread_number);
+
     let mut vr = SimilarVideos::new();
 
     vr.set_included_directory(directories.directories);
@@ -488,6 +516,7 @@ fn similar_videos(similar_videos: SimilarVideosArgs) {
 
 fn bad_extensions(bad_extensions: BadExtensionsArgs) {
     let BadExtensionsArgs {
+        thread_number,
         directories,
         excluded_directories,
         excluded_items,
@@ -498,6 +527,8 @@ fn bad_extensions(bad_extensions: BadExtensionsArgs) {
         allowed_extensions,
     } = bad_extensions;
 
+    set_number_of_threads(thread_number.thread_number);
+
     let mut be = BadExtensions::new();
 
     be.set_included_directory(directories.directories);

czkawka_core/src/bad_extensions.rs

@@ -13,7 +13,7 @@ use mime_guess::get_mime_extensions;
 use rayon::prelude::*;
 
 use crate::common::{prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads};
-use crate::common_dir_traversal::{CheckingMethod, DirTraversalBuilder, DirTraversalResult, FileEntry, ProgressData};
+use crate::common_dir_traversal::{CheckingMethod, DirTraversalBuilder, DirTraversalResult, FileEntry, ProgressData, ToolType};
 use crate::common_directory::Directories;
 use crate::common_extensions::Extensions;
 use crate::common_items::ExcludedItems;
@@ -124,6 +124,7 @@ const WORKAROUNDS: &[(&str, &str)] = &[
     ("xml", "vbox"),      // VirtualBox
     ("xml", "vbox-prev"), // VirtualBox
     ("xml", "vcproj"),    // VisualStudio
+    ("xml", "vcxproj"),   // VisualStudio
     ("xml", "xba"),       // Libreoffice
     ("xml", "xcd"),       // Libreoffice files
     ("zip", "apk"),       // Android apk
@@ -172,6 +173,7 @@ impl Info {
 }
 
 pub struct BadExtensions {
+    tool_type: ToolType,
     text_messages: Messages,
     information: Info,
     files_to_check: Vec<FileEntry>,
@@ -191,6 +193,7 @@ impl BadExtensions {
     #[must_use]
     pub fn new() -> Self {
         Self {
+            tool_type: ToolType::BadExtensions,
             text_messages: Messages::new(),
             information: Info::new(),
             recursive_search: true,
@@ -314,7 +317,7 @@ impl BadExtensions {
 
     fn look_for_bad_extensions_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
         let (progress_thread_handle, progress_thread_run, atomic_counter, check_was_stopped) =
-            prepare_thread_handler_common(progress_sender, 1, 1, self.files_to_check.len(), CheckingMethod::None);
+            prepare_thread_handler_common(progress_sender, 1, 1, self.files_to_check.len(), CheckingMethod::None, self.tool_type);
 
         let files_to_check = mem::take(&mut self.files_to_check);
 

czkawka_core/src/big_file.rs

@@ -14,7 +14,7 @@ use humansize::BINARY;
 use rayon::prelude::*;
 
 use crate::common::{check_folder_children, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads, split_path};
-use crate::common_dir_traversal::{common_get_entry_data_metadata, common_read_dir, get_lowercase_name, get_modified_time, CheckingMethod, ProgressData};
+use crate::common_dir_traversal::{common_get_entry_data_metadata, common_read_dir, get_lowercase_name, get_modified_time, CheckingMethod, ProgressData, ToolType};
 use crate::common_directory::Directories;
 use crate::common_extensions::Extensions;
 use crate::common_items::ExcludedItems;
@@ -55,6 +55,7 @@ impl Info {
 
 /// Struct with required information's to work
 pub struct BigFile {
+    tool_type: ToolType,
     text_messages: Messages,
     information: Info,
     big_files: Vec<(u64, FileEntry)>,
@@ -72,6 +73,7 @@ impl BigFile {
     #[must_use]
     pub fn new() -> Self {
         Self {
+            tool_type: ToolType::BigFile,
             text_messages: Default::default(),
             information: Info::new(),
             big_files: Default::default(),
@@ -148,7 +150,8 @@ impl BigFile {
             folders_to_check.push(id.clone());
         }
 
-        let (progress_thread_handle, progress_thread_run, atomic_counter, _check_was_stopped) = prepare_thread_handler_common(progress_sender, 0, 0, 0, CheckingMethod::None);
+        let (progress_thread_handle, progress_thread_run, atomic_counter, _check_was_stopped) =
+            prepare_thread_handler_common(progress_sender, 0, 0, 0, CheckingMethod::None, self.tool_type);
 
         while !folders_to_check.is_empty() {
             if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() {