optimize/eliminate fs read in cloc phase (#908)

Author: TilakMaddy

aderyn_core/src/fscloc/cloc.rs

@@ -389,3 +389,26 @@ mod parse_comments_to_rightfully_ignore_lines {
         }
     }
 }
+
+#[cfg(test)]
+mod cloc_tests {
+    use super::*;
+
+    #[test]
+    fn test_print_cloc_heavily_commented_contract() {
+        let content = include_str!(
+            "../../../tests/contract-playground/src/cloc/HeavilyCommentedContract.sol"
+        );
+        let stats = get_stats(content, false);
+        assert_eq!(stats.code, 21);
+    }
+
+    #[test]
+    fn test_print_cloc_another_heavily_commented_contract() {
+        let content = include_str!(
+            "../../../tests/contract-playground/src/cloc/AnotherHeavilyCommentedContract.sol"
+        );
+        let stats = get_stats(content, false);
+        assert_eq!(stats.code, 32);
+    }
+}

aderyn_core/src/fscloc/engine.rs

@@ -1,84 +1,32 @@
-use crate::fscloc::cloc;
-use ignore::{DirEntry, WalkBuilder, WalkState::Continue};
+use crate::context::workspace_context::WorkspaceContext;
 use rayon::prelude::*;
 use std::{
-    collections::{HashMap, HashSet},
+    collections::HashMap,
     path::{Path, PathBuf},
-    sync::Mutex,
+    str::FromStr,
 };
 
-use super::cloc::Stats;
+use super::cloc::{self, Stats};
 
 pub fn count_lines_of_code_and_collect_line_numbers_to_ignore(
-    src: &Path,
-    src_filepaths: &[String],
+    root: &Path,
     skip_cloc: bool,
-    included: &HashSet<PathBuf>,
-) -> Mutex<HashMap<String, Stats>> {
-    let (tx, rx) = crossbeam_channel::unbounded();
-
-    let form_path = |path: &String| {
-        // Soldiity compiler shenanigans ??
-        // In the line  `==== ??? ====` of the output, we're supposed to see the filename
-        // But sometimes solc puts filenames with path containing two forward slashes
-        // Example `contracts/templegold//AuctionBase.sol` in there
-        // Although there is a separate entry for `contracts/templegold/AuctionBase.sol`.
-        // We want to omit reading the former
-        if path.contains("//") {
-            // When using foundry-compilers-aletheia, there is no separate entry so we'll
-            // adjust the same entry
-            let adjusted_path = path.replace("//", "/");
-            Path::new(&src).join(adjusted_path)
-        } else {
-            Path::new(&src).join(path)
-        }
-    };
-
-    let src_filepaths_as_paths = src_filepaths
-        .iter()
-        .map(form_path)
-        .filter(|s| included.contains(s.strip_prefix(src).unwrap()))
-        .collect::<Vec<_>>();
-
-    if !src_filepaths_as_paths.is_empty() {
-        // Only add the paths to WalkBuilder that we want to do analysis on.
-        let mut walker = WalkBuilder::new(src_filepaths_as_paths[0].clone());
-
-        for item in src_filepaths_as_paths.iter().skip(1) {
-            walker.add(item);
-        }
-
-        walker.build_parallel().run(|| {
-            let tx = tx.clone();
-            Box::new(move |res| {
-                if let Ok(target) = res {
-                    if target.file_type().unwrap().is_file() {
-                        let send = target.to_owned();
-                        tx.send(send).unwrap();
-                    }
-                }
-                Continue
-            })
-        });
-    }
-
-    drop(tx); // without this, the program would not terminate .. because receiver would
-              // think that the `tx` is still waiting to send something.. but we're done
-              // the clones have been dropped but not the original
-              // refer rust docs for more on automatic garbage collection :)
-
-    let lines_of_code = Mutex::new(HashMap::new());
-
-    let receive = |target_file: DirEntry| {
-        // println!("Processing: {:?}", target_file.path());
-        let r_content = std::fs::read_to_string(target_file.path()).unwrap();
-        let stats = cloc::get_stats(&r_content, skip_cloc);
-        let key = String::from(target_file.path().to_string_lossy());
-        let mut lock = lines_of_code.lock().unwrap();
-        // println!("Inserting: {} - {}", key, stats.code);
-        lock.insert(key, stats);
-    };
-
-    rx.into_iter().par_bridge().for_each(receive);
-    lines_of_code
+    context: &WorkspaceContext,
+) -> HashMap<String, Stats> {
+    context
+        .source_units()
+        .par_iter()
+        .map(|source_unit| {
+            let path = source_unit.absolute_path.clone().expect("absolute path not inserted");
+            let path = path.replace("//", "/"); // Condense entries that look like `contracts/templegold//AuctionBase.sol`
+            if !context.included.contains(&PathBuf::from_str(&path).unwrap()) {
+                return None;
+            }
+            let content = source_unit.source.as_ref().expect("source not filled");
+            let stats = cloc::get_stats(content, skip_cloc);
+            let full_path = root.join(&path).to_string_lossy().to_string();
+            Some((full_path, stats))
+        })
+        .flatten()
+        .collect()
 }

aderyn_core/src/fscloc/mod.rs

@@ -10,66 +10,3 @@ pub struct CodeLines {
     pub actual_first_line: isize,
     pub last_line_has_code: bool,
 }
-
-#[cfg(test)]
-mod cloc_tests {
-    use std::{collections::HashSet, path::PathBuf, str::FromStr};
-
-    use crate::fscloc::engine;
-
-    #[test]
-    fn test_print_loc() {
-        let src_filepaths = vec![
-            "HeavilyCommentedContract.sol".to_string(),
-            "AnotherHeavilyCommentedContract.sol".to_string(),
-        ];
-        let included = HashSet::from_iter(vec![
-            PathBuf::from_str("HeavilyCommentedContract.sol").unwrap(),
-            PathBuf::from_str("AnotherHeavilyCommentedContract.sol").unwrap(),
-        ]);
-        let sol = engine::count_lines_of_code_and_collect_line_numbers_to_ignore(
-            PathBuf::from("../tests/contract-playground/src/cloc").as_path(),
-            &src_filepaths,
-            false,
-            &included,
-        );
-        let result = sol.lock().unwrap();
-        result.iter().for_each(|element| println!("{} - {}", element.0, element.1.code));
-        assert_eq!(
-            result
-                .get("../tests/contract-playground/src/cloc/HeavilyCommentedContract.sol")
-                .unwrap()
-                .code,
-            21
-        );
-        assert_eq!(
-            result
-                .get("../tests/contract-playground/src/cloc/AnotherHeavilyCommentedContract.sol")
-                .unwrap()
-                .code,
-            32
-        );
-    }
-
-    #[test]
-    fn test_print_loc_specific_file() {
-        let src_filepaths = vec!["HeavilyCommentedContract.sol".to_string()];
-        let included =
-            HashSet::from_iter(vec![PathBuf::from_str("HeavilyCommentedContract.sol").unwrap()]);
-        let sol = engine::count_lines_of_code_and_collect_line_numbers_to_ignore(
-            PathBuf::from("../tests/contract-playground/src/cloc").as_path(),
-            &src_filepaths,
-            false,
-            &included,
-        );
-        let result = sol.lock().unwrap();
-        assert_eq!(result.len(), 1);
-        assert_eq!(
-            result
-                .get("../tests/contract-playground/src/cloc/HeavilyCommentedContract.sol")
-                .unwrap()
-                .code,
-            21
-        );
-    }
-}

aderyn_driver/src/process.rs

@@ -57,23 +57,14 @@ pub fn make_context(
         let absolute_root_path = &ensure_valid_root_path(&root_path);
         let stats = fscloc::engine::count_lines_of_code_and_collect_line_numbers_to_ignore(
             absolute_root_path.as_path(),
-            &context.src_filepaths,
             common.skip_cloc,
-            &context.included,
+            context,
         );
-        let sloc_stats: HashMap<String, usize> = stats
-            .lock()
-            .expect("failed to lock")
-            .iter()
-            .map(|(key, value)| (key.to_owned(), value.code))
-            .collect();
-
-        let ignore_line_stats: HashMap<String, Vec<fscloc::cloc::IgnoreLine>> = stats
-            .lock()
-            .expect("failed to lock")
-            .iter()
-            .map(|(key, value)| (key.to_owned(), value.ignore_lines.clone()))
-            .collect();
+        let sloc_stats: HashMap<String, usize> =
+            stats.iter().map(|(key, value)| (key.to_owned(), value.code)).collect();
+
+        let ignore_line_stats: HashMap<String, Vec<fscloc::cloc::IgnoreLine>> =
+            stats.iter().map(|(key, value)| (key.to_owned(), value.ignore_lines.clone())).collect();
 
         context.set_sloc_stats(sloc_stats);
         context.set_ignore_lines_stats(ignore_line_stats);