Skip to content

Commit

Permalink
Simplify scanning/analyzing code
Browse files Browse the repository at this point in the history
  • Loading branch information
@muglug
muglug committed Jul 24, 2024
1 parent 437e1c4 commit 78380d4
Show file tree
Hide file tree
Showing 2 changed files with 136 additions and 217 deletions.
144 changes: 55 additions & 89 deletions src/file_scanner_analyzer/analyzer.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,116 +69,82 @@ pub fn analyze_files(
None
};

if path_groups.len() == 1 {
let codebase = &scan_data.codebase;
let interner = &scan_data.interner;
let resolved_names = &scan_data.resolved_names;

let mut new_analysis_result =
AnalysisResult::new(config.graph_kind, SymbolReferences::new());

for (i, str_path) in path_groups[&0].iter().enumerate() {
let file_path = FilePath(interner.get(str_path).unwrap());
if let Some(resolved_names) = resolved_names.get(&file_path) {
*file_analysis_time += analyze_file(
file_path,
str_path,
scan_data
.file_system
.file_hashes_and_times
.get(&file_path)
.map(|k| k.1),
codebase,
interner,
&config,
&mut new_analysis_result,
resolved_names,
&logger,
);
}

update_progressbar(i as u64, bar.clone());
}

analysis_result.lock().unwrap().extend(new_analysis_result);
} else {
let mut handles = vec![];

let files_processed = Arc::new(Mutex::new(0));

let arc_file_analysis_time = Arc::new(Mutex::new(Duration::default()));
let mut handles = vec![];

for (_, path_group) in path_groups {
let scan_data = scan_data.clone();
let files_processed = Arc::new(Mutex::new(0));

let pgc = path_group.iter().map(|c| (*c).clone()).collect::<Vec<_>>();
let arc_file_analysis_time = Arc::new(Mutex::new(Duration::default()));

let analysis_result = analysis_result.clone();
for (_, path_group) in path_groups {
let scan_data = scan_data.clone();

let analysis_config = config.clone();
let pgc = path_group.iter().map(|c| (*c).clone()).collect::<Vec<_>>();

let files_processed = files_processed.clone();
let bar = bar.clone();
let analysis_result = analysis_result.clone();

let logger = logger.clone();
let analysis_config = config.clone();

let arc_file_analysis_time = arc_file_analysis_time.clone();
let files_processed = files_processed.clone();
let bar = bar.clone();

let handle = std::thread::spawn(move || {
let codebase = &scan_data.codebase;
let interner = &scan_data.interner;
let resolved_names = &scan_data.resolved_names;
let logger = logger.clone();

let mut file_analysis_time = Duration::default();
let arc_file_analysis_time = arc_file_analysis_time.clone();

let mut new_analysis_result =
AnalysisResult::new(analysis_config.graph_kind, SymbolReferences::new());
let handle = std::thread::spawn(move || {
let codebase = &scan_data.codebase;
let interner = &scan_data.interner;
let resolved_names = &scan_data.resolved_names;

for str_path in &pgc {
let file_path = FilePath(interner.get(str_path).unwrap());
let mut file_analysis_time = Duration::default();

if let Some(resolved_names) = resolved_names.get(&file_path) {
file_analysis_time += analyze_file(
file_path,
str_path,
scan_data
.file_system
.file_hashes_and_times
.get(&file_path)
.map(|k| k.1),
codebase,
interner,
&analysis_config,
&mut new_analysis_result,
resolved_names,
&logger,
);
}
let mut new_analysis_result =
AnalysisResult::new(analysis_config.graph_kind, SymbolReferences::new());

let mut tally = files_processed.lock().unwrap();
*tally += 1;
for str_path in &pgc {
let file_path = FilePath(interner.get(str_path).unwrap());

update_progressbar(*tally, bar.clone());
if let Some(resolved_names) = resolved_names.get(&file_path) {
file_analysis_time += analyze_file(
file_path,
str_path,
scan_data
.file_system
.file_hashes_and_times
.get(&file_path)
.map(|k| k.1),
codebase,
interner,
&analysis_config,
&mut new_analysis_result,
resolved_names,
&logger,
);
}

let mut t = arc_file_analysis_time.lock().unwrap();
*t += file_analysis_time;
analysis_result.lock().unwrap().extend(new_analysis_result);
});
let mut tally = files_processed.lock().unwrap();
*tally += 1;

handles.push(handle);
}
update_progressbar(*tally, bar.clone());
}

for handle in handles {
handle.join().unwrap();
}
let mut t = arc_file_analysis_time.lock().unwrap();
*t += file_analysis_time;
analysis_result.lock().unwrap().extend(new_analysis_result);
});

*file_analysis_time = Arc::try_unwrap(arc_file_analysis_time)
.unwrap()
.into_inner()
.unwrap();
handles.push(handle);
}

for handle in handles {
handle.join().unwrap();
}

*file_analysis_time = Arc::try_unwrap(arc_file_analysis_time)
.unwrap()
.into_inner()
.unwrap();

if let Some(bar) = &bar {
bar.finish_and_clear();
}
Expand Down
209 changes: 81 additions & 128 deletions src/file_scanner_analyzer/scanner.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -286,145 +286,98 @@ pub fn scan_files(
.push(str_path);
}

if path_groups.len() == 1 {
let mut new_codebase = CodebaseInfo::new();
let mut new_interner = ThreadedInterner::new(interner.clone());
let empty_name_context = NameContext::new(&mut new_interner);

let analyze_map = files_to_analyze.iter().collect::<FxHashSet<_>>();

for (i, file_path) in path_groups[&0].iter().enumerate() {
let str_path = new_interner
.parent
.lock()
.unwrap()
.lookup(&file_path.0)
.to_string();

match scan_file(
&str_path,
**file_path,
&config.all_custom_issues,
&mut new_codebase,
&mut new_interner,
empty_name_context.clone(),
analyze_map.contains(&str_path),
!config.test_files.iter().any(|p| p.matches(&str_path)),
&logger,
) {
Ok(scanner_result) => {
resolved_names
.lock()
.unwrap()
.insert(**file_path, scanner_result);
}
Err(parser_error) => {
resolved_names.lock().unwrap().remove(*file_path);
new_codebase.files.insert(
**file_path,
FileInfo {
parser_error: Some(parser_error),
..FileInfo::default()
},
);
invalid_files.lock().unwrap().push(**file_path);
}
}

update_progressbar(i as u64, bar.clone());
}

if config.ast_diff {
codebase_diff.extend(get_diff(&existing_changed_files, &new_codebase.files));
}

codebase.extend(new_codebase);
} else {
let mut handles = vec![];

let thread_codebases = Arc::new(Mutex::new(vec![]));

for (_, path_group) in path_groups {
let pgc = path_group.iter().map(|c| *(*c)).collect::<Vec<_>>();

let codebases = thread_codebases.clone();

let bar = bar.clone();
let files_processed = files_processed.clone();

let analyze_map = files_to_analyze
.clone()
.into_iter()
.collect::<FxHashSet<_>>();

let interner = interner.clone();

let resolved_names = resolved_names.clone();

let config = config.clone();
let logger = logger.clone();
let invalid_files = invalid_files.clone();

let handle = std::thread::spawn(move || {
let mut new_codebase = CodebaseInfo::new();
let mut new_interner = ThreadedInterner::new(interner);
let empty_name_context = NameContext::new(&mut new_interner);
let mut local_resolved_names = FxHashMap::default();

for file_path in &pgc {
let str_path = new_interner
.parent
.lock()
.unwrap()
.lookup(&file_path.0)
.to_string();

if let Ok(scanner_result) = scan_file(
&str_path,
*file_path,
&config.all_custom_issues,
&mut new_codebase,
&mut new_interner,
empty_name_context.clone(),
analyze_map.contains(&str_path),
!config.test_files.iter().any(|p| p.matches(&str_path)),
&logger.clone(),
) {
let mut handles = vec![];

let thread_codebases = Arc::new(Mutex::new(vec![]));

for (_, path_group) in path_groups {
let pgc = path_group.iter().map(|c| *(*c)).collect::<Vec<_>>();

let codebases = thread_codebases.clone();

let bar = bar.clone();
let files_processed = files_processed.clone();

let analyze_map = files_to_analyze
.clone()
.into_iter()
.collect::<FxHashSet<_>>();

let interner = interner.clone();

let resolved_names = resolved_names.clone();

let config = config.clone();
let logger = logger.clone();
let invalid_files = invalid_files.clone();

let handle = std::thread::spawn(move || {
let mut new_codebase = CodebaseInfo::new();
let mut new_interner = ThreadedInterner::new(interner);
let empty_name_context = NameContext::new(&mut new_interner);
let mut local_resolved_names = FxHashMap::default();

for file_path in &pgc {
let str_path = new_interner
.parent
.lock()
.unwrap()
.lookup(&file_path.0)
.to_string();

match scan_file(
&str_path,
*file_path,
&config.all_custom_issues,
&mut new_codebase,
&mut new_interner,
empty_name_context.clone(),
analyze_map.contains(&str_path),
!config.test_files.iter().any(|p| p.matches(&str_path)),
&logger.clone(),
) {
Ok(scanner_result) => {
local_resolved_names.insert(*file_path, scanner_result);
} else {
}
Err(parser_error) => {
local_resolved_names.remove(file_path);
new_codebase.files.insert(*file_path, FileInfo::default());
new_codebase.files.insert(
*file_path,
FileInfo {
parser_error: Some(parser_error),
..FileInfo::default()
},
);
invalid_files.lock().unwrap().push(*file_path);
};

let mut tally = files_processed.lock().unwrap();
*tally += 1;
}
};

update_progressbar(*tally, bar.clone());
}
let mut tally = files_processed.lock().unwrap();
*tally += 1;

resolved_names.lock().unwrap().extend(local_resolved_names);
update_progressbar(*tally, bar.clone());
}

let mut codebases = codebases.lock().unwrap();
codebases.push(new_codebase);
});
resolved_names.lock().unwrap().extend(local_resolved_names);

handles.push(handle);
}
let mut codebases = codebases.lock().unwrap();
codebases.push(new_codebase);
});

for handle in handles {
handle.join().unwrap();
}
handles.push(handle);
}

if let Ok(thread_codebases) = Arc::try_unwrap(thread_codebases) {
for thread_codebase in thread_codebases.into_inner().unwrap().into_iter() {
if config.ast_diff {
codebase_diff
.extend(get_diff(&existing_changed_files, &thread_codebase.files));
}
for handle in handles {
handle.join().unwrap();
}

codebase.extend(thread_codebase.clone());
if let Ok(thread_codebases) = Arc::try_unwrap(thread_codebases) {
for thread_codebase in thread_codebases.into_inner().unwrap().into_iter() {
if config.ast_diff {
codebase_diff.extend(get_diff(&existing_changed_files, &thread_codebase.files));
}

codebase.extend(thread_codebase.clone());
}
}

Expand Down

0 comments on commit 78380d4

Please sign in to comment.