Skip to content

Commit 9d7c67b

Browse files
jfrolichRoland Peelen
jfrolich
authored and
Roland Peelen
committed
⚡ - Efficient tree traversal (#12)
1 parent 4efe6c7 commit 9d7c67b

File tree

3 files changed

+100
-120
lines changed

3 files changed

+100
-120
lines changed

‎src/build.rs

Lines changed: 94 additions & 119 deletions
Original file line numberDiff line numberDiff line change
@@ -375,6 +375,12 @@ pub fn generate_asts<'a>(
375375
results
376376
.into_iter()
377377
.for_each(|(module_name, ast_path, iast_path, deps, _namespace)| {
378+
deps.iter().for_each(|dep_name| {
379+
if let Some(module) = modules.get_mut(dep_name) {
380+
module.reverse_deps.insert(module_name.to_string());
381+
}
382+
});
383+
378384
if let Some(module) = modules.get_mut(&module_name) {
379385
module.deps = deps;
380386
match ast_path {
@@ -508,6 +514,7 @@ pub fn parse_packages(
508514
Module {
509515
source_type: SourceType::MlMap(MlMap { dirty: false }),
510516
deps: deps,
517+
reverse_deps: AHashSet::new(),
511518
package: package.to_owned(),
512519
compile_dirty: false,
513520
},
@@ -559,6 +566,7 @@ pub fn parse_packages(
559566
interface: None,
560567
}),
561568
deps: AHashSet::new(),
569+
reverse_deps: AHashSet::new(),
562570
package: package.to_owned(),
563571
compile_dirty: true,
564572
});
@@ -596,6 +604,7 @@ pub fn parse_packages(
596604
}),
597605
}),
598606
deps: AHashSet::new(),
607+
reverse_deps: AHashSet::new(),
599608
package: package.to_owned(),
600609
compile_dirty: true,
601610
});
@@ -868,6 +877,7 @@ pub fn build(path: &str) -> Result<AHashMap<std::string::String, Module>, ()> {
868877
let timing_package_tree = Instant::now();
869878
let packages = package_tree::make(&project_root);
870879
let timing_package_tree_elapsed = timing_package_tree.elapsed();
880+
871881
println!(
872882
"{}\r{} {}Built package tree in {:.2}s",
873883
LINE_CLEAR,
@@ -954,28 +964,15 @@ pub fn build(path: &str) -> Result<AHashMap<std::string::String, Module>, ()> {
954964
}
955965
}
956966

957-
let pb = ProgressBar::new(modules.len().try_into().unwrap());
958-
pb.set_style(
959-
ProgressStyle::with_template(&format!(
960-
"{} {} Compiling... {{wide_bar}} {{pos}}/{{len}} {{msg}}",
961-
style("[5/5]").bold().dim(),
962-
SWORDS
963-
))
964-
.unwrap(),
965-
);
966967
let start_compiling = Instant::now();
967968

968969
let mut compiled_modules = AHashSet::<String>::new();
969-
// println!("Clean modules:");
970970
let dirty_modules = modules
971971
.iter()
972972
.filter_map(|(module_name, module)| {
973-
// if is_dirty(module) {
974973
if module.compile_dirty {
975-
// println!("> {}", module_name);
976974
Some(module_name.to_owned())
977975
} else {
978-
// println!("> {}", module_name);
979976
None
980977
}
981978
})
@@ -993,108 +990,81 @@ pub fn build(path: &str) -> Result<AHashMap<std::string::String, Module>, ()> {
993990
let mut files_current_loop_count;
994991
let mut compile_errors = "".to_string();
995992
let mut compile_warnings = "".to_string();
996-
let total_modules = modules.len();
997993
let mut num_compiled_modules = 0;
998994
let mut sorted_modules = modules
999995
.iter()
1000996
.map(|(module_name, _)| module_name.to_owned())
1001997
.collect::<Vec<String>>();
1002998
sorted_modules.sort();
1003999

1000+
// for module in dirty_modules.clone() {
1001+
// println!("dirty module: {}", module);
1002+
// }
1003+
1004+
// this is the whole "compile universe" all modules that might be dirty
1005+
// we get this by traversing from the dirty modules to all the modules that
1006+
// are dependent on them
1007+
let mut compile_universe = dirty_modules.clone();
1008+
1009+
let mut current_step_modules = dirty_modules.clone();
10041010
loop {
1005-
if dirty_modules.len() == 0 {
1006-
break;
1011+
let mut reverse_deps: AHashSet<String> = AHashSet::new();
1012+
for dirty_module in current_step_modules.iter() {
1013+
reverse_deps.extend(modules.get(dirty_module).unwrap().reverse_deps.clone());
10071014
}
1008-
let mut indirectly_dirty_modules = dirty_modules.clone();
1009-
let mut checked_modules = AHashSet::with_capacity(modules.len());
1010-
loop {
1011-
let mut num_checked_modules = 0;
1012-
for (module_name, module) in modules.iter() {
1013-
if !checked_modules.contains(module_name) {
1014-
num_checked_modules += 1;
1015-
if module.deps.is_subset(&checked_modules) {
1016-
checked_modules.insert(module_name.to_string());
1017-
let is_dirty = module.deps.iter().any(|dep| {
1018-
if clean_modules.contains(dep) {
1019-
return false;
1020-
}
1021-
if indirectly_dirty_modules.contains(dep) {
1022-
return true;
1023-
}
1024-
return false;
1025-
});
1026-
// if !module.deps.is_disjoint(&indirectly_dirty_modules) {
1027-
// indirectly_dirty_modules.insert(module_name.to_string());
1028-
// }
1029-
if is_dirty {
1030-
indirectly_dirty_modules.insert(module_name.to_string());
1031-
}
1032-
}
1033-
}
1034-
}
1035-
if num_checked_modules == 0 {
1036-
break;
1037-
}
1015+
current_step_modules = reverse_deps
1016+
.difference(&compile_universe)
1017+
.map(|s| s.to_string())
1018+
.collect::<AHashSet<String>>();
1019+
compile_universe.extend(current_step_modules.clone());
1020+
if current_step_modules.is_empty() {
1021+
break;
10381022
}
1039-
// let to_check_modules: Vec<(&String, &Module)> = modules
1040-
// .iter()
1041-
// .filter(|(module_name, _)| !checked_modules.contains(*module_name))
1042-
// .collect();
1043-
// to_check_modules.iter().for_each(|(module_name, _)| {
1044-
// let _ = checked_modules.insert(module_name.to_string());
1045-
// });
1046-
// Parallel version -- much faster in a debug build but slower in a release build...
1047-
// to_check_modules
1048-
// .par_iter()
1049-
// .map(|(module_name, module)| {
1050-
// if module.deps.is_subset(&checked_modules) {
1051-
// // checked_modules.insert(module_name.to_string());
1052-
// let is_dirty = module.deps.iter().any(|dep| {
1053-
// if clean_modules.contains(dep) {
1054-
// return false;
1055-
// }
1056-
// if indirectly_dirty_modules.contains(dep) {
1057-
// return true;
1058-
// }
1059-
// return false;
1060-
// });
1061-
1062-
// if is_dirty {
1063-
// return Some(module_name);
1064-
// }
1065-
// }
1066-
// return None;
1067-
// })
1068-
// .filter_map(|x| x)
1069-
// .collect::<Vec<&&String>>()
1070-
// .iter()
1071-
// .for_each(|module_name| {
1072-
// indirectly_dirty_modules.insert(module_name.to_string());
1073-
// });
1074-
// if to_check_modules.len() == 0 {
1075-
// break;
1076-
// }
1023+
}
1024+
let pb = ProgressBar::new(compile_universe.len().try_into().unwrap());
1025+
pb.set_style(
1026+
ProgressStyle::with_template(&format!(
1027+
"{} {} Compiling... {{wide_bar}} {{pos}}/{{len}} {{msg}}",
1028+
style("[5/5]").bold().dim(),
1029+
SWORDS
1030+
))
1031+
.unwrap(),
1032+
);
10771033

1034+
// start off with all modules that have no deps in this compile universe
1035+
let mut in_progress_modules = compile_universe
1036+
.iter()
1037+
.filter(|module_name| {
1038+
let module = modules.get(*module_name).unwrap();
1039+
module.deps.intersection(&compile_universe).count() == 0
1040+
})
1041+
.map(|module_name| module_name.to_string())
1042+
.collect::<AHashSet<String>>();
1043+
1044+
loop {
10781045
files_current_loop_count = 0;
10791046
loop_count += 1;
10801047

10811048
info!(
10821049
"Compiled: {} out of {}. Compile loop: {}",
10831050
files_total_count,
1084-
modules.len(),
1051+
compile_universe.len(),
10851052
loop_count,
10861053
);
10871054

1088-
sorted_modules
1089-
// .iter()
1055+
in_progress_modules
1056+
.clone()
10901057
.par_iter()
10911058
.map(|module_name| {
10921059
let module = modules.get(module_name).unwrap();
1093-
if module.deps.is_subset(&compiled_modules)
1094-
&& !compiled_modules.contains(module_name)
1060+
// all dependencies that we care about are compiled
1061+
if module
1062+
.deps
1063+
.intersection(&compile_universe)
1064+
.all(|dep| compiled_modules.contains(dep))
10951065
{
1096-
if !indirectly_dirty_modules.contains(module_name) {
1097-
// we are sure we don't have to compile this, so we can mark it as compiled
1066+
if !module.compile_dirty {
1067+
// we are sure we don't have to compile this, so we can mark it as compiled and clean
10981068
return Some((
10991069
module_name.to_string(),
11001070
Ok(None),
@@ -1145,10 +1115,6 @@ pub fn build(path: &str) -> Result<AHashMap<std::string::String, Module>, ()> {
11451115
}
11461116
_ => None,
11471117
};
1148-
// if let Some(Err(error)) = interface_result.to_owned() {
1149-
// println!("{}", error);
1150-
// panic!("Interface compilation error!");
1151-
// }
11521118
let result = compile_file(
11531119
&module.package.name,
11541120
&helpers::get_ast_path(
@@ -1168,19 +1134,7 @@ pub fn build(path: &str) -> Result<AHashMap<std::string::String, Module>, ()> {
11681134

11691135
let is_clean = match (cmi_digest, cmi_digest_after) {
11701136
(Some(cmi_digest), Some(cmi_digest_after)) => {
1171-
if cmi_digest.eq(&cmi_digest_after) {
1172-
// println!(
1173-
// "{} is clean -- {:x} {:x}",
1174-
// cmi_path, cmi_digest, cmi_digest_after
1175-
// );
1176-
true
1177-
} else {
1178-
// println!(
1179-
// "{} is dirty -- {:x} {:x}",
1180-
// cmi_path, cmi_digest, cmi_digest_after
1181-
// );
1182-
false
1183-
}
1137+
cmi_digest.eq(&cmi_digest_after)
11841138
}
11851139

11861140
_ => false,
@@ -1210,12 +1164,15 @@ pub fn build(path: &str) -> Result<AHashMap<std::string::String, Module>, ()> {
12101164
.iter()
12111165
.for_each(|result| match result {
12121166
Some((module_name, result, interface_result, is_clean, is_compiled)) => {
1167+
in_progress_modules.remove(module_name);
1168+
12131169
if !(log_enabled!(Info)) {
12141170
pb.inc(1);
12151171
}
12161172
if *is_compiled {
12171173
num_compiled_modules += 1;
12181174
}
1175+
12191176
files_current_loop_count += 1;
12201177
compiled_modules.insert(module_name.to_string());
12211178

@@ -1224,6 +1181,21 @@ pub fn build(path: &str) -> Result<AHashMap<std::string::String, Module>, ()> {
12241181
clean_modules.insert(module_name.to_string());
12251182
}
12261183

1184+
let module_reverse_deps =
1185+
modules.get(module_name).unwrap().reverse_deps.clone();
1186+
1187+
// if not clean -- compile modules that depend on this module
1188+
for dep in module_reverse_deps.iter() {
1189+
let dep_module = modules.get_mut(dep).unwrap();
1190+
// mark the reverse dep as dirty when the source is not clean
1191+
if !*is_clean {
1192+
dep_module.compile_dirty = true;
1193+
}
1194+
if !compiled_modules.contains(dep) {
1195+
in_progress_modules.insert(dep.to_string());
1196+
}
1197+
}
1198+
12271199
let module = modules.get_mut(module_name).unwrap();
12281200

12291201
match module.source_type {
@@ -1263,13 +1235,16 @@ pub fn build(path: &str) -> Result<AHashMap<std::string::String, Module>, ()> {
12631235

12641236
files_total_count += files_current_loop_count;
12651237

1266-
if files_total_count == total_modules {
1238+
// if files_total_count == total_modules {
1239+
// break;
1240+
// }
1241+
// if files_current_loop_count == 0 {
1242+
// // we probably want to find the cycle(s), and give a helpful error message here
1243+
// compile_errors.push_str("Can't continue... Dependency cycle\n")
1244+
// }
1245+
if in_progress_modules.len() == 0 {
12671246
break;
12681247
}
1269-
if files_current_loop_count == 0 {
1270-
// we probably want to find the cycle(s), and give a helpful error message here
1271-
compile_errors.push_str("Can't continue... Dependency cycle\n")
1272-
}
12731248
if compile_errors.len() > 0 {
12741249
break;
12751250
};
@@ -1279,6 +1254,10 @@ pub fn build(path: &str) -> Result<AHashMap<std::string::String, Module>, ()> {
12791254
pb.finish();
12801255
clean::cleanup_after_build(&modules, &compiled_modules, &all_modules, &project_root);
12811256
if compile_errors.len() > 0 {
1257+
if helpers::contains_ascii_characters(&compile_warnings) {
1258+
println!("{}", &compile_warnings);
1259+
}
1260+
println!("{}", &compile_errors);
12821261
println!(
12831262
"{}\r{} {}Compiled {} modules in {:.2}s",
12841263
LINE_CLEAR,
@@ -1287,12 +1266,11 @@ pub fn build(path: &str) -> Result<AHashMap<std::string::String, Module>, ()> {
12871266
num_compiled_modules,
12881267
compile_duration.as_secs_f64()
12891268
);
1269+
return Err(());
1270+
} else {
12901271
if helpers::contains_ascii_characters(&compile_warnings) {
12911272
println!("{}", &compile_warnings);
12921273
}
1293-
println!("{}", &compile_errors);
1294-
return Err(());
1295-
} else {
12961274
println!(
12971275
"{}\r{} {}Compiled {} modules in {:.2}s",
12981276
LINE_CLEAR,
@@ -1301,9 +1279,6 @@ pub fn build(path: &str) -> Result<AHashMap<std::string::String, Module>, ()> {
13011279
num_compiled_modules,
13021280
compile_duration.as_secs_f64()
13031281
);
1304-
if helpers::contains_ascii_characters(&compile_warnings) {
1305-
println!("{}", &compile_warnings);
1306-
}
13071282
}
13081283

13091284
let timing_total_elapsed = timing_total.elapsed();

‎src/build_types.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ pub enum SourceType {
5656
pub struct Module {
5757
pub source_type: SourceType,
5858
pub deps: AHashSet<String>,
59+
pub reverse_deps: AHashSet<String>,
5960
pub package: package_tree::Package,
6061
pub compile_dirty: bool,
6162
}

‎src/clean.rs

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,13 +41,16 @@ fn remove_asts(source_file: &str, package_name: &str, namespace: &Option<String>
4141
));
4242
}
4343

44+
fn remove_mjs_file(source_file: &str) {
45+
let _ = std::fs::remove_file(helpers::change_extension(source_file, "mjs"));
46+
}
47+
4448
fn remove_compile_assets(
4549
source_file: &str,
4650
package_name: &str,
4751
namespace: &Option<String>,
4852
root_path: &str,
4953
) {
50-
let _ = std::fs::remove_file(helpers::change_extension(source_file, "mjs"));
5154
// optimization
5255
// only issue cmti if htere is an interfacce file
5356
for extension in &["cmj", "cmi", "cmt", "cmti"] {
@@ -192,6 +195,7 @@ pub fn cleanup_previous_build(
192195
package_namespace,
193196
root_path,
194197
);
198+
remove_mjs_file(&res_file_location)
195199
});
196200

197201
ast_rescript_file_locations

0 commit comments

Comments
 (0)