free some mem but shoold do that instant with a Mutex<File>

src/main.rs

@@ -1,5 +1,5 @@
 use std::error::Error;
-use std::fs::File;
+use std::fs::{File, OpenOptions};
 use std::io::{BufReader, BufRead};
 use std::sync::{Arc, Mutex};
 
@@ -15,6 +15,7 @@ use indicatif::{ProgressBar, ProgressStyle};
 fn main() -> Result<(), Box<dyn Error>> {
     let filename = "/Turbine-pool/LAL-T_ChIP/data/all_sub.tsv";
     let enhancer_prefix = "EH38E";
+    let res_tsv = "/Turbine-pool/LAL-T_ChIP/data/correlations_enh_rnas.tsv";
 
     let file = File::open(filename)?;
     let reader = BufReader::new(file);
@@ -54,6 +55,19 @@ fn main() -> Result<(), Box<dyn Error>> {
             .template("[{elapsed_precise}] [{bar:40.cyan/blue}] {pos}/{len} ({eta}) {msg}")?
     );
 
+    let mut rna_keys: Vec<&String> = rna_set.keys().collect();
+    rna_keys.sort();
+
+    let mut file = File::create(res_tsv)?;
+
+    // Write the header row
+    write!(file, "ID\t")?; 
+    for rna_key in &rna_keys {
+        write!(file, "{}\t", rna_key)?;
+    }
+    writeln!(file)?; // \n
+
+
     // let best_corr: Arc<Mutex<HashMap<String, (f32, Vec<String>)>>> = Arc::new(Mutex::new(HashMap::new()));
 
     let enhancers_to_rnas: Arc<Mutex<HashMap<String, HashMap<String, f32>>>> = Arc::new(Mutex::new(HashMap::new()));
@@ -103,6 +117,32 @@ fn main() -> Result<(), Box<dyn Error>> {
         //     best_corr_guard.insert(a_k.to_string(), (*min, minimal_keys));
             
         // }
+
+        if enhancers_to_rnas_guard.len() > 100 {
+            bar.set_message(format!("Writing {}", res_tsv));
+            let mut file = OpenOptions::new()
+                .write(true)
+                .append(true)
+                .open(res_tsv)
+                .unwrap();
+
+            // Iterate over the enhancers
+            for (enhancer, inner_map) in enhancers_to_rnas_guard.iter() {
+                write!(file, "{}\t", enhancer).unwrap();
+
+                // Iterate over the RNA keys
+                for rna_key in &rna_keys {
+                    // Write the value if present, otherwise write an empty cell
+                    if let Some(value) = inner_map.get(*rna_key) {
+                        write!(file, "{}\t", value).unwrap();
+                    } else {
+                        write!(file, "\t").unwrap();
+                    }
+                }
+                writeln!(file).unwrap();
+            }
+            enhancers_to_rnas_guard.clear();
+        }
     });
     bar.finish();
 
@@ -115,18 +155,7 @@ fn main() -> Result<(), Box<dyn Error>> {
             .template("[{elapsed_precise}] [{bar:40.cyan/blue}] {pos}/{len} ({eta}) {msg}")?
     );
     bar.set_message("Writing output.tsv");
-    let mut rna_keys: Vec<&String> = rna_set.keys().collect();
-    rna_keys.sort();
-
-    let mut file = File::create("output.tsv")?;
-
-    // Write the header row
-    write!(file, "ID\t")?; 
-    for rna_key in &rna_keys {
-        write!(file, "{}\t", rna_key)?;
-    }
-    writeln!(file)?; // \n
-
+    
 
     // Iterate over the enhancers
     for (enhancer, inner_map) in enhancers_to_rnas_guard.iter() {