mod.rs 33 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994
  1. //! Genome-wide uniqueness index based on suffix arrays.
  2. //!
  3. //! This module computes, stores, and queries minimal unique sequence lengths
  4. //! across a genome. Multi-contig references are indexed by concatenating the
  5. //! selected contigs with reserved separator bytes, building one global suffix
  6. //! array, and constraining all reported intervals to a single contig. The
  7. //! coordinates exposed by [`GenomeIndex`] remain contig-local.
  8. //!
  9. //! For each genomic position `pos`, the index stores the shortest sequence
  10. //! starting at `pos` that occurs exactly once in the genome:
  11. //!
  12. //! ```text
  13. //! genome[pos .. pos + k)
  14. //! ```
  15. //!
  16. //! where `k` is the minimal unique length at `pos`.
  17. //!
  18. //! Internally, uniqueness is computed from a suffix array and LCP
  19. //! (Longest Common Prefix) array, allowing construction in `O(n log n)`
  20. //! time and constant-time uniqueness queries.
  21. //!
  22. //! # Query types
  23. //!
  24. //! The index supports two complementary views of uniqueness:
  25. //!
  26. //! ## Unique intervals containing a position
  27. //!
  28. //! Given a genomic position, the index can return a unique interval that
  29. //! contains that position.
  30. //!
  31. //! ```text
  32. //! start <= pos < end
  33. //! ```
  34. //!
  35. //! Several intervals may satisfy this condition; [`AnchorBias`] controls
  36. //! how a candidate interval is selected.
  37. //!
  38. //! ## Independent left/right uniqueness
  39. //!
  40. //! The index can also report how much sequence is required on either side
  41. //! of a position to obtain uniqueness independently.
  42. //!
  43. //! For a position `pos`:
  44. //!
  45. //! ```text
  46. //! before: genome[pos - k .. pos)
  47. //! after : genome[pos .. pos + k)
  48. //! ```
  49. //!
  50. //! [`UniqueContext`] returns the smallest unique sequence ending at `pos`
  51. //! (`before`) and the smallest unique sequence starting at `pos` (`after`).
  52. //!
  53. //! To support efficient left-side queries, the index stores uniqueness
  54. //! information for both the forward genome and its reverse complement-free
  55. //! reversal.
  56. //!
  57. //! # Caching
  58. //!
  59. //! Construction can be expensive for large genomes. The module therefore
  60. //! supports disk-backed caching of precomputed uniqueness vectors.
  61. //!
  62. //! Cached data are keyed by genome metadata and loaded transparently when
  63. //! available.
  64. //!
  65. //! # Main types
  66. //!
  67. //! - [`UniquenessIndex`] — uniqueness index for a single sequence.
  68. //! - [`GenomeIndex`] — multi-contig genome-wide index.
  69. //! - [`UniqueInterval`] — unique interval containing a queried position.
  70. //! - [`UniqueContext`] — independent left/right uniqueness lengths.
  71. //! - [`GenomicPos`] — genomic coordinate used by [`GenomeIndex`].
  72. use log::{info, warn};
  73. use rayon::prelude::*;
  74. use std::io::{self};
  75. use std::path::Path;
  76. use std::time::Instant;
  77. pub type GenomeCoord = u32;
  78. pub type UniqueLen = u32;
  79. pub const NOT_UNIQUE: UniqueLen = u32::MAX;
  80. mod cache;
  81. mod genome_index;
  82. mod suffix;
  83. pub use cache::CacheKey;
  84. pub use genome_index::{ContigSpan, GenomeIndex, GenomicPos};
  85. use cache::{load_from_cache, save_to_cache, MappedUniqueLenCache};
  86. use suffix::build_suffix_array_and_lcp;
  87. /// Pure computation: genome bytes → min_unique_len vector.
  88. ///
  89. /// `contig_end_at[p]` is the exclusive end coordinate of the contig containing
  90. /// `p`. Separator positions must use `contig_end_at[p] = p`, which makes every
  91. /// separator position non-queryable.
  92. fn build_min_unique_len(genome: &[u8], contig_end_at: &[GenomeCoord]) -> Vec<UniqueLen> {
  93. let n = genome.len();
  94. assert_eq!(n, contig_end_at.len());
  95. assert!(
  96. n <= u32::MAX as usize,
  97. "uniqueness index only supports sequences up to u32::MAX bases"
  98. );
  99. info!(
  100. "[uniqueness] computing minimal unique lengths ({:.1} Mbp)",
  101. n as f64 / 1e6
  102. );
  103. let total_started = Instant::now();
  104. let (sa, lcp) = build_suffix_array_and_lcp(genome);
  105. info!("[uniqueness] building suffix rank array");
  106. let started = Instant::now();
  107. let mut rank = vec![0u32; n];
  108. for (i, &s) in sa.iter().enumerate() {
  109. rank[s as usize] = i as u32;
  110. }
  111. drop(sa);
  112. info!(
  113. "[uniqueness] suffix rank array built in {:.1}s",
  114. started.elapsed().as_secs_f64()
  115. );
  116. info!("[uniqueness] computing boundary-aware unique length vector");
  117. let started = Instant::now();
  118. let min_unique_len = (0..n)
  119. .into_par_iter()
  120. .map(|p| {
  121. let r = rank[p] as usize;
  122. let left = if r > 0 { lcp[r] } else { 0 };
  123. let right = if r + 1 < n { lcp[r + 1] } else { 0 };
  124. let Some(k) = left.max(right).checked_add(1) else {
  125. return NOT_UNIQUE;
  126. };
  127. let Some(end) = (p as GenomeCoord).checked_add(k) else {
  128. return NOT_UNIQUE;
  129. };
  130. if end <= contig_end_at[p] {
  131. k
  132. } else {
  133. NOT_UNIQUE
  134. }
  135. })
  136. .collect();
  137. drop(rank);
  138. drop(lcp);
  139. info!(
  140. "[uniqueness] unique length vector computed in {:.1}s (total {:.1}s)",
  141. started.elapsed().as_secs_f64(),
  142. total_started.elapsed().as_secs_f64()
  143. );
  144. min_unique_len
  145. }
  146. enum UniqueLenStore {
  147. Owned(Vec<UniqueLen>),
  148. Mapped(MappedUniqueLenCache),
  149. }
  150. impl UniqueLenStore {
  151. fn get(&self, index: usize) -> UniqueLen {
  152. match self {
  153. Self::Owned(values) => values[index],
  154. Self::Mapped(values) => values.get(index),
  155. }
  156. }
  157. #[cfg(test)]
  158. fn len(&self) -> usize {
  159. match self {
  160. Self::Owned(values) => values.len(),
  161. Self::Mapped(values) => values.len(),
  162. }
  163. }
  164. #[cfg(test)]
  165. fn to_vec(&self) -> Vec<UniqueLen> {
  166. (0..self.len()).map(|idx| self.get(idx)).collect()
  167. }
  168. }
  169. #[derive(Debug, Clone, Copy)]
  170. pub struct UniqueContext {
  171. /// Bases required before `pos`.
  172. pub before: Option<UniqueLen>,
  173. /// Bases required from `pos` onward.
  174. pub after: Option<UniqueLen>,
  175. }
  176. #[derive(Debug, Clone, PartialEq, Eq)]
  177. pub struct UniqueInterval {
  178. pub start: GenomeCoord,
  179. pub end: GenomeCoord,
  180. }
  181. impl UniqueInterval {
  182. pub fn len(&self) -> UniqueLen {
  183. self.end - self.start
  184. }
  185. }
  186. #[derive(Debug, Clone, Copy)]
  187. pub enum AnchorBias {
  188. Left,
  189. Right,
  190. Auto,
  191. }
  192. pub struct UniquenessIndex {
  193. min_unique_len: UniqueLenStore,
  194. rev_min_unique_len: UniqueLenStore,
  195. genome_len: GenomeCoord,
  196. }
  197. impl UniquenessIndex {
  198. /// Build from genome bytes, using a disk cache keyed on `cache_key`.
  199. /// Cache lives in `cache_dir` (created if absent).
  200. /// If the cache file exists and is valid, loading is O(n) read — no recomputation.
  201. pub fn build_with_cache(
  202. genome: &[u8],
  203. contig_end_at: &[GenomeCoord],
  204. rev_contig_end_at: &[GenomeCoord],
  205. key: &CacheKey,
  206. cache_dir: &Path,
  207. ) -> io::Result<Self> {
  208. assert_eq!(genome.len(), contig_end_at.len());
  209. assert_eq!(genome.len(), rev_contig_end_at.len());
  210. std::fs::create_dir_all(cache_dir)?;
  211. let fwd_path = cache_dir.join(key.file_name());
  212. let rev_path = cache_dir.join(format!("rev.{}", key.file_name()));
  213. let min_unique_len = match load_from_cache(&fwd_path, genome.len()) {
  214. Ok(mul) => {
  215. info!(
  216. "[cache] forward uniqueness cache hit: {}",
  217. fwd_path.display()
  218. );
  219. mul
  220. }
  221. Err(e) => {
  222. if fwd_path.exists() {
  223. warn!("[cache] invalid forward ({e}), rebuilding...");
  224. } else {
  225. info!(
  226. "[cache] forward uniqueness cache miss: {}",
  227. fwd_path.display()
  228. );
  229. }
  230. let mul = build_min_unique_len(genome, contig_end_at);
  231. info!(
  232. "[cache] saving forward uniqueness cache: {}",
  233. fwd_path.display()
  234. );
  235. save_to_cache(&fwd_path, &mul)?;
  236. mul
  237. }
  238. };
  239. let rev_min_unique_len = match load_from_cache(&rev_path, genome.len()) {
  240. Ok(mul) => {
  241. info!(
  242. "[cache] reverse uniqueness cache hit: {}",
  243. rev_path.display()
  244. );
  245. mul
  246. }
  247. Err(e) => {
  248. if rev_path.exists() {
  249. warn!("[cache] invalid reverse ({e}), rebuilding...");
  250. } else {
  251. info!(
  252. "[cache] reverse uniqueness cache miss: {}",
  253. rev_path.display()
  254. );
  255. }
  256. info!("[uniqueness] reversing genome for reverse uniqueness pass");
  257. let started = Instant::now();
  258. let mut rev = genome.to_vec();
  259. rev.reverse();
  260. info!(
  261. "[uniqueness] reversed genome prepared in {:.1}s",
  262. started.elapsed().as_secs_f64()
  263. );
  264. let mul = build_min_unique_len(&rev, rev_contig_end_at);
  265. info!(
  266. "[cache] saving reverse uniqueness cache: {}",
  267. rev_path.display()
  268. );
  269. save_to_cache(&rev_path, &mul)?;
  270. mul
  271. }
  272. };
  273. Ok(Self::from_raw(min_unique_len, rev_min_unique_len))
  274. }
  275. /// Construct directly from a precomputed `min_unique_len` vector.
  276. /// Used when loading from disk cache — skips SA/LCP computation entirely.
  277. pub fn from_raw(min_unique_len: Vec<UniqueLen>, rev_min_unique_len: Vec<UniqueLen>) -> Self {
  278. assert_eq!(min_unique_len.len(), rev_min_unique_len.len());
  279. assert!(
  280. min_unique_len.len() <= u32::MAX as usize,
  281. "uniqueness index only supports sequences up to u32::MAX bases"
  282. );
  283. let genome_len = min_unique_len.len() as u32;
  284. Self {
  285. min_unique_len: UniqueLenStore::Owned(min_unique_len),
  286. rev_min_unique_len: UniqueLenStore::Owned(rev_min_unique_len),
  287. genome_len,
  288. }
  289. }
  290. pub(crate) fn from_mapped(
  291. min_unique_len: MappedUniqueLenCache,
  292. rev_min_unique_len: MappedUniqueLenCache,
  293. ) -> Self {
  294. assert_eq!(min_unique_len.len(), rev_min_unique_len.len());
  295. assert!(
  296. min_unique_len.len() <= u32::MAX as usize,
  297. "uniqueness index only supports sequences up to u32::MAX bases"
  298. );
  299. let genome_len = min_unique_len.len() as u32;
  300. Self {
  301. min_unique_len: UniqueLenStore::Mapped(min_unique_len),
  302. rev_min_unique_len: UniqueLenStore::Mapped(rev_min_unique_len),
  303. genome_len,
  304. }
  305. }
  306. /// Build without cache (e.g. for tests or one-shot use).
  307. pub fn build(genome: &[u8]) -> Self {
  308. let n = genome.len() as GenomeCoord;
  309. let contig_end_at = vec![n; genome.len()];
  310. let min_unique_len = build_min_unique_len(genome, &contig_end_at);
  311. let mut rev = genome.to_vec();
  312. rev.reverse();
  313. let rev_contig_end_at = vec![n; genome.len()];
  314. let rev_min_unique_len = build_min_unique_len(&rev, &rev_contig_end_at);
  315. Self::from_raw(min_unique_len, rev_min_unique_len)
  316. }
  317. /// Returns the number of nucleotides before and after `pos` needed to form
  318. /// a unique sequence containing `pos`.
  319. ///
  320. /// The returned flank defines the half-open interval:
  321. ///
  322. /// `[pos - front, pos + back)`
  323. ///
  324. /// where `front` is the number of nucleotides before `pos`, and `back` is the
  325. /// number of nucleotides from `pos` onward, including the nucleotide at `pos`.
  326. ///
  327. /// Returns `None` if no unique sequence containing `pos` can be found.
  328. ///
  329. /// # Panics
  330. ///
  331. /// Panics if `pos >= self.genome_len()`.
  332. pub fn minimal_unique_interval_containing(
  333. &self,
  334. pos: GenomeCoord,
  335. bias: AnchorBias,
  336. ) -> Option<UniqueInterval> {
  337. assert!(pos < self.genome_len);
  338. let pos_usize = pos as usize;
  339. match bias {
  340. AnchorBias::Left => {
  341. let k = self.min_unique_len.get(pos_usize);
  342. if k == NOT_UNIQUE {
  343. return None;
  344. }
  345. Some(UniqueInterval {
  346. start: pos,
  347. end: pos + k,
  348. })
  349. }
  350. AnchorBias::Right => {
  351. for a in (0..=pos).rev() {
  352. let k = self.min_unique_len.get(a as usize);
  353. if k == NOT_UNIQUE {
  354. continue;
  355. }
  356. if a + k > pos {
  357. return Some(UniqueInterval {
  358. start: a,
  359. end: a + k,
  360. });
  361. }
  362. // a + k <= pos and a is decreasing: gap can only grow, bail
  363. if pos - a >= self.genome_len {
  364. break;
  365. }
  366. }
  367. None
  368. }
  369. AnchorBias::Auto => {
  370. let window = 500;
  371. let mut best: Option<UniqueInterval> = None;
  372. for a in pos.saturating_sub(window)..=pos {
  373. let k = self.min_unique_len.get(a as usize);
  374. if k == NOT_UNIQUE {
  375. continue;
  376. }
  377. if a + k <= pos {
  378. continue;
  379. }
  380. let c = UniqueInterval {
  381. start: a,
  382. end: a + k,
  383. };
  384. best = Some(match best {
  385. None => c,
  386. Some(b) => {
  387. if c.len() < b.len() {
  388. c
  389. } else {
  390. b
  391. }
  392. }
  393. });
  394. }
  395. best
  396. }
  397. }
  398. }
  399. pub fn genome_len(&self) -> GenomeCoord {
  400. self.genome_len
  401. }
  402. /// Returns how much sequence is needed before and after `pos`
  403. /// to obtain uniqueness independently on each side.
  404. ///
  405. /// `before = Some(k)` means:
  406. ///
  407. /// `genome[pos - k .. pos)` is the shortest unique sequence ending at `pos`.
  408. ///
  409. /// `after = Some(k)` means:
  410. ///
  411. /// `genome[pos .. pos + k)` is the shortest unique sequence starting at `pos`.
  412. ///
  413. /// Returns `None` for a side if no unique sequence exists on that side.
  414. ///
  415. /// # Panics
  416. ///
  417. /// Panics if `pos > self.genome_len()`.
  418. pub fn unique_context_at(&self, pos: GenomeCoord) -> UniqueContext {
  419. assert!(pos <= self.genome_len);
  420. let after = if pos == self.genome_len {
  421. None
  422. } else {
  423. match self.min_unique_len.get(pos as usize) {
  424. NOT_UNIQUE => None,
  425. k => Some(k),
  426. }
  427. };
  428. let before = if pos == 0 {
  429. None
  430. } else {
  431. let rev_pos = self.genome_len - pos;
  432. match self.rev_min_unique_len.get(rev_pos as usize) {
  433. NOT_UNIQUE => None,
  434. k => Some(k),
  435. }
  436. };
  437. UniqueContext { before, after }
  438. }
  439. }
  440. // tests/uniqueness.rs
  441. #[cfg(test)]
  442. mod tests {
  443. use std::{fs::File};
  444. use std::io::{BufWriter, Write};
  445. use rust_htslib::bam::{self, Read};
  446. use crate::{
  447. helpers::get_genome_sizes,
  448. uniqueness::genome_index::{GenomeIndex, GenomicPos},
  449. };
  450. use super::*;
  451. fn toy_genome() -> &'static [u8] {
  452. // "banana" — well-known SA example, easy to verify by hand
  453. b"ACGTACGTNNACGT"
  454. }
  455. fn single_contig_end_at(genome: &[u8]) -> Vec<GenomeCoord> {
  456. vec![genome.len() as GenomeCoord; genome.len()]
  457. }
  458. #[test]
  459. fn test_cache_roundtrip() {
  460. let genome = toy_genome();
  461. let dir = std::path::Path::new("/home/t_steimle/tmp/");
  462. let key = CacheKey::from_genome("chr_test", genome);
  463. let contig_end_at = single_contig_end_at(genome);
  464. let idx1 =
  465. UniquenessIndex::build_with_cache(genome, &contig_end_at, &contig_end_at, &key, dir)
  466. .unwrap();
  467. // Second call must hit cache
  468. let idx2 =
  469. UniquenessIndex::build_with_cache(genome, &contig_end_at, &contig_end_at, &key, dir)
  470. .unwrap();
  471. assert_eq!(idx1.min_unique_len.to_vec(), idx2.min_unique_len.to_vec());
  472. }
  473. #[test]
  474. fn test_cache_invalidated_on_length_mismatch() {
  475. let genome1 = b"ACGTACGT".as_slice();
  476. let genome2 = b"ACGTACGTNN".as_slice();
  477. let dir = std::path::Path::new("/home/t_steimle/tmp/");
  478. let key = CacheKey::from_genome("chr_test", genome1);
  479. let contig_end_at = single_contig_end_at(genome1);
  480. UniquenessIndex::build_with_cache(genome1, &contig_end_at, &contig_end_at, &key, dir)
  481. .unwrap();
  482. // genome2 has different len → load_from_cache returns Err → rebuild
  483. let key2 = CacheKey::from_genome("chr_test", genome2);
  484. // different hash → different file, no collision
  485. assert_ne!(key.file_name(), key2.file_name());
  486. }
  487. #[test]
  488. fn old_cache_format_is_rejected() {
  489. let path = std::env::temp_dir().join(format!(
  490. "pandora_old_uniqueness_cache_{}.uniqidx",
  491. std::process::id()
  492. ));
  493. let mut bytes = Vec::new();
  494. bytes.extend_from_slice(b"UNIQIDX1");
  495. bytes.extend_from_slice(&1u32.to_le_bytes());
  496. bytes.extend_from_slice(&1u64.to_le_bytes());
  497. bytes.extend_from_slice(&1u64.to_le_bytes());
  498. std::fs::write(&path, bytes).unwrap();
  499. assert!(cache::load_from_cache(&path, 1).is_err());
  500. let _ = std::fs::remove_file(path);
  501. }
  502. // ── helpers ───────────────────────────────────────────────────────────────
  503. /// Verify that a given interval is actually unique in the genome:
  504. /// the subsequence appears exactly once.
  505. fn assert_unique(genome: &[u8], iv: &UniqueInterval) {
  506. let needle = &genome[iv.start as usize..iv.end as usize];
  507. let count = genome
  508. .windows(needle.len())
  509. .filter(|w| *w == needle)
  510. .count();
  511. assert_eq!(
  512. count,
  513. 1,
  514. "interval [{}, {}) = {:?} appears {count}x in genome — not unique",
  515. iv.start,
  516. iv.end,
  517. std::str::from_utf8(needle).unwrap_or("<binary>"),
  518. );
  519. }
  520. /// Verify that no strictly shorter sub-interval anchored at the same start
  521. /// is also unique (minimality check for Left bias).
  522. fn assert_minimal_left(genome: &[u8], iv: &UniqueInterval) {
  523. if iv.len() <= 1 {
  524. return;
  525. }
  526. let shorter = UniqueInterval {
  527. start: iv.start,
  528. end: iv.end - 1,
  529. };
  530. let needle = &genome[shorter.start as usize..shorter.end as usize];
  531. let count = genome
  532. .windows(needle.len())
  533. .filter(|w| *w == needle)
  534. .count();
  535. assert!(
  536. count > 1,
  537. "interval [{}, {}) = {:?} is shorter and already unique — minimality violated",
  538. shorter.start,
  539. shorter.end,
  540. std::str::from_utf8(needle).unwrap_or("<binary>"),
  541. );
  542. }
  543. /// Same for Right bias: no strictly shorter interval ending at the same end.
  544. fn assert_minimal_right(genome: &[u8], iv: &UniqueInterval) {
  545. if iv.len() <= 1 {
  546. return;
  547. }
  548. let shorter = UniqueInterval {
  549. start: iv.start + 1,
  550. end: iv.end,
  551. };
  552. let needle = &genome[shorter.start as usize..shorter.end as usize];
  553. let count = genome
  554. .windows(needle.len())
  555. .filter(|w| *w == needle)
  556. .count();
  557. assert!(
  558. count > 1,
  559. "interval [{}, {}) = {:?} is shorter and already unique — minimality violated",
  560. shorter.start,
  561. shorter.end,
  562. std::str::from_utf8(needle).unwrap_or("<binary>"),
  563. );
  564. }
  565. /// Assert that no interval of the same length covering pos is shorter
  566. /// (Auto minimality: no other anchor yields a shorter unique window).
  567. fn assert_minimal_auto(genome: &[u8], iv: &UniqueInterval, pos: GenomeCoord) {
  568. // Try all anchors in [pos - len + 1 .. pos] and verify none gives shorter
  569. let window = iv.len().saturating_sub(1);
  570. for a in pos.saturating_sub(window)..=pos {
  571. if a + 1 > genome.len() as GenomeCoord {
  572. break;
  573. }
  574. let max_end = (a + iv.len()).min(genome.len() as GenomeCoord);
  575. for end in (a + 1)..max_end {
  576. let needle = &genome[a as usize..end as usize];
  577. let count = genome
  578. .windows(needle.len())
  579. .filter(|w| *w == needle)
  580. .count();
  581. if count == 1 {
  582. panic!(
  583. "found shorter unique interval [{a}, {end}) len={} \
  584. covering pos={pos}, but Auto returned [{}, {}) len={}",
  585. end - a,
  586. iv.start,
  587. iv.end,
  588. iv.len()
  589. );
  590. }
  591. }
  592. }
  593. }
  594. // ── genome fixtures ───────────────────────────────────────────────────────
  595. /// Simple genome with obvious unique regions.
  596. ///
  597. /// Layout (positions):
  598. /// 0123456789...
  599. /// AAAA TTTT AAAA GCTAGCTA NNNN GCTAGCTA
  600. /// ^unique^ ^repeat^
  601. ///
  602. /// "GCTAGCTA" appears twice → not unique
  603. /// "TTTT" appears once → unique but short
  604. /// The N-run should return None for all biases.
  605. fn genome_simple() -> Vec<u8> {
  606. //0 1 2 3
  607. //0123456789012345678901234567890123456789
  608. b"AAAATTTTAAAAGCTAGCTANNNNNNNNNNNGCTAGCTA".to_vec()
  609. }
  610. /// Genome where Left/Right/Auto produce verifiably different intervals.
  611. ///
  612. /// ACGT ACGT TGCA TGCA AAAA
  613. /// 0 4 8 12 16
  614. ///
  615. /// "ACGTACGT" covers [0..8) — only one occurrence at pos 0
  616. /// "TGCATGCA" covers [8..16) — only one occurrence
  617. /// Repeated motifs force longer unique strings.
  618. fn genome_repeats() -> Vec<u8> {
  619. b"ACGTACGTTGCATGCAAAAACCCCCGGGGG".to_vec()
  620. }
  621. /// Genome constructed so that Left, Right and Auto give three distinct
  622. /// intervals for a single query position.
  623. ///
  624. /// ATCGATCG GGGG ATCGATCG TTTT ATCG CCCC
  625. /// 0 8 12 20 24 28
  626. ///
  627. /// pos=6 (inside first ATCGATCG repeat region):
  628. /// Left → must extend right to break the repeat
  629. /// Right → extends left into earlier unique context
  630. /// Auto → picks whichever anchor gives shortest total span
  631. fn genome_biased() -> Vec<u8> {
  632. //0 1 2 3
  633. //012345678901234567890123456789012345
  634. b"ATCGATCGGGGGGGGATCGATCGTTTTTATCGCCCCCCCC".to_vec()
  635. }
  636. // ── Left bias tests ───────────────────────────────────────────────────────
  637. #[test]
  638. fn left_interval_starts_at_pos() {
  639. let g = genome_repeats();
  640. let idx = UniquenessIndex::build(&g);
  641. for pos in [0, 4, 8, 12] {
  642. if let Some(iv) = idx.minimal_unique_interval_containing(pos, AnchorBias::Left) {
  643. assert_eq!(iv.start, pos, "Left: interval must start at pos={pos}");
  644. assert!(iv.end > pos, "Left: interval must extend past pos");
  645. assert_unique(&g, &iv);
  646. assert_minimal_left(&g, &iv);
  647. }
  648. }
  649. }
  650. #[test]
  651. fn left_interval_is_unique() {
  652. let g = genome_simple();
  653. let idx = UniquenessIndex::build(&g);
  654. // pos=4 is in the TTTT region — unique and short
  655. let iv = idx
  656. .minimal_unique_interval_containing(4, AnchorBias::Left)
  657. .expect("TTTT region should have a unique interval");
  658. assert_eq!(iv.start, 4);
  659. assert_unique(&g, &iv);
  660. assert_minimal_left(&g, &iv);
  661. }
  662. #[test]
  663. fn left_returns_none_for_n_run() {
  664. let g = genome_simple();
  665. let idx = UniquenessIndex::build(&g);
  666. // N-run starts at pos 20 in genome_simple
  667. // Ns produce repeated k-mers → no unique interval anchored there
  668. let result = idx.minimal_unique_interval_containing(22, AnchorBias::Left);
  669. // We don't assert None strictly (an N-run touching unique flanks might
  670. // resolve), but if Some, it must be genuinely unique.
  671. if let Some(iv) = result {
  672. assert_unique(&g, &iv);
  673. }
  674. }
  675. // ── Right bias tests ──────────────────────────────────────────────────────
  676. #[test]
  677. fn right_interval_covers_pos() {
  678. let g = genome_repeats();
  679. let idx = UniquenessIndex::build(&g);
  680. for pos in [5, 10, 15, 20] {
  681. if pos >= g.len() as GenomeCoord {
  682. continue;
  683. }
  684. if let Some(iv) = idx.minimal_unique_interval_containing(pos, AnchorBias::Right) {
  685. assert!(iv.start <= pos, "Right: interval must start ≤ pos={pos}");
  686. assert!(iv.end > pos, "Right: interval must end > pos={pos}");
  687. assert_unique(&g, &iv);
  688. assert_minimal_right(&g, &iv);
  689. }
  690. }
  691. }
  692. #[test]
  693. fn right_interval_ends_as_early_as_possible() {
  694. let g = genome_biased();
  695. let idx = UniquenessIndex::build(&g);
  696. let pos = 6;
  697. let iv_right = idx
  698. .minimal_unique_interval_containing(pos, AnchorBias::Right)
  699. .expect("should find a unique interval with Right bias");
  700. let iv_left = idx
  701. .minimal_unique_interval_containing(pos, AnchorBias::Left)
  702. .expect("should find a unique interval with Left bias");
  703. // Right-biased interval uses a left anchor → end is generally earlier
  704. // than Left-biased interval's end (which is anchored at pos and must
  705. // extend further right to become unique).
  706. // This is not guaranteed in all genomes, but holds for genome_biased.
  707. assert!(
  708. iv_right.end <= iv_left.end,
  709. "Right end={} should be ≤ Left end={} for this genome",
  710. iv_right.end,
  711. iv_left.end,
  712. );
  713. assert_unique(&g, &iv_right);
  714. }
  715. // ── Auto bias tests ───────────────────────────────────────────────────────
  716. #[test]
  717. fn auto_is_shortest_among_all_biases() {
  718. let g = genome_biased();
  719. let idx = UniquenessIndex::build(&g);
  720. for pos in 0..g.len() as GenomeCoord {
  721. let auto = idx.minimal_unique_interval_containing(pos, AnchorBias::Auto);
  722. let left = idx.minimal_unique_interval_containing(pos, AnchorBias::Left);
  723. let right = idx.minimal_unique_interval_containing(pos, AnchorBias::Right);
  724. if let Some(ref a) = auto {
  725. assert_unique(&g, a);
  726. assert_minimal_auto(&g, a, pos);
  727. // Auto must be ≤ Left in length
  728. if let Some(ref l) = left {
  729. assert!(
  730. a.len() <= l.len(),
  731. "pos={pos}: Auto len={} > Left len={} — Auto must be minimal",
  732. a.len(),
  733. l.len()
  734. );
  735. }
  736. // Auto must be ≤ Right in length
  737. if let Some(ref r) = right {
  738. assert!(
  739. a.len() <= r.len(),
  740. "pos={pos}: Auto len={} > Right len={} — Auto must be minimal",
  741. a.len(),
  742. r.len()
  743. );
  744. }
  745. }
  746. }
  747. }
  748. #[test]
  749. fn auto_covers_pos() {
  750. let g = genome_repeats();
  751. let idx = UniquenessIndex::build(&g);
  752. for pos in 0..g.len() as GenomeCoord {
  753. if let Some(iv) = idx.minimal_unique_interval_containing(pos, AnchorBias::Auto) {
  754. assert!(
  755. iv.start <= pos && iv.end > pos,
  756. "Auto interval [{}, {}) must contain pos={pos}",
  757. iv.start,
  758. iv.end
  759. );
  760. assert_unique(&g, &iv);
  761. }
  762. }
  763. }
  764. // ── Cross-bias consistency ────────────────────────────────────────────────
  765. #[test]
  766. fn all_biases_return_unique_intervals() {
  767. let g = genome_simple();
  768. let idx = UniquenessIndex::build(&g);
  769. for pos in 0..g.len() as GenomeCoord {
  770. for bias in [AnchorBias::Left, AnchorBias::Right, AnchorBias::Auto] {
  771. if let Some(iv) = idx.minimal_unique_interval_containing(pos, bias) {
  772. assert!(iv.start <= pos, "start must be ≤ pos");
  773. assert!(iv.end > pos, "end must be > pos");
  774. assert!(iv.end <= g.len() as GenomeCoord, "end must be in bounds");
  775. assert_unique(&g, &iv);
  776. }
  777. }
  778. }
  779. }
  780. #[test]
  781. fn left_start_constraint_holds_for_all_positions() {
  782. let g = genome_repeats();
  783. let idx = UniquenessIndex::build(&g);
  784. for pos in 0..g.len() as GenomeCoord {
  785. if let Some(iv) = idx.minimal_unique_interval_containing(pos, AnchorBias::Left) {
  786. assert_eq!(
  787. iv.start, pos,
  788. "Left bias must anchor at pos={pos}, got start={}",
  789. iv.start
  790. );
  791. }
  792. }
  793. }
  794. // ── Cache integration ─────────────────────────────────────────────────────
  795. #[test]
  796. fn cache_preserves_query_results() {
  797. let g = genome_biased();
  798. let dir = Path::new("/home/t_steimle/tmp/");
  799. let key = CacheKey::from_genome("test_contig", &g);
  800. let contig_end_at = single_contig_end_at(&g);
  801. let idx_built =
  802. UniquenessIndex::build_with_cache(&g, &contig_end_at, &contig_end_at, &key, dir)
  803. .unwrap();
  804. let idx_cached =
  805. UniquenessIndex::build_with_cache(&g, &contig_end_at, &contig_end_at, &key, dir)
  806. .unwrap();
  807. for pos in 0..g.len() as GenomeCoord {
  808. for bias in [AnchorBias::Left, AnchorBias::Right, AnchorBias::Auto] {
  809. assert_eq!(
  810. idx_built.minimal_unique_interval_containing(pos, bias),
  811. idx_cached.minimal_unique_interval_containing(pos, bias),
  812. "cache mismatch at pos={pos} bias={bias:?}",
  813. );
  814. }
  815. }
  816. }
  817. use crate::helpers::test_init;
  818. fn mean(v: &[u32]) -> Option<f64> {
  819. (!v.is_empty()).then(|| v.iter().sum::<u32>() as f64 / v.len() as f64)
  820. }
  821. #[test]
  822. fn uniqueness_genome() -> anyhow::Result<()> {
  823. test_init();
  824. let index = GenomeIndex::build(
  825. "/home/t_steimle/ref/hs1/chm13v2.0.fa",
  826. "/home/t_steimle/ref/hs1/hs1_uniqueness",
  827. None, // None = tout indexer
  828. )?;
  829. let mean_read_len = 150u32;
  830. let interval = 1_000_000u32;
  831. let reader = bam::Reader::from_path(
  832. "/mnt/beegfs02/scratch/t_steimle/data/wgs/DUMCO/diag/DUMCO_diag_hs1.bam",
  833. )
  834. .unwrap();
  835. let header = bam::Header::from_template(reader.header());
  836. let genome_sizes = get_genome_sizes(&header).unwrap();
  837. let file = File::create("res_150.tsv")?;
  838. let mut writer = BufWriter::new(file);
  839. for (contig, size) in genome_sizes {
  840. let from = 0u32;
  841. let to = size as u32;
  842. let half_mean_read_len = mean_read_len / 2;
  843. let mut current_interval = from;
  844. loop {
  845. let mut n_above_half = 0;
  846. for step in 0..interval {
  847. let pos = GenomicPos {
  848. contig: contig.clone(),
  849. pos: current_interval + step,
  850. };
  851. if let Some(UniqueContext {
  852. before: Some(before),
  853. after: Some(after),
  854. }) = index.unique_context_at(&pos)
  855. {
  856. if before > half_mean_read_len || after > half_mean_read_len {
  857. n_above_half += 1;
  858. }
  859. // left.push(before);
  860. // right.push(after);
  861. }
  862. }
  863. println!(
  864. "{contig}:{current_interval}-{}\t{n_above_half}",
  865. current_interval + interval
  866. );
  867. writeln!(
  868. writer,
  869. "{contig}:{current_interval}-{}\t{n_above_half}",
  870. current_interval + interval
  871. )?;
  872. current_interval += interval;
  873. if current_interval >= to {
  874. break;
  875. }
  876. }
  877. }
  878. writer.flush()?;
  879. Ok(())
  880. }
  881. }