From b0dd92aee8ce17d4dd9f32a27fbe7af917d626ce Mon Sep 17 00:00:00 2001 From: Chris Beck Date: Thu, 18 Dec 2025 11:04:27 -0700 Subject: [PATCH] more efficient line iterator --- signal-gateway-code-tool/src/cached.rs | 219 ++++++++++++++++++++++++- signal-gateway-code-tool/src/lib.rs | 57 +++---- 2 files changed, 242 insertions(+), 34 deletions(-) diff --git a/signal-gateway-code-tool/src/cached.rs b/signal-gateway-code-tool/src/cached.rs index cdfc129..23cba6e 100644 --- a/signal-gateway-code-tool/src/cached.rs +++ b/signal-gateway-code-tool/src/cached.rs @@ -10,16 +10,130 @@ use tracing::info; #[derive(Debug, Clone)] pub struct CachedFile { /// The file contents as a string (lossy UTF-8 conversion). - pub content: String, + content: Box, + /// Indices of newline characters in content. + newline_indices: Vec, /// Whether this file looks like binary data. - pub is_binary: bool, + is_binary: bool, } impl CachedFile { /// Create a new CachedFile from content, detecting if it looks binary. fn new(content: String) -> Self { + assert!( + content.len() <= u32::MAX as usize, + "File content exceeds 4GB limit" + ); + let is_binary = looks_binary(&content); - Self { content, is_binary } + let newline_indices: Vec = content + .bytes() + .enumerate() + .filter(|(_, b)| *b == b'\n') + .map(|(i, _)| i as u32) + .collect(); + + Self { + content: content.into_boxed_str(), + newline_indices, + is_binary, + } + } + + /// Get the file content as a string slice. + pub fn as_str(&self) -> &str { + &self.content + } + + /// Check if file looks like binary data. + pub fn is_binary(&self) -> bool { + self.is_binary + } + + /// Get the number of lines in the file. + pub fn line_count(&self) -> u32 { + if self.content.is_empty() { + 0 + } else { + // Number of lines = number of newlines + 1 (unless file ends with newline) + let count = self.newline_indices.len() as u32; + if self.content.ends_with('\n') { + count + } else { + count + 1 + } + } + } + + /// Iterate over a range of lines (1-indexed, inclusive). + /// + /// - `start`: Starting line number (1-indexed). Defaults to 1. + /// - `end`: Ending line number (inclusive). Defaults to last line. + pub fn line_range(&self, start: Option, end: Option) -> impl Iterator { + let total_lines = self.line_count(); + let start_line = start.unwrap_or(1).saturating_sub(1); // Convert to 0-indexed + let end_line = end.unwrap_or(total_lines).min(total_lines).saturating_sub(1); // Convert to 0-indexed + + LineRangeIter { + content: &self.content, + newline_indices: &self.newline_indices, + current_line: start_line, + end_line, + } + } + + /// Find which line (1-indexed) a character index falls on. + /// + /// Returns the number of newline characters before `idx`, plus 1. + pub fn idx_to_line(&self, idx: usize) -> u32 { + let idx = idx as u32; + // Binary search to find how many newlines are before idx + match self.newline_indices.binary_search(&idx) { + Ok(pos) => pos as u32 + 1, // idx is exactly on a newline + Err(pos) => pos as u32 + 1, // pos is the number of newlines before idx + } + } + +} + +/// Iterator over a range of lines. +struct LineRangeIter<'a> { + content: &'a str, + newline_indices: &'a [u32], + current_line: u32, // 0-indexed + end_line: u32, // 0-indexed, inclusive +} + +impl<'a> Iterator for LineRangeIter<'a> { + type Item = &'a str; + + fn next(&mut self) -> Option { + if self.current_line > self.end_line { + return None; + } + + let line = self.current_line as usize; + + // Get start offset: after previous line's newline, or 0 for first line + let start = if line == 0 { + 0 + } else { + self.newline_indices.get(line - 1).map(|&i| i as usize + 1)? + }; + + // Get end offset: at this line's newline, or end of content for last line + let end = self + .newline_indices + .get(line) + .map(|&i| i as usize) + .unwrap_or(self.content.len()); + + if start > self.content.len() { + return None; + } + + self.current_line += 1; + Some(&self.content[start..end]) } } @@ -132,3 +246,102 @@ fn looks_binary(content: &str) -> bool { .count(); non_printable > sample.len() / 10 } + +#[cfg(test)] +mod tests { + use super::*; + + fn make_file(content: &str) -> CachedFile { + CachedFile::new(content.to_string()) + } + + #[test] + fn test_line_count_empty() { + let file = make_file(""); + assert_eq!(file.line_count(), 0); + } + + #[test] + fn test_line_count_single_line_no_newline() { + let file = make_file("hello"); + assert_eq!(file.line_count(), 1); + } + + #[test] + fn test_line_count_single_line_with_newline() { + let file = make_file("hello\n"); + assert_eq!(file.line_count(), 1); + } + + #[test] + fn test_line_count_multiple_lines() { + let file = make_file("line1\nline2\nline3"); + assert_eq!(file.line_count(), 3); + } + + #[test] + fn test_line_count_multiple_lines_trailing_newline() { + let file = make_file("line1\nline2\nline3\n"); + assert_eq!(file.line_count(), 3); + } + + #[test] + fn test_line_range_all_lines() { + let file = make_file("line1\nline2\nline3"); + let lines: Vec<_> = file.line_range(None, None).collect(); + assert_eq!(lines, vec!["line1", "line2", "line3"]); + } + + #[test] + fn test_line_range_subset() { + let file = make_file("line1\nline2\nline3\nline4\nline5"); + let lines: Vec<_> = file.line_range(Some(2), Some(4)).collect(); + assert_eq!(lines, vec!["line2", "line3", "line4"]); + } + + #[test] + fn test_line_range_single_line() { + let file = make_file("line1\nline2\nline3"); + let lines: Vec<_> = file.line_range(Some(2), Some(2)).collect(); + assert_eq!(lines, vec!["line2"]); + } + + #[test] + fn test_line_range_past_end() { + let file = make_file("line1\nline2"); + let lines: Vec<_> = file.line_range(Some(1), Some(10)).collect(); + assert_eq!(lines, vec!["line1", "line2"]); + } + + #[test] + fn test_idx_to_line_first_line() { + let file = make_file("hello\nworld\ntest"); + // "hello" is at indices 0-4, newline at 5 + assert_eq!(file.idx_to_line(0), 1); + assert_eq!(file.idx_to_line(4), 1); + assert_eq!(file.idx_to_line(5), 1); // newline itself is on line 1 + } + + #[test] + fn test_idx_to_line_second_line() { + let file = make_file("hello\nworld\ntest"); + // "world" is at indices 6-10, newline at 11 + assert_eq!(file.idx_to_line(6), 2); + assert_eq!(file.idx_to_line(10), 2); + assert_eq!(file.idx_to_line(11), 2); // newline itself is on line 2 + } + + #[test] + fn test_idx_to_line_third_line() { + let file = make_file("hello\nworld\ntest"); + // "test" is at indices 12-15 + assert_eq!(file.idx_to_line(12), 3); + assert_eq!(file.idx_to_line(15), 3); + } + + #[test] + fn test_as_str() { + let file = make_file("hello\nworld"); + assert_eq!(file.as_str(), "hello\nworld"); + } +} diff --git a/signal-gateway-code-tool/src/lib.rs b/signal-gateway-code-tool/src/lib.rs index 9fa4577..e531d80 100644 --- a/signal-gateway-code-tool/src/lib.rs +++ b/signal-gateway-code-tool/src/lib.rs @@ -323,23 +323,20 @@ impl CodeTool { .get(path) .ok_or_else(|| format!("File not found: {}", path))?; - let lines: Vec<&str> = file.content.lines().collect(); + let total_lines = file.line_count(); + let start = line_start.unwrap_or(1) as u32; + let end = line_end.map(|e| e as u32); - // Handle line range (1-indexed) - let start = line_start.unwrap_or(1).saturating_sub(1); - let end = line_end.unwrap_or(lines.len()).min(lines.len()); - - if start >= lines.len() { + if start > total_lines { return Ok(format!( "Line {} is past end of file ({} lines)", - start + 1, - lines.len() + start, total_lines )); } let mut output = String::new(); - for (i, line) in lines[start..end].iter().enumerate() { - writeln!(&mut output, "{:>6}\t{}", start + i + 1, line) + for (i, line) in file.line_range(Some(start), end).enumerate() { + writeln!(&mut output, "{:>6}\t{}", start as usize + i, line) .map_err(|e| format!("Format error: {e}"))?; } @@ -378,15 +375,18 @@ impl CodeTool { } // Skip binary-looking files - if file.is_binary { + if file.is_binary() { continue; } - let lines: Vec<&str> = file.content.lines().collect(); - let mut file_matches = Vec::new(); + // Find all matches and map byte positions to line numbers + let content = file.as_str(); + let mut file_matches: Vec = Vec::new(); - for (line_num, line) in lines.iter().enumerate() { - if regex.is_match(line) { + for m in regex.find_iter(content) { + let line_num = file.idx_to_line(m.start()); + // Deduplicate: only add if this line isn't already recorded + if file_matches.last() != Some(&line_num) { file_matches.push(line_num); match_count += 1; if match_count >= MAX_MATCHES { @@ -397,30 +397,25 @@ impl CodeTool { if !file_matches.is_empty() { file_count += 1; + let total_lines = file.line_count(); if context == 0 { // No context, just print matches for &line_num in &file_matches { - writeln!( - &mut output, - "{}:{}: {}", - path, - line_num + 1, - lines[line_num] - ) - .map_err(|e| format!("Format error: {e}"))?; + let line = file.line_range(Some(line_num), Some(line_num)).next().unwrap_or(""); + writeln!(&mut output, "{}:{}: {}", path, line_num, line) + .map_err(|e| format!("Format error: {e}"))?; } } else { // Print with context writeln!(&mut output, "=== {} ===", path) .map_err(|e| format!("Format error: {e}"))?; - let context = context as usize; let mut printed = std::collections::BTreeSet::new(); for &match_line in &file_matches { - let start = match_line.saturating_sub(context); - let end = (match_line + context + 1).min(lines.len()); + let start = match_line.saturating_sub(context).max(1); + let end = (match_line + context).min(total_lines); // Add separator if there's a gap if let Some(&last) = printed.iter().next_back() @@ -430,11 +425,11 @@ impl CodeTool { .map_err(|e| format!("Format error: {e}"))?; } - for (i, line) in lines[start..end].iter().enumerate() { - let line_idx = start + i; - if printed.insert(line_idx) { - let marker = if line_idx == match_line { ">" } else { " " }; - writeln!(&mut output, "{}{:>5}\t{}", marker, line_idx + 1, line) + for (i, line) in file.line_range(Some(start), Some(end)).enumerate() { + let line_num = start + i as u32; + if printed.insert(line_num) { + let marker = if line_num == match_line { ">" } else { " " }; + writeln!(&mut output, "{}{:>5}\t{}", marker, line_num, line) .map_err(|e| format!("Format error: {e}"))?; } }