diff --git a/Cargo.lock b/Cargo.lock index 3936205..a809b9b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -398,6 +398,16 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "fdeflate" version = "0.3.7" @@ -407,6 +417,18 @@ dependencies = [ "simd-adler32", ] +[[package]] +name = "filetime" +version = "0.2.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc0505cd1b6fa6580283f6bdf70a73fcf4aba1184038c90902b92b3dd0df63ed" +dependencies = [ + "cfg-if", + "libc", + "libredox", + "windows-sys 0.60.2", +] + [[package]] name = "find-msvc-tools" version = "0.1.5" @@ -1028,8 +1050,15 @@ checksum = "416f7e718bdb06000964960ffa43b4335ad4012ae8b99060261aa4a8088d5ccb" dependencies = [ "bitflags 2.10.0", "libc", + "redox_syscall", ] +[[package]] +name = "linux-raw-sys" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" + [[package]] name = "litemap" version = "0.8.1" @@ -1414,6 +1443,15 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags 2.10.0", +] + [[package]] name = "redox_users" version = "0.5.2" @@ -1521,6 +1559,19 @@ dependencies = [ "semver", ] +[[package]] +name = "rustix" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e" +dependencies = [ + "bitflags 2.10.0", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + [[package]] name = "rustls" version = "0.23.35" @@ -1692,6 +1743,23 @@ dependencies = [ "walkdir", ] +[[package]] +name = "signal-gateway-app-code" +version = "0.1.0" +dependencies = [ + "async-trait", + "conf", + "flate2", + "regex", + "reqwest", + "serde", + "serde_json", + "signal-gateway", + "tar", + "tokio", + "tracing", +] + [[package]] name = "signal-gateway-bin" version = "0.1.0" @@ -1817,6 +1885,17 @@ dependencies = [ "time", ] +[[package]] +name = "tar" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d863878d212c87a19c1a610eb53bb01fe12951c0501cf5a0d65f724914a667a" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "thiserror" version = "2.0.17" @@ -2579,6 +2658,16 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] + [[package]] name = "yeslogic-fontconfig-sys" version = "6.0.0" diff --git a/Cargo.toml b/Cargo.toml index b59e521..fcd644e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ resolver = "2" members = [ "prometheus-http-client", "signal-gateway", + "signal-gateway-app-code", "signal-gateway-bin", ] @@ -41,7 +42,9 @@ hyper-util = { version = "0.1", features = ["tokio", "server", "server-auto"] } jsonrpsee = { version = "0.26", features = ["macros", "async-client"] } plotters = { version = "0.3", default-features = false, features = ["bitmap_backend", "bitmap_encoder", "ttf", "datetime", "area_series", "line_series", "full_palette"] } rand = "0.9" +flate2 = "1" regex = "1" +tar = "0.4" reqwest = { version = "0.12", default-features = false, features = ["json"] } serde = { version = "1", features = ["derive"] } serde_json = "1" diff --git a/signal-gateway-app-code/Cargo.toml b/signal-gateway-app-code/Cargo.toml new file mode 100644 index 0000000..12451c4 --- /dev/null +++ b/signal-gateway-app-code/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "signal-gateway-app-code" +version = "0.1.0" +edition.workspace = true + +[lints] +workspace = true + +[dependencies] +signal-gateway = { path = "../signal-gateway", default-features = false } + +async-trait = { workspace = true } +conf = { workspace = true } +flate2 = { workspace = true } +regex = { workspace = true } +reqwest = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +tar = { workspace = true } +tokio = { workspace = true } +tracing = { workspace = true } diff --git a/signal-gateway-app-code/src/lib.rs b/signal-gateway-app-code/src/lib.rs new file mode 100644 index 0000000..5bfb973 --- /dev/null +++ b/signal-gateway-app-code/src/lib.rs @@ -0,0 +1,722 @@ +//! Application source code browsing via GitHub tarball downloads. +//! +//! This crate provides tools for browsing application source code by downloading +//! tarballs from GitHub and caching them in memory. + +use async_trait::async_trait; +use conf::Conf; +use flate2::read::GzDecoder; +use regex::Regex; +use serde::Deserialize; +use signal_gateway::claude::{Tool, ToolExecutor, ToolResult}; +use std::{ + collections::HashMap, error::Error, fmt::Write, io::Read, path::PathBuf, str::FromStr, + sync::Arc, +}; +use tar::Archive; +use tokio::sync::{Mutex, MutexGuard}; +use tracing::info; + +/// A GitHub repository identifier (owner/repo). +#[derive(Clone, Debug)] +pub struct GitHubRepo { + /// The repository owner (user or organization). + pub owner: String, + /// The repository name. + pub repo: String, +} + +impl FromStr for GitHubRepo { + type Err = String; + + fn from_str(s: &str) -> Result { + let parts: Vec<&str> = s.split('/').collect(); + if parts.len() != 2 || parts[0].is_empty() || parts[1].is_empty() { + return Err(format!( + "invalid GitHub repo '{}': expected 'owner/repo' format", + s + )); + } + Ok(Self { + owner: parts[0].to_string(), + repo: parts[1].to_string(), + }) + } +} + +/// Configuration for an application's source code access. +#[derive(Clone, Conf, Debug)] +#[conf(serde)] +pub struct AppCodeConfig { + /// Name of the application (used to identify it in tool calls). + #[conf(long, env)] + pub name: String, + /// GitHub repository in "owner/repo" format. + #[conf(long, env, serde(use_value_parser))] + pub github: GitHubRepo, + /// Path to file containing the GitHub personal access token. + #[conf(long, env)] + pub token_file: PathBuf, +} + +/// A file stored in memory from the tarball. +#[derive(Debug, Clone)] +struct CachedFile { + /// The file contents as a string (lossy UTF-8 conversion). + content: String, +} + +/// Cached tarball contents. +struct CachedTarball { + /// The git SHA this tarball corresponds to. + sha: String, + /// Map from file path to file contents. + files: HashMap, +} + +/// Callback type for getting the current deployed git SHA. +pub type ShaCallback = Arc Result> + Send + Sync>; + +/// Application source code browser. +/// +/// Downloads and caches GitHub tarballs for browsing application source code. +pub struct AppCode { + config: AppCodeConfig, + token: String, + get_sha: ShaCallback, + client: reqwest::Client, + cache: Mutex>, +} + +impl AppCode { + /// Create a new AppCode instance from configuration. + /// + /// The `get_sha` callback is called to determine which git SHA to download. + /// It should return `None` if the SHA is not yet known. + pub fn new(config: AppCodeConfig, get_sha: ShaCallback) -> Result { + let token = std::fs::read_to_string(&config.token_file)? + .trim() + .to_string(); + + Ok(Self { + config, + token, + get_sha, + client: reqwest::Client::new(), + cache: Mutex::new(None), + }) + } + + /// Get the application name. + pub fn name(&self) -> &str { + &self.config.name + } + + /// Get the current tarball, downloading if necessary. + /// + /// Returns a mutex guard containing the cached tarball. After this succeeds, + /// the Option is guaranteed to be Some. + async fn get_current_tarball(&self) -> Result>, String> { + let current_sha = (self.get_sha)().map_err(|e| format!("SHA not available: {e}"))?; + + let mut cache = self.cache.lock().await; + + // Check if we already have this SHA cached + let needs_download = match &*cache { + Some(cached) => cached.sha != current_sha, + None => true, + }; + + if needs_download { + info!( + "Downloading tarball for {} at {}", + self.config.name, current_sha + ); + + let tarball = self.download_tarball(¤t_sha).await?; + let files = self.extract_tarball(&tarball)?; + + *cache = Some(CachedTarball { + sha: current_sha, + files, + }); + } + + Ok(cache) + } + + /// Download a tarball from GitHub for the given SHA. + async fn download_tarball(&self, sha: &str) -> Result, String> { + let url = format!( + "https://api.github.com/repos/{}/{}/tarball/{}", + self.config.github.owner, self.config.github.repo, sha + ); + + let response = self + .client + .get(&url) + .header("Authorization", format!("Bearer {}", self.token)) + .header("Accept", "application/vnd.github+json") + .header("User-Agent", "signal-gateway") + .header("X-GitHub-Api-Version", "2022-11-28") + .send() + .await + .map_err(|e| format!("HTTP request failed: {e}"))?; + + if !response.status().is_success() { + return Err(format!( + "GitHub API error: {} {}", + response.status(), + response.text().await.unwrap_or_default() + )); + } + + response + .bytes() + .await + .map(|b| b.to_vec()) + .map_err(|e| format!("Failed to read response body: {e}")) + } + + /// Extract a tarball into a map of file paths to contents. + fn extract_tarball(&self, tarball: &[u8]) -> Result, String> { + let decoder = GzDecoder::new(tarball); + let mut archive = Archive::new(decoder); + + let mut files = HashMap::new(); + + for entry in archive + .entries() + .map_err(|e| format!("Failed to read tarball: {e}"))? + { + let mut entry = entry.map_err(|e| format!("Failed to read entry: {e}"))?; + + // Skip directories + if entry.header().entry_type().is_dir() { + continue; + } + + let path = entry + .path() + .map_err(|e| format!("Failed to get path: {e}"))? + .to_string_lossy() + .to_string(); + + // GitHub tarballs have a prefix like "owner-repo-sha/" + // Strip the first component + let path = path.split('/').skip(1).collect::>().join("/"); + + if path.is_empty() { + continue; + } + + // Read file contents + let mut contents = Vec::new(); + if entry.read_to_end(&mut contents).is_err() { + continue; // Skip files we can't read + } + + // Convert to string (lossy) + let content = String::from_utf8_lossy(&contents).to_string(); + + files.insert(path, CachedFile { content }); + } + + info!("Extracted {} files from tarball", files.len()); + Ok(files) + } + + /// List files in a directory (like `ls`). + /// + /// If `path` is None or empty, lists the root directory. + pub async fn ls(&self, path: Option<&str>) -> Result { + let cache = self.get_current_tarball().await?; + let cached = cache + .as_ref() + .expect("cache guaranteed after get_current_tarball"); + + let prefix = path.unwrap_or("").trim_start_matches('/'); + let prefix = if prefix.is_empty() { + String::new() + } else if prefix.ends_with('/') { + prefix.to_string() + } else { + format!("{}/", prefix) + }; + + let mut entries = std::collections::BTreeSet::new(); + + for file_path in cached.files.keys() { + if prefix.is_empty() || file_path.starts_with(&prefix) { + // Get the part after the prefix + let remainder = if prefix.is_empty() { + file_path.as_str() + } else { + &file_path[prefix.len()..] + }; + + // Get just the first component (file or directory name) + if let Some(first) = remainder.split('/').next() + && !first.is_empty() { + // Check if it's a directory (has more components) + let is_dir = remainder.contains('/'); + let entry = if is_dir { + format!("{}/", first) + } else { + first.to_string() + }; + entries.insert(entry); + } + } + } + + if entries.is_empty() { + Ok(format!( + "No files found in '{}'", + prefix.trim_end_matches('/') + )) + } else { + Ok(entries.into_iter().collect::>().join("\n")) + } + } + + /// Find files matching a glob pattern (like `find`). + /// + /// Supports simple glob patterns with `*` wildcards. + pub async fn find(&self, pattern: Option<&str>) -> Result { + let cache = self.get_current_tarball().await?; + let cached = cache + .as_ref() + .expect("cache guaranteed after get_current_tarball"); + + let pattern = pattern.unwrap_or("*"); + + // Convert glob pattern to regex + let regex_pattern = glob_to_regex(pattern); + let regex = Regex::new(®ex_pattern).map_err(|e| format!("Invalid pattern: {e}"))?; + + let mut matches: Vec<&str> = cached + .files + .keys() + .filter(|path| regex.is_match(path)) + .map(|s| s.as_str()) + .collect(); + + matches.sort(); + + if matches.is_empty() { + Ok(format!("No files matching '{}'", pattern)) + } else { + Ok(matches.join("\n")) + } + } + + /// Read a file's contents. + /// + /// If `line_range` is provided, only returns those lines (1-indexed, inclusive). + pub async fn read( + &self, + path: &str, + line_start: Option, + line_end: Option, + ) -> Result { + let cache = self.get_current_tarball().await?; + let cached = cache + .as_ref() + .expect("cache guaranteed after get_current_tarball"); + + let path = path.trim_start_matches('/'); + let file = cached + .files + .get(path) + .ok_or_else(|| format!("File not found: {}", path))?; + + let lines: Vec<&str> = file.content.lines().collect(); + + // Handle line range (1-indexed) + let start = line_start.unwrap_or(1).saturating_sub(1); + let end = line_end.unwrap_or(lines.len()).min(lines.len()); + + if start >= lines.len() { + return Ok(format!( + "Line {} is past end of file ({} lines)", + start + 1, + lines.len() + )); + } + + let mut output = String::new(); + for (i, line) in lines[start..end].iter().enumerate() { + writeln!(&mut output, "{:>6}\t{}", start + i + 1, line) + .map_err(|e| format!("Format error: {e}"))?; + } + + Ok(output) + } + + /// Search for a regex pattern in all files. + /// + /// - `pattern`: The regex pattern to search for. + /// - `context`: Number of context lines to show (like `grep -C`). + /// - `path_prefix`: Optional path prefix to limit search scope. + pub async fn search( + &self, + pattern: &str, + context: u32, + path_prefix: Option<&str>, + ) -> Result { + let regex = Regex::new(pattern).map_err(|e| format!("Invalid regex: {e}"))?; + + let cache = self.get_current_tarball().await?; + let cached = cache + .as_ref() + .expect("cache guaranteed after get_current_tarball"); + + let prefix = path_prefix.map(|p| p.trim_start_matches('/')); + + let mut output = String::new(); + let mut match_count = 0; + let mut file_count = 0; + const MAX_MATCHES: usize = 100; + + let mut sorted_files: Vec<_> = cached.files.iter().collect(); + sorted_files.sort_by_key(|(path, _)| *path); + + 'outer: for (path, file) in sorted_files { + // Skip if path doesn't match prefix + if let Some(prefix) = prefix + && !path.starts_with(prefix) { + continue; + } + + // Skip binary-looking files + if looks_binary(&file.content) { + continue; + } + + let lines: Vec<&str> = file.content.lines().collect(); + let mut file_matches = Vec::new(); + + for (line_num, line) in lines.iter().enumerate() { + if regex.is_match(line) { + file_matches.push(line_num); + match_count += 1; + if match_count >= MAX_MATCHES { + break 'outer; + } + } + } + + if !file_matches.is_empty() { + file_count += 1; + + if context == 0 { + // No context, just print matches + for &line_num in &file_matches { + writeln!( + &mut output, + "{}:{}: {}", + path, + line_num + 1, + lines[line_num] + ) + .map_err(|e| format!("Format error: {e}"))?; + } + } else { + // Print with context + writeln!(&mut output, "=== {} ===", path) + .map_err(|e| format!("Format error: {e}"))?; + + let context = context as usize; + let mut printed = std::collections::BTreeSet::new(); + + for &match_line in &file_matches { + let start = match_line.saturating_sub(context); + let end = (match_line + context + 1).min(lines.len()); + + // Add separator if there's a gap + if let Some(&last) = printed.iter().next_back() + && start > last + 1 { + writeln!(&mut output, "---") + .map_err(|e| format!("Format error: {e}"))?; + } + + for (i, line) in lines[start..end].iter().enumerate() { + let line_idx = start + i; + if printed.insert(line_idx) { + let marker = if line_idx == match_line { ">" } else { " " }; + writeln!(&mut output, "{}{:>5}\t{}", marker, line_idx + 1, line) + .map_err(|e| format!("Format error: {e}"))?; + } + } + } + writeln!(&mut output).map_err(|e| format!("Format error: {e}"))?; + } + } + } + + if match_count == 0 { + Ok(format!("No matches for '{}'", pattern)) + } else { + let truncated = if match_count >= MAX_MATCHES { + format!(" (truncated at {} matches)", MAX_MATCHES) + } else { + String::new() + }; + Ok(format!( + "{}\n[{} matches in {} files{}]", + output.trim_end(), + match_count, + file_count, + truncated + )) + } + } +} + +/// Convert a simple glob pattern to a regex. +fn glob_to_regex(pattern: &str) -> String { + let mut regex = String::from("^"); + for c in pattern.chars() { + match c { + '*' => regex.push_str(".*"), + '?' => regex.push('.'), + '.' | '+' | '(' | ')' | '[' | ']' | '{' | '}' | '^' | '$' | '|' | '\\' => { + regex.push('\\'); + regex.push(c); + } + _ => regex.push(c), + } + } + regex.push('$'); + regex +} + +/// Check if content looks like binary data. +fn looks_binary(content: &str) -> bool { + // Check first 1000 chars for null bytes or high ratio of non-printable chars + let sample: String = content.chars().take(1000).collect(); + let non_printable = sample + .chars() + .filter(|c| !c.is_ascii_graphic() && !c.is_ascii_whitespace()) + .count(); + non_printable > sample.len() / 10 +} + +/// Tool executor for multiple application source code browsers. +pub struct AppCodeTools { + apps: Vec, +} + +impl AppCodeTools { + /// Create a new AppCodeTools instance. + pub fn new(apps: Vec) -> Self { + Self { apps } + } + + /// Find an app by name. + fn find_app(&self, name: &str) -> Option<&AppCode> { + self.apps.iter().find(|app| app.name() == name) + } + + /// Get list of app names for error messages. + fn app_names(&self) -> String { + self.apps + .iter() + .map(|a| a.name()) + .collect::>() + .join(", ") + } +} + +#[derive(Deserialize)] +struct LsInput { + app: String, + path: Option, +} + +#[derive(Deserialize)] +struct FindInput { + app: String, + pattern: Option, +} + +#[derive(Deserialize)] +struct ReadInput { + app: String, + path: String, + line_start: Option, + line_end: Option, +} + +#[derive(Deserialize)] +struct SearchInput { + app: String, + pattern: String, + #[serde(default)] + context: u32, + path_prefix: Option, +} + +#[async_trait] +impl ToolExecutor for AppCodeTools { + fn tools(&self) -> Vec { + vec![ + Tool { + name: "code_ls", + description: "List files in a directory of an application's source code.", + input_schema: serde_json::json!({ + "type": "object", + "properties": { + "app": { + "type": "string", + "description": "Name of the application" + }, + "path": { + "type": "string", + "description": "Directory path to list (optional, defaults to root)" + } + }, + "required": ["app"] + }), + }, + Tool { + name: "code_find", + description: "Find files matching a glob pattern in an application's source code.", + input_schema: serde_json::json!({ + "type": "object", + "properties": { + "app": { + "type": "string", + "description": "Name of the application" + }, + "pattern": { + "type": "string", + "description": "Glob pattern to match (e.g., '*.rs', 'src/*.py')" + } + }, + "required": ["app"] + }), + }, + Tool { + name: "code_read", + description: "Read a file from an application's source code.", + input_schema: serde_json::json!({ + "type": "object", + "properties": { + "app": { + "type": "string", + "description": "Name of the application" + }, + "path": { + "type": "string", + "description": "Path to the file to read" + }, + "line_start": { + "type": "integer", + "description": "Starting line number (1-indexed, optional)" + }, + "line_end": { + "type": "integer", + "description": "Ending line number (inclusive, optional)" + } + }, + "required": ["app", "path"] + }), + }, + Tool { + name: "code_search", + description: + "Search for a regex pattern in an application's source code (like grep).", + input_schema: serde_json::json!({ + "type": "object", + "properties": { + "app": { + "type": "string", + "description": "Name of the application" + }, + "pattern": { + "type": "string", + "description": "Regex pattern to search for" + }, + "context": { + "type": "integer", + "description": "Number of context lines to show (like grep -C, default 0)" + }, + "path_prefix": { + "type": "string", + "description": "Optional path prefix to limit search scope" + } + }, + "required": ["app", "pattern"] + }), + }, + ] + } + + fn has_tool(&self, name: &str) -> bool { + matches!(name, "code_ls" | "code_find" | "code_read" | "code_search") + } + + async fn execute(&self, name: &str, input: &serde_json::Value) -> Result { + match name { + "code_ls" => { + let input: LsInput = serde_json::from_value(input.clone()) + .map_err(|e| format!("Invalid input: {e}"))?; + let app = self.find_app(&input.app).ok_or_else(|| { + format!( + "Unknown app '{}'. Available: {}", + input.app, + self.app_names() + ) + })?; + let result = app.ls(input.path.as_deref()).await?; + Ok(ToolResult::new(result)) + } + "code_find" => { + let input: FindInput = serde_json::from_value(input.clone()) + .map_err(|e| format!("Invalid input: {e}"))?; + let app = self.find_app(&input.app).ok_or_else(|| { + format!( + "Unknown app '{}'. Available: {}", + input.app, + self.app_names() + ) + })?; + let result = app.find(input.pattern.as_deref()).await?; + Ok(ToolResult::new(result)) + } + "code_read" => { + let input: ReadInput = serde_json::from_value(input.clone()) + .map_err(|e| format!("Invalid input: {e}"))?; + let app = self.find_app(&input.app).ok_or_else(|| { + format!( + "Unknown app '{}'. Available: {}", + input.app, + self.app_names() + ) + })?; + let result = app + .read(&input.path, input.line_start, input.line_end) + .await?; + Ok(ToolResult::new(result)) + } + "code_search" => { + let input: SearchInput = serde_json::from_value(input.clone()) + .map_err(|e| format!("Invalid input: {e}"))?; + let app = self.find_app(&input.app).ok_or_else(|| { + format!( + "Unknown app '{}'. Available: {}", + input.app, + self.app_names() + ) + })?; + let result = app + .search(&input.pattern, input.context, input.path_prefix.as_deref()) + .await?; + Ok(ToolResult::new(result)) + } + _ => Err(format!("Unknown tool: {name}")), + } + } +}