diff --git a/signal-gateway-repo-code/src/config.rs b/signal-gateway-repo-code/src/config.rs index d4a75d4..d220550 100644 --- a/signal-gateway-repo-code/src/config.rs +++ b/signal-gateway-repo-code/src/config.rs @@ -4,7 +4,7 @@ use serde::Deserialize; use std::{path::PathBuf, str::FromStr}; /// A GitHub repository identifier (owner/repo). -#[derive(Clone, Debug, Deserialize)] +#[derive(Clone, Debug, Deserialize, PartialEq, Eq)] #[serde(try_from = "String")] pub struct GitHubRepo { /// The repository owner (user or organization). @@ -39,16 +39,34 @@ impl TryFrom for GitHubRepo { } } +/// Source of repository code - either a GitHub repo or a local tarball file. +#[derive(Clone, Debug, Deserialize, PartialEq, Eq)] +pub enum Source { + /// GitHub repository source. Downloads tarballs from GitHub API. + #[serde(rename = "github")] + GitHub { + /// GitHub repository in "owner/repo" format. + repo: GitHubRepo, + /// Path to file containing the GitHub personal access token. + /// Optional for public repositories (unauthenticated access has lower rate limits). + token_file: Option, + }, + /// Local tarball file source. Reads a .tar.gz file from disk. + #[serde(rename = "file")] + File { + /// Path to the tarball file (.tar.gz). + path: PathBuf, + }, +} + /// Configuration for an application's source code access. -#[derive(Clone, Debug, Deserialize)] +#[derive(Clone, Debug, Deserialize, PartialEq, Eq)] pub struct RepoCodeConfig { /// Name of the application (used to identify it in tool calls). pub name: String, - /// GitHub repository in "owner/repo" format. - pub github: GitHubRepo, - /// Path to file containing the GitHub personal access token. - /// Optional for public repositories (unauthenticated access has lower rate limits). - pub token_file: Option, + /// Source of the repository code. + #[serde(flatten)] + pub source: Source, /// Glob patterns to filter which files are included from the tarball. /// If non-empty, only files matching at least one pattern are kept. /// Uses gitignore-style glob syntax (e.g., "*.rs", "src/**/*.rs"). @@ -59,3 +77,109 @@ pub struct RepoCodeConfig { #[serde(default)] pub include_non_utf8: bool, } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_github_source() { + let json = r#"{ + "name": "my-app", + "github": { + "repo": "owner/repo-name", + "token_file": "/path/to/token" + } + }"#; + + let config: RepoCodeConfig = serde_json::from_str(json).unwrap(); + assert_eq!(config.name, "my-app"); + assert_eq!( + config.source, + Source::GitHub { + repo: GitHubRepo { + owner: "owner".to_string(), + repo: "repo-name".to_string(), + }, + token_file: Some(PathBuf::from("/path/to/token")), + } + ); + assert!(config.glob.is_empty()); + assert!(!config.include_non_utf8); + } + + #[test] + fn test_parse_github_source_no_token() { + let json = r#"{ + "name": "public-app", + "github": { + "repo": "org/public-repo" + } + }"#; + + let config: RepoCodeConfig = serde_json::from_str(json).unwrap(); + assert_eq!(config.name, "public-app"); + assert_eq!( + config.source, + Source::GitHub { + repo: GitHubRepo { + owner: "org".to_string(), + repo: "public-repo".to_string(), + }, + token_file: None, + } + ); + } + + #[test] + fn test_parse_file_source() { + let json = r#"{ + "name": "local-app", + "file": { + "path": "/tmp/source.tar.gz" + } + }"#; + + let config: RepoCodeConfig = serde_json::from_str(json).unwrap(); + assert_eq!(config.name, "local-app"); + assert_eq!( + config.source, + Source::File { + path: PathBuf::from("/tmp/source.tar.gz"), + } + ); + } + + #[test] + fn test_parse_with_glob_and_options() { + let json = r#"{ + "name": "filtered-app", + "github": { + "repo": "owner/repo" + }, + "glob": ["**/*.rs", "Cargo.toml"], + "include_non_utf8": true + }"#; + + let config: RepoCodeConfig = serde_json::from_str(json).unwrap(); + assert_eq!(config.name, "filtered-app"); + assert_eq!(config.glob, vec!["**/*.rs", "Cargo.toml"]); + assert!(config.include_non_utf8); + } + + #[test] + fn test_parse_github_repo_string() { + let repo: GitHubRepo = "owner/repo".parse().unwrap(); + assert_eq!(repo.owner, "owner"); + assert_eq!(repo.repo, "repo"); + } + + #[test] + fn test_parse_github_repo_invalid() { + assert!("invalid".parse::().is_err()); + assert!("".parse::().is_err()); + assert!("/repo".parse::().is_err()); + assert!("owner/".parse::().is_err()); + assert!("a/b/c".parse::().is_err()); + } +} diff --git a/signal-gateway-repo-code/src/lib.rs b/signal-gateway-repo-code/src/lib.rs index c254583..13b6246 100644 --- a/signal-gateway-repo-code/src/lib.rs +++ b/signal-gateway-repo-code/src/lib.rs @@ -5,7 +5,7 @@ mod config; -pub use config::{GitHubRepo, RepoCodeConfig}; +pub use config::{GitHubRepo, RepoCodeConfig, Source}; use async_trait::async_trait; use flate2::read::GzDecoder; @@ -42,13 +42,26 @@ pub type ShaCallback = Arc< + Sync, >; +/// Internal representation of the resolved source. +enum ResolvedSource { + GitHub { + owner: String, + repo: String, + token: Option, + }, + File { + path: std::path::PathBuf, + }, +} + /// Application source code browser. /// /// Downloads and caches GitHub tarballs for browsing application source code. pub struct RepoCode { - config: RepoCodeConfig, - token: Option, + name: String, + source: ResolvedSource, glob_filter: Option, + include_non_utf8: bool, get_sha: ShaCallback, client: reqwest::Client, cache: Mutex>, @@ -57,36 +70,46 @@ pub struct RepoCode { impl RepoCode { /// Create a new RepoCode instance from configuration. /// - /// The `get_sha` callback is called to determine which git SHA to download. - /// It should return `None` if the SHA is not yet known. + /// The `get_sha` callback is called to determine which git SHA to download/load. + /// For GitHub sources, this is the commit SHA. For file sources, this can be + /// used to track file modification (e.g., mtime or a version string). pub fn new(config: RepoCodeConfig, get_sha: ShaCallback) -> Result { - let token = config - .token_file - .as_ref() - .map(|path| std::fs::read_to_string(path).map(|s| s.trim().to_string())) - .transpose()?; + let source = match config.source { + Source::GitHub { repo, token_file } => { + let token = token_file + .as_ref() + .map(|path| std::fs::read_to_string(path).map(|s| s.trim().to_string())) + .transpose()?; + ResolvedSource::GitHub { + owner: repo.owner, + repo: repo.repo, + token, + } + } + Source::File { path } => ResolvedSource::File { path }, + }; // Compile glob patterns if any are specified - let glob_filter = - if config.glob.is_empty() { - None - } else { - let mut builder = GlobSetBuilder::new(); - for pattern in &config.glob { - let glob = Glob::new(pattern).map_err(|e| { - std::io::Error::other(format!("invalid glob pattern '{}': {}", pattern, e)) - })?; - builder.add(glob); - } - Some(builder.build().map_err(|e| { - std::io::Error::other(format!("failed to build glob set: {}", e)) - })?) - }; + let glob_filter = if config.glob.is_empty() { + None + } else { + let mut builder = GlobSetBuilder::new(); + for pattern in &config.glob { + let glob = Glob::new(pattern).map_err(|e| { + std::io::Error::other(format!("invalid glob pattern '{}': {}", pattern, e)) + })?; + builder.add(glob); + } + Some(builder.build().map_err(|e| { + std::io::Error::other(format!("failed to build glob set: {}", e)) + })?) + }; Ok(Self { - config, - token, + name: config.name, + source, glob_filter, + include_non_utf8: config.include_non_utf8, get_sha, client: reqwest::Client::new(), cache: Mutex::new(None), @@ -95,16 +118,16 @@ impl RepoCode { /// Get the application name. pub fn name(&self) -> &str { - &self.config.name + &self.name } - /// Get the current tarball, downloading if necessary. + /// Get the current tarball, downloading or reading from file as needed. /// /// Returns a mutex guard containing the cached tarball. This method never fails; /// instead it logs warnings and returns whatever is currently cached: /// /// - If the SHA callback fails, returns the existing cache (possibly stale or None) - /// - If the tarball download fails, returns the existing cache + /// - If the tarball download/read fails, returns the existing cache /// - If tarball extraction fails, returns the existing cache /// /// This design assumes that stale code is better than no code, since most of the @@ -113,29 +136,29 @@ impl RepoCode { let current_sha = match (self.get_sha)().await { Ok(sha) => sha, Err(e) => { - warn!("Failed to get current SHA for {}: {e}", self.config.name); + warn!("Failed to get current SHA for {}: {e}", self.name); return self.cache.lock().await; } }; let mut cache = self.cache.lock().await; - // Check if we already have this SHA cached - let needs_download = match &*cache { - Some(cached) => cached.sha != current_sha, - None => true, + // Check if we need to load/reload the tarball. + // For file sources, we only load once (no refresh after initial load). + // For GitHub sources, we reload when the SHA changes. + let needs_refresh = match (&*cache, &self.source) { + (None, _) => true, + (Some(_), ResolvedSource::File { .. }) => false, + (Some(cached), ResolvedSource::GitHub { .. }) => cached.sha != current_sha, }; - if needs_download { - info!( - "Downloading tarball for {} at {}", - self.config.name, current_sha - ); + if needs_refresh { + info!("Loading tarball for {} at {}", self.name, current_sha); - let tarball = match self.download_tarball(¤t_sha).await { + let tarball = match self.load_tarball(¤t_sha).await { Ok(t) => t, Err(e) => { - error!("Failed to download tarball for {}: {e}", self.config.name); + error!("Failed to load tarball for {}: {e}", self.name); return cache; } }; @@ -143,7 +166,7 @@ impl RepoCode { let files = match self.extract_tarball(&tarball) { Ok(f) => f, Err(e) => { - error!("Failed to extract tarball for {}: {e}", self.config.name); + error!("Failed to extract tarball for {}: {e}", self.name); return cache; } }; @@ -157,11 +180,30 @@ impl RepoCode { cache } + /// Load a tarball either from GitHub or from a local file. + async fn load_tarball(&self, sha: &str) -> Result, String> { + match &self.source { + ResolvedSource::GitHub { owner, repo, token } => { + self.download_tarball_from_github(owner, repo, token.as_deref(), sha) + .await + } + ResolvedSource::File { path } => std::fs::read(path).map_err(|e| { + format!("Failed to read tarball from {}: {e}", path.display()) + }), + } + } + /// Download a tarball from GitHub for the given SHA. - async fn download_tarball(&self, sha: &str) -> Result, String> { + async fn download_tarball_from_github( + &self, + owner: &str, + repo: &str, + token: Option<&str>, + sha: &str, + ) -> Result, String> { let url = format!( "https://api.github.com/repos/{}/{}/tarball/{}", - self.config.github.owner, self.config.github.repo, sha + owner, repo, sha ); let mut request = self @@ -171,7 +213,7 @@ impl RepoCode { .header("User-Agent", "signal-gateway") .header("X-GitHub-Api-Version", "2022-11-28"); - if let Some(token) = &self.token { + if let Some(token) = token { request = request.header("Authorization", format!("Bearer {}", token)); } @@ -244,7 +286,7 @@ impl RepoCode { let content = match String::from_utf8(contents) { Ok(s) => s, Err(e) => { - if self.config.include_non_utf8 { + if self.include_non_utf8 { // Use lossy conversion if configured to include non-UTF-8 String::from_utf8_lossy(e.as_bytes()).into_owned() } else { diff --git a/signal-gateway-repo-code/tests/integration.rs b/signal-gateway-repo-code/tests/integration.rs index 44358f4..d3e029b 100644 --- a/signal-gateway-repo-code/tests/integration.rs +++ b/signal-gateway-repo-code/tests/integration.rs @@ -3,7 +3,7 @@ //! These tests exercise the GitHub tarball download and file browsing functionality //! against a real public repository at a pinned commit. -use signal_gateway_repo_code::{GitHubRepo, RepoCode, RepoCodeConfig, ShaCallback}; +use signal_gateway_repo_code::{GitHubRepo, RepoCode, RepoCodeConfig, ShaCallback, Source}; use std::sync::Arc; /// Test against cbeck88/ver-stub-rs at a known commit. @@ -12,18 +12,20 @@ const TEST_OWNER: &str = "cbeck88"; const TEST_REPO: &str = "ver-stub-rs"; const TEST_SHA: &str = "79b98e25f27ae4f5dd73a5a3d8f37dad655a57e8"; -fn create_test_app_code() -> RepoCode { - create_test_app_code_with_glob(vec![]) +fn create_test_repo_code() -> RepoCode { + create_test_repo_code_with_glob(vec![]) } -fn create_test_app_code_with_glob(glob: Vec) -> RepoCode { +fn create_test_repo_code_with_glob(glob: Vec) -> RepoCode { let config = RepoCodeConfig { name: "test-app".to_string(), - github: GitHubRepo { - owner: TEST_OWNER.to_string(), - repo: TEST_REPO.to_string(), + source: Source::GitHub { + repo: GitHubRepo { + owner: TEST_OWNER.to_string(), + repo: TEST_REPO.to_string(), + }, + token_file: None, // Public repo, no auth needed }, - token_file: None, // Public repo, no auth needed glob, include_non_utf8: false, }; @@ -39,7 +41,7 @@ fn create_test_app_code_with_glob(glob: Vec) -> RepoCode { #[tokio::test] async fn test_ls_root() { - let app = create_test_app_code(); + let app = create_test_repo_code(); let result = app.ls(None).await.expect("ls failed"); @@ -59,7 +61,7 @@ async fn test_ls_root() { #[tokio::test] async fn test_ls_subdirectory() { - let app = create_test_app_code(); + let app = create_test_repo_code(); let result = app.ls(Some("ver-stub")).await.expect("ls failed"); @@ -70,7 +72,7 @@ async fn test_ls_subdirectory() { #[tokio::test] async fn test_find_rust_files() { - let app = create_test_app_code(); + let app = create_test_repo_code(); let result = app.find(Some("*.rs")).await.expect("find failed"); @@ -87,7 +89,7 @@ async fn test_find_rust_files() { #[tokio::test] async fn test_find_with_path_pattern() { - let app = create_test_app_code(); + let app = create_test_repo_code(); let result = app .find(Some("ver-stub-build/src/*.rs")) @@ -103,7 +105,7 @@ async fn test_find_with_path_pattern() { #[tokio::test] async fn test_read_cargo_toml() { - let app = create_test_app_code(); + let app = create_test_repo_code(); let result = app .read("Cargo.toml", None, None) @@ -121,7 +123,7 @@ async fn test_read_cargo_toml() { #[tokio::test] async fn test_read_with_line_range() { - let app = create_test_app_code(); + let app = create_test_repo_code(); // Read just the first 5 lines let result = app @@ -142,7 +144,7 @@ async fn test_read_with_line_range() { #[tokio::test] async fn test_read_nonexistent_file() { - let app = create_test_app_code(); + let app = create_test_repo_code(); let result = app.read("nonexistent-file.txt", None, None).await; @@ -155,7 +157,7 @@ async fn test_read_nonexistent_file() { #[tokio::test] async fn test_search_simple() { - let app = create_test_app_code(); + let app = create_test_repo_code(); let result = app .search("workspace", 0, None) @@ -171,7 +173,7 @@ async fn test_search_simple() { #[tokio::test] async fn test_search_with_context() { - let app = create_test_app_code(); + let app = create_test_repo_code(); let result = app .search("resolver", 2, None) @@ -192,7 +194,7 @@ async fn test_search_with_context() { #[tokio::test] async fn test_search_with_path_prefix() { - let app = create_test_app_code(); + let app = create_test_repo_code(); // Search only in ver-stub-build directory let result = app @@ -215,7 +217,7 @@ async fn test_search_with_path_prefix() { #[tokio::test] async fn test_search_no_matches() { - let app = create_test_app_code(); + let app = create_test_repo_code(); let result = app .search("xyzzy_unlikely_string_12345", 0, None) @@ -228,7 +230,7 @@ async fn test_search_no_matches() { #[tokio::test] async fn test_glob_filter_rust_files_only() { // Only include .rs files - let app = create_test_app_code_with_glob(vec!["**/*.rs".to_string()]); + let app = create_test_repo_code_with_glob(vec!["**/*.rs".to_string()]); let result = app.find(Some("*")).await.expect("find failed"); @@ -250,7 +252,7 @@ async fn test_glob_filter_rust_files_only() { #[tokio::test] async fn test_glob_filter_specific_directory() { // Only include files in ver-stub/src - let app = create_test_app_code_with_glob(vec!["ver-stub/src/**".to_string()]); + let app = create_test_repo_code_with_glob(vec!["ver-stub/src/**".to_string()]); let result = app.find(Some("*")).await.expect("find failed"); @@ -274,7 +276,7 @@ async fn test_glob_filter_specific_directory() { #[tokio::test] async fn test_glob_filter_multiple_patterns() { // Include both Cargo.toml files and shell scripts - let app = create_test_app_code_with_glob(vec!["**/Cargo.toml".to_string(), "*.sh".to_string()]); + let app = create_test_repo_code_with_glob(vec!["**/Cargo.toml".to_string(), "*.sh".to_string()]); let result = app.find(Some("*")).await.expect("find failed"); @@ -295,7 +297,7 @@ async fn test_glob_filter_multiple_patterns() { #[tokio::test] async fn test_glob_filter_ls_shows_filtered_dirs() { // Only include files in ver-stub directory - let app = create_test_app_code_with_glob(vec!["ver-stub/**".to_string()]); + let app = create_test_repo_code_with_glob(vec!["ver-stub/**".to_string()]); let result = app.ls(None).await.expect("ls failed");