make signal-gateway-repo-code also support reading code from a static tarball

signal-gateway-repo-code/src/config.rs

@@ -4,7 +4,7 @@ use serde::Deserialize;
 use std::{path::PathBuf, str::FromStr};
 
 /// A GitHub repository identifier (owner/repo).
-#[derive(Clone, Debug, Deserialize)]
+#[derive(Clone, Debug, Deserialize, PartialEq, Eq)]
 #[serde(try_from = "String")]
 pub struct GitHubRepo {
     /// The repository owner (user or organization).
@@ -39,16 +39,34 @@ impl TryFrom<String> for GitHubRepo {
     }
 }
 
+/// Source of repository code - either a GitHub repo or a local tarball file.
+#[derive(Clone, Debug, Deserialize, PartialEq, Eq)]
+pub enum Source {
+    /// GitHub repository source. Downloads tarballs from GitHub API.
+    #[serde(rename = "github")]
+    GitHub {
+        /// GitHub repository in "owner/repo" format.
+        repo: GitHubRepo,
+        /// Path to file containing the GitHub personal access token.
+        /// Optional for public repositories (unauthenticated access has lower rate limits).
+        token_file: Option<PathBuf>,
+    },
+    /// Local tarball file source. Reads a .tar.gz file from disk.
+    #[serde(rename = "file")]
+    File {
+        /// Path to the tarball file (.tar.gz).
+        path: PathBuf,
+    },
+}
+
 /// Configuration for an application's source code access.
-#[derive(Clone, Debug, Deserialize)]
+#[derive(Clone, Debug, Deserialize, PartialEq, Eq)]
 pub struct RepoCodeConfig {
     /// Name of the application (used to identify it in tool calls).
     pub name: String,
-    /// GitHub repository in "owner/repo" format.
+    /// Source of the repository code.
-    pub github: GitHubRepo,
+    #[serde(flatten)]
-    /// Path to file containing the GitHub personal access token.
+    pub source: Source,
-    /// Optional for public repositories (unauthenticated access has lower rate limits).
-    pub token_file: Option<PathBuf>,
     /// Glob patterns to filter which files are included from the tarball.
     /// If non-empty, only files matching at least one pattern are kept.
     /// Uses gitignore-style glob syntax (e.g., "*.rs", "src/**/*.rs").
@@ -59,3 +77,109 @@ pub struct RepoCodeConfig {
     #[serde(default)]
     pub include_non_utf8: bool,
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_parse_github_source() {
+        let json = r#"{
+            "name": "my-app",
+            "github": {
+                "repo": "owner/repo-name",
+                "token_file": "/path/to/token"
+            }
+        }"#;
+
+        let config: RepoCodeConfig = serde_json::from_str(json).unwrap();
+        assert_eq!(config.name, "my-app");
+        assert_eq!(
+            config.source,
+            Source::GitHub {
+                repo: GitHubRepo {
+                    owner: "owner".to_string(),
+                    repo: "repo-name".to_string(),
+                },
+                token_file: Some(PathBuf::from("/path/to/token")),
+            }
+        );
+        assert!(config.glob.is_empty());
+        assert!(!config.include_non_utf8);
+    }
+
+    #[test]
+    fn test_parse_github_source_no_token() {
+        let json = r#"{
+            "name": "public-app",
+            "github": {
+                "repo": "org/public-repo"
+            }
+        }"#;
+
+        let config: RepoCodeConfig = serde_json::from_str(json).unwrap();
+        assert_eq!(config.name, "public-app");
+        assert_eq!(
+            config.source,
+            Source::GitHub {
+                repo: GitHubRepo {
+                    owner: "org".to_string(),
+                    repo: "public-repo".to_string(),
+                },
+                token_file: None,
+            }
+        );
+    }
+
+    #[test]
+    fn test_parse_file_source() {
+        let json = r#"{
+            "name": "local-app",
+            "file": {
+                "path": "/tmp/source.tar.gz"
+            }
+        }"#;
+
+        let config: RepoCodeConfig = serde_json::from_str(json).unwrap();
+        assert_eq!(config.name, "local-app");
+        assert_eq!(
+            config.source,
+            Source::File {
+                path: PathBuf::from("/tmp/source.tar.gz"),
+            }
+        );
+    }
+
+    #[test]
+    fn test_parse_with_glob_and_options() {
+        let json = r#"{
+            "name": "filtered-app",
+            "github": {
+                "repo": "owner/repo"
+            },
+            "glob": ["**/*.rs", "Cargo.toml"],
+            "include_non_utf8": true
+        }"#;
+
+        let config: RepoCodeConfig = serde_json::from_str(json).unwrap();
+        assert_eq!(config.name, "filtered-app");
+        assert_eq!(config.glob, vec!["**/*.rs", "Cargo.toml"]);
+        assert!(config.include_non_utf8);
+    }
+
+    #[test]
+    fn test_parse_github_repo_string() {
+        let repo: GitHubRepo = "owner/repo".parse().unwrap();
+        assert_eq!(repo.owner, "owner");
+        assert_eq!(repo.repo, "repo");
+    }
+
+    #[test]
+    fn test_parse_github_repo_invalid() {
+        assert!("invalid".parse::<GitHubRepo>().is_err());
+        assert!("".parse::<GitHubRepo>().is_err());
+        assert!("/repo".parse::<GitHubRepo>().is_err());
+        assert!("owner/".parse::<GitHubRepo>().is_err());
+        assert!("a/b/c".parse::<GitHubRepo>().is_err());
+    }
+}

signal-gateway-repo-code/src/lib.rs

@@ -5,7 +5,7 @@
 
 mod config;
 
-pub use config::{GitHubRepo, RepoCodeConfig};
+pub use config::{GitHubRepo, RepoCodeConfig, Source};
 
 use async_trait::async_trait;
 use flate2::read::GzDecoder;
@@ -42,13 +42,26 @@ pub type ShaCallback = Arc<
         + Sync,
 >;
 
+/// Internal representation of the resolved source.
+enum ResolvedSource {
+    GitHub {
+        owner: String,
+        repo: String,
+        token: Option<String>,
+    },
+    File {
+        path: std::path::PathBuf,
+    },
+}
+
 /// Application source code browser.
 ///
 /// Downloads and caches GitHub tarballs for browsing application source code.
 pub struct RepoCode {
-    config: RepoCodeConfig,
+    name: String,
-    token: Option<String>,
+    source: ResolvedSource,
     glob_filter: Option<GlobSet>,
+    include_non_utf8: bool,
     get_sha: ShaCallback,
     client: reqwest::Client,
     cache: Mutex<Option<CachedTarball>>,
@@ -57,18 +70,27 @@ pub struct RepoCode {
 impl RepoCode {
     /// Create a new RepoCode instance from configuration.
     ///
-    /// The `get_sha` callback is called to determine which git SHA to download.
+    /// The `get_sha` callback is called to determine which git SHA to download/load.
-    /// It should return `None` if the SHA is not yet known.
+    /// For GitHub sources, this is the commit SHA. For file sources, this can be
+    /// used to track file modification (e.g., mtime or a version string).
     pub fn new(config: RepoCodeConfig, get_sha: ShaCallback) -> Result<Self, std::io::Error> {
-        let token = config
+        let source = match config.source {
-            .token_file
+            Source::GitHub { repo, token_file } => {
+                let token = token_file
                     .as_ref()
                     .map(|path| std::fs::read_to_string(path).map(|s| s.trim().to_string()))
                     .transpose()?;
+                ResolvedSource::GitHub {
+                    owner: repo.owner,
+                    repo: repo.repo,
+                    token,
+                }
+            }
+            Source::File { path } => ResolvedSource::File { path },
+        };
 
         // Compile glob patterns if any are specified
-        let glob_filter =
+        let glob_filter = if config.glob.is_empty() {
-            if config.glob.is_empty() {
             None
         } else {
             let mut builder = GlobSetBuilder::new();
@@ -84,9 +106,10 @@ impl RepoCode {
         };
 
         Ok(Self {
-            config,
+            name: config.name,
-            token,
+            source,
             glob_filter,
+            include_non_utf8: config.include_non_utf8,
             get_sha,
             client: reqwest::Client::new(),
             cache: Mutex::new(None),
@@ -95,16 +118,16 @@ impl RepoCode {
 
     /// Get the application name.
     pub fn name(&self) -> &str {
-        &self.config.name
+        &self.name
     }
 
-    /// Get the current tarball, downloading if necessary.
+    /// Get the current tarball, downloading or reading from file as needed.
     ///
     /// Returns a mutex guard containing the cached tarball. This method never fails;
     /// instead it logs warnings and returns whatever is currently cached:
     ///
     /// - If the SHA callback fails, returns the existing cache (possibly stale or None)
-    /// - If the tarball download fails, returns the existing cache
+    /// - If the tarball download/read fails, returns the existing cache
     /// - If tarball extraction fails, returns the existing cache
     ///
     /// This design assumes that stale code is better than no code, since most of the
@@ -113,29 +136,29 @@ impl RepoCode {
         let current_sha = match (self.get_sha)().await {
             Ok(sha) => sha,
             Err(e) => {
-                warn!("Failed to get current SHA for {}: {e}", self.config.name);
+                warn!("Failed to get current SHA for {}: {e}", self.name);
                 return self.cache.lock().await;
             }
         };
 
         let mut cache = self.cache.lock().await;
 
-        // Check if we already have this SHA cached
+        // Check if we need to load/reload the tarball.
-        let needs_download = match &*cache {
+        // For file sources, we only load once (no refresh after initial load).
-            Some(cached) => cached.sha != current_sha,
+        // For GitHub sources, we reload when the SHA changes.
-            None => true,
+        let needs_refresh = match (&*cache, &self.source) {
+            (None, _) => true,
+            (Some(_), ResolvedSource::File { .. }) => false,
+            (Some(cached), ResolvedSource::GitHub { .. }) => cached.sha != current_sha,
         };
 
-        if needs_download {
+        if needs_refresh {
-            info!(
+            info!("Loading tarball for {} at {}", self.name, current_sha);
-                "Downloading tarball for {} at {}",
-                self.config.name, current_sha
-            );
 
-            let tarball = match self.download_tarball(&current_sha).await {
+            let tarball = match self.load_tarball(&current_sha).await {
                 Ok(t) => t,
                 Err(e) => {
-                    error!("Failed to download tarball for {}: {e}", self.config.name);
+                    error!("Failed to load tarball for {}: {e}", self.name);
                     return cache;
                 }
             };
@@ -143,7 +166,7 @@ impl RepoCode {
             let files = match self.extract_tarball(&tarball) {
                 Ok(f) => f,
                 Err(e) => {
-                    error!("Failed to extract tarball for {}: {e}", self.config.name);
+                    error!("Failed to extract tarball for {}: {e}", self.name);
                     return cache;
                 }
             };
@@ -157,11 +180,30 @@ impl RepoCode {
         cache
     }
 
+    /// Load a tarball either from GitHub or from a local file.
+    async fn load_tarball(&self, sha: &str) -> Result<Vec<u8>, String> {
+        match &self.source {
+            ResolvedSource::GitHub { owner, repo, token } => {
+                self.download_tarball_from_github(owner, repo, token.as_deref(), sha)
+                    .await
+            }
+            ResolvedSource::File { path } => std::fs::read(path).map_err(|e| {
+                format!("Failed to read tarball from {}: {e}", path.display())
+            }),
+        }
+    }
+
     /// Download a tarball from GitHub for the given SHA.
-    async fn download_tarball(&self, sha: &str) -> Result<Vec<u8>, String> {
+    async fn download_tarball_from_github(
+        &self,
+        owner: &str,
+        repo: &str,
+        token: Option<&str>,
+        sha: &str,
+    ) -> Result<Vec<u8>, String> {
         let url = format!(
             "https://api.github.com/repos/{}/{}/tarball/{}",
-            self.config.github.owner, self.config.github.repo, sha
+            owner, repo, sha
         );
 
         let mut request = self
@@ -171,7 +213,7 @@ impl RepoCode {
             .header("User-Agent", "signal-gateway")
             .header("X-GitHub-Api-Version", "2022-11-28");
 
-        if let Some(token) = &self.token {
+        if let Some(token) = token {
             request = request.header("Authorization", format!("Bearer {}", token));
         }
 
@@ -244,7 +286,7 @@ impl RepoCode {
             let content = match String::from_utf8(contents) {
                 Ok(s) => s,
                 Err(e) => {
-                    if self.config.include_non_utf8 {
+                    if self.include_non_utf8 {
                         // Use lossy conversion if configured to include non-UTF-8
                         String::from_utf8_lossy(e.as_bytes()).into_owned()
                     } else {

signal-gateway-repo-code/tests/integration.rs

@@ -3,7 +3,7 @@
 //! These tests exercise the GitHub tarball download and file browsing functionality
 //! against a real public repository at a pinned commit.
 
-use signal_gateway_repo_code::{GitHubRepo, RepoCode, RepoCodeConfig, ShaCallback};
+use signal_gateway_repo_code::{GitHubRepo, RepoCode, RepoCodeConfig, ShaCallback, Source};
 use std::sync::Arc;
 
 /// Test against cbeck88/ver-stub-rs at a known commit.
@@ -12,18 +12,20 @@ const TEST_OWNER: &str = "cbeck88";
 const TEST_REPO: &str = "ver-stub-rs";
 const TEST_SHA: &str = "79b98e25f27ae4f5dd73a5a3d8f37dad655a57e8";
 
-fn create_test_app_code() -> RepoCode {
+fn create_test_repo_code() -> RepoCode {
-    create_test_app_code_with_glob(vec![])
+    create_test_repo_code_with_glob(vec![])
 }
 
-fn create_test_app_code_with_glob(glob: Vec<String>) -> RepoCode {
+fn create_test_repo_code_with_glob(glob: Vec<String>) -> RepoCode {
     let config = RepoCodeConfig {
         name: "test-app".to_string(),
-        github: GitHubRepo {
+        source: Source::GitHub {
+            repo: GitHubRepo {
                 owner: TEST_OWNER.to_string(),
                 repo: TEST_REPO.to_string(),
             },
             token_file: None, // Public repo, no auth needed
+        },
         glob,
         include_non_utf8: false,
     };
@@ -39,7 +41,7 @@ fn create_test_app_code_with_glob(glob: Vec<String>) -> RepoCode {
 
 #[tokio::test]
 async fn test_ls_root() {
-    let app = create_test_app_code();
+    let app = create_test_repo_code();
 
     let result = app.ls(None).await.expect("ls failed");
 
@@ -59,7 +61,7 @@ async fn test_ls_root() {
 
 #[tokio::test]
 async fn test_ls_subdirectory() {
-    let app = create_test_app_code();
+    let app = create_test_repo_code();
 
     let result = app.ls(Some("ver-stub")).await.expect("ls failed");
 
@@ -70,7 +72,7 @@ async fn test_ls_subdirectory() {
 
 #[tokio::test]
 async fn test_find_rust_files() {
-    let app = create_test_app_code();
+    let app = create_test_repo_code();
 
     let result = app.find(Some("*.rs")).await.expect("find failed");
 
@@ -87,7 +89,7 @@ async fn test_find_rust_files() {
 
 #[tokio::test]
 async fn test_find_with_path_pattern() {
-    let app = create_test_app_code();
+    let app = create_test_repo_code();
 
     let result = app
         .find(Some("ver-stub-build/src/*.rs"))
@@ -103,7 +105,7 @@ async fn test_find_with_path_pattern() {
 
 #[tokio::test]
 async fn test_read_cargo_toml() {
-    let app = create_test_app_code();
+    let app = create_test_repo_code();
 
     let result = app
         .read("Cargo.toml", None, None)
@@ -121,7 +123,7 @@ async fn test_read_cargo_toml() {
 
 #[tokio::test]
 async fn test_read_with_line_range() {
-    let app = create_test_app_code();
+    let app = create_test_repo_code();
 
     // Read just the first 5 lines
     let result = app
@@ -142,7 +144,7 @@ async fn test_read_with_line_range() {
 
 #[tokio::test]
 async fn test_read_nonexistent_file() {
-    let app = create_test_app_code();
+    let app = create_test_repo_code();
 
     let result = app.read("nonexistent-file.txt", None, None).await;
 
@@ -155,7 +157,7 @@ async fn test_read_nonexistent_file() {
 
 #[tokio::test]
 async fn test_search_simple() {
-    let app = create_test_app_code();
+    let app = create_test_repo_code();
 
     let result = app
         .search("workspace", 0, None)
@@ -171,7 +173,7 @@ async fn test_search_simple() {
 
 #[tokio::test]
 async fn test_search_with_context() {
-    let app = create_test_app_code();
+    let app = create_test_repo_code();
 
     let result = app
         .search("resolver", 2, None)
@@ -192,7 +194,7 @@ async fn test_search_with_context() {
 
 #[tokio::test]
 async fn test_search_with_path_prefix() {
-    let app = create_test_app_code();
+    let app = create_test_repo_code();
 
     // Search only in ver-stub-build directory
     let result = app
@@ -215,7 +217,7 @@ async fn test_search_with_path_prefix() {
 
 #[tokio::test]
 async fn test_search_no_matches() {
-    let app = create_test_app_code();
+    let app = create_test_repo_code();
 
     let result = app
         .search("xyzzy_unlikely_string_12345", 0, None)
@@ -228,7 +230,7 @@ async fn test_search_no_matches() {
 #[tokio::test]
 async fn test_glob_filter_rust_files_only() {
     // Only include .rs files
-    let app = create_test_app_code_with_glob(vec!["**/*.rs".to_string()]);
+    let app = create_test_repo_code_with_glob(vec!["**/*.rs".to_string()]);
 
     let result = app.find(Some("*")).await.expect("find failed");
 
@@ -250,7 +252,7 @@ async fn test_glob_filter_rust_files_only() {
 #[tokio::test]
 async fn test_glob_filter_specific_directory() {
     // Only include files in ver-stub/src
-    let app = create_test_app_code_with_glob(vec!["ver-stub/src/**".to_string()]);
+    let app = create_test_repo_code_with_glob(vec!["ver-stub/src/**".to_string()]);
 
     let result = app.find(Some("*")).await.expect("find failed");
 
@@ -274,7 +276,7 @@ async fn test_glob_filter_specific_directory() {
 #[tokio::test]
 async fn test_glob_filter_multiple_patterns() {
     // Include both Cargo.toml files and shell scripts
-    let app = create_test_app_code_with_glob(vec!["**/Cargo.toml".to_string(), "*.sh".to_string()]);
+    let app = create_test_repo_code_with_glob(vec!["**/Cargo.toml".to_string(), "*.sh".to_string()]);
 
     let result = app.find(Some("*")).await.expect("find failed");
 
@@ -295,7 +297,7 @@ async fn test_glob_filter_multiple_patterns() {
 #[tokio::test]
 async fn test_glob_filter_ls_shows_filtered_dirs() {
     // Only include files in ver-stub directory
-    let app = create_test_app_code_with_glob(vec!["ver-stub/**".to_string()]);
+    let app = create_test_repo_code_with_glob(vec!["ver-stub/**".to_string()]);
 
     let result = app.ls(None).await.expect("ls failed");