Labyricorn/signal-gateway
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

make signal-gateway-repo-code also support reading code from a static tarball

Browse Source
This commit is contained in:
Chris Beck
2025-12-18 09:50:23 -07:00
parent 6f1af0d01f
commit 63c7974184
3 changed files with 245 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files
signal-gateway-repo-code/src/config.rs
+131 -7
View File
@@ -4,7 +4,7 @@ use serde::Deserialize;
use std::{path::PathBuf, str::FromStr};
/// A GitHub repository identifier (owner/repo).
#[derive(Clone, Debug, Deserialize)]
#[derive(Clone, Debug, Deserialize, PartialEq, Eq)]
#[serde(try_from = "String")]
pub struct GitHubRepo {
/// The repository owner (user or organization).
@@ -39,16 +39,34 @@ impl TryFrom<String> for GitHubRepo {
}
}
/// Source of repository code - either a GitHub repo or a local tarball file.
#[derive(Clone, Debug, Deserialize, PartialEq, Eq)]
pub enum Source {
/// GitHub repository source. Downloads tarballs from GitHub API.
#[serde(rename = "github")]
GitHub {
/// GitHub repository in "owner/repo" format.
repo: GitHubRepo,
/// Path to file containing the GitHub personal access token.
/// Optional for public repositories (unauthenticated access has lower rate limits).
token_file: Option<PathBuf>,
},
/// Local tarball file source. Reads a .tar.gz file from disk.
#[serde(rename = "file")]
File {
/// Path to the tarball file (.tar.gz).
path: PathBuf,
},
}
/// Configuration for an application's source code access.
#[derive(Clone, Debug, Deserialize)]
#[derive(Clone, Debug, Deserialize, PartialEq, Eq)]
pub struct RepoCodeConfig {
/// Name of the application (used to identify it in tool calls).
pub name: String,
/// GitHub repository in "owner/repo" format.
pub github: GitHubRepo,
/// Path to file containing the GitHub personal access token.
/// Optional for public repositories (unauthenticated access has lower rate limits).
pub token_file: Option<PathBuf>,
/// Source of the repository code.
#[serde(flatten)]
pub source: Source,
/// Glob patterns to filter which files are included from the tarball.
/// If non-empty, only files matching at least one pattern are kept.
/// Uses gitignore-style glob syntax (e.g., "*.rs", "src/**/*.rs").
@@ -59,3 +77,109 @@ pub struct RepoCodeConfig {
#[serde(default)]
pub include_non_utf8: bool,
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_parse_github_source() {
let json = r#"{
"name": "my-app",
"github": {
"repo": "owner/repo-name",
"token_file": "/path/to/token"
}
}"#;
let config: RepoCodeConfig = serde_json::from_str(json).unwrap();
assert_eq!(config.name, "my-app");
assert_eq!(
config.source,
Source::GitHub {
repo: GitHubRepo {
owner: "owner".to_string(),
repo: "repo-name".to_string(),
},
token_file: Some(PathBuf::from("/path/to/token")),
}
);
assert!(config.glob.is_empty());
assert!(!config.include_non_utf8);
}
#[test]
fn test_parse_github_source_no_token() {
let json = r#"{
"name": "public-app",
"github": {
"repo": "org/public-repo"
}
}"#;
let config: RepoCodeConfig = serde_json::from_str(json).unwrap();
assert_eq!(config.name, "public-app");
assert_eq!(
config.source,
Source::GitHub {
repo: GitHubRepo {
owner: "org".to_string(),
repo: "public-repo".to_string(),
},
token_file: None,
}
);
}
#[test]
fn test_parse_file_source() {
let json = r#"{
"name": "local-app",
"file": {
"path": "/tmp/source.tar.gz"
}
}"#;
let config: RepoCodeConfig = serde_json::from_str(json).unwrap();
assert_eq!(config.name, "local-app");
assert_eq!(
config.source,
Source::File {
path: PathBuf::from("/tmp/source.tar.gz"),
}
);
}
#[test]
fn test_parse_with_glob_and_options() {
let json = r#"{
"name": "filtered-app",
"github": {
"repo": "owner/repo"
},
"glob": ["**/*.rs", "Cargo.toml"],
"include_non_utf8": true
}"#;
let config: RepoCodeConfig = serde_json::from_str(json).unwrap();
assert_eq!(config.name, "filtered-app");
assert_eq!(config.glob, vec!["**/*.rs", "Cargo.toml"]);
assert!(config.include_non_utf8);
}
#[test]
fn test_parse_github_repo_string() {
let repo: GitHubRepo = "owner/repo".parse().unwrap();
assert_eq!(repo.owner, "owner");
assert_eq!(repo.repo, "repo");
}
#[test]
fn test_parse_github_repo_invalid() {
assert!("invalid".parse::<GitHubRepo>().is_err());
assert!("".parse::<GitHubRepo>().is_err());
assert!("/repo".parse::<GitHubRepo>().is_err());
assert!("owner/".parse::<GitHubRepo>().is_err());
assert!("a/b/c".parse::<GitHubRepo>().is_err());
}
}
signal-gateway-repo-code/src/lib.rs
+89 -47
View File
@@ -5,7 +5,7 @@
mod config;
pub use config::{GitHubRepo, RepoCodeConfig};
pub use config::{GitHubRepo, RepoCodeConfig, Source};
use async_trait::async_trait;
use flate2::read::GzDecoder;
@@ -42,13 +42,26 @@ pub type ShaCallback = Arc<
+ Sync,
>;
/// Internal representation of the resolved source.
enum ResolvedSource {
GitHub {
owner: String,
repo: String,
token: Option<String>,
},
File {
path: std::path::PathBuf,
},
}
/// Application source code browser.
///
/// Downloads and caches GitHub tarballs for browsing application source code.
pub struct RepoCode {
config: RepoCodeConfig,
token: Option<String>,
name: String,
source: ResolvedSource,
glob_filter: Option<GlobSet>,
include_non_utf8: bool,
get_sha: ShaCallback,
client: reqwest::Client,
cache: Mutex<Option<CachedTarball>>,
@@ -57,36 +70,46 @@ pub struct RepoCode {
impl RepoCode {
/// Create a new RepoCode instance from configuration.
///
/// The `get_sha` callback is called to determine which git SHA to download.
/// It should return `None` if the SHA is not yet known.
/// The `get_sha` callback is called to determine which git SHA to download/load.
/// For GitHub sources, this is the commit SHA. For file sources, this can be
/// used to track file modification (e.g., mtime or a version string).
pub fn new(config: RepoCodeConfig, get_sha: ShaCallback) -> Result<Self, std::io::Error> {
let token = config
.token_file
.as_ref()
.map(|path| std::fs::read_to_string(path).map(|s| s.trim().to_string()))
.transpose()?;
let source = match config.source {
Source::GitHub { repo, token_file } => {
let token = token_file
.as_ref()
.map(|path| std::fs::read_to_string(path).map(|s| s.trim().to_string()))
.transpose()?;
ResolvedSource::GitHub {
owner: repo.owner,
repo: repo.repo,
token,
}
}
Source::File { path } => ResolvedSource::File { path },
};
// Compile glob patterns if any are specified
let glob_filter =
if config.glob.is_empty() {
None
} else {
let mut builder = GlobSetBuilder::new();
for pattern in &config.glob {
let glob = Glob::new(pattern).map_err(|e| {
std::io::Error::other(format!("invalid glob pattern '{}': {}", pattern, e))
})?;
builder.add(glob);
}
Some(builder.build().map_err(|e| {
std::io::Error::other(format!("failed to build glob set: {}", e))
})?)
};
let glob_filter = if config.glob.is_empty() {
None
} else {
let mut builder = GlobSetBuilder::new();
for pattern in &config.glob {
let glob = Glob::new(pattern).map_err(|e| {
std::io::Error::other(format!("invalid glob pattern '{}': {}", pattern, e))
})?;
builder.add(glob);
}
Some(builder.build().map_err(|e| {
std::io::Error::other(format!("failed to build glob set: {}", e))
})?)
};
Ok(Self {
config,
token,
name: config.name,
source,
glob_filter,
include_non_utf8: config.include_non_utf8,
get_sha,
client: reqwest::Client::new(),
cache: Mutex::new(None),
@@ -95,16 +118,16 @@ impl RepoCode {
/// Get the application name.
pub fn name(&self) -> &str {
&self.config.name
&self.name
}
/// Get the current tarball, downloading if necessary.
/// Get the current tarball, downloading or reading from file as needed.
///
/// Returns a mutex guard containing the cached tarball. This method never fails;
/// instead it logs warnings and returns whatever is currently cached:
///
/// - If the SHA callback fails, returns the existing cache (possibly stale or None)
/// - If the tarball download fails, returns the existing cache
/// - If the tarball download/read fails, returns the existing cache
/// - If tarball extraction fails, returns the existing cache
///
/// This design assumes that stale code is better than no code, since most of the
@@ -113,29 +136,29 @@ impl RepoCode {
let current_sha = match (self.get_sha)().await {
Ok(sha) => sha,
Err(e) => {
warn!("Failed to get current SHA for {}: {e}", self.config.name);
warn!("Failed to get current SHA for {}: {e}", self.name);
return self.cache.lock().await;
}
};
let mut cache = self.cache.lock().await;
// Check if we already have this SHA cached
let needs_download = match &*cache {
Some(cached) => cached.sha != current_sha,
None => true,
// Check if we need to load/reload the tarball.
// For file sources, we only load once (no refresh after initial load).
// For GitHub sources, we reload when the SHA changes.
let needs_refresh = match (&*cache, &self.source) {
(None, _) => true,
(Some(_), ResolvedSource::File { .. }) => false,
(Some(cached), ResolvedSource::GitHub { .. }) => cached.sha != current_sha,
};
if needs_download {
info!(
"Downloading tarball for {} at {}",
self.config.name, current_sha
);
if needs_refresh {
info!("Loading tarball for {} at {}", self.name, current_sha);
let tarball = match self.download_tarball(&current_sha).await {
let tarball = match self.load_tarball(&current_sha).await {
Ok(t) => t,
Err(e) => {
error!("Failed to download tarball for {}: {e}", self.config.name);
error!("Failed to load tarball for {}: {e}", self.name);
return cache;
}
};
@@ -143,7 +166,7 @@ impl RepoCode {
let files = match self.extract_tarball(&tarball) {
Ok(f) => f,
Err(e) => {
error!("Failed to extract tarball for {}: {e}", self.config.name);
error!("Failed to extract tarball for {}: {e}", self.name);
return cache;
}
};
@@ -157,11 +180,30 @@ impl RepoCode {
cache
}
/// Load a tarball either from GitHub or from a local file.
async fn load_tarball(&self, sha: &str) -> Result<Vec<u8>, String> {
match &self.source {
ResolvedSource::GitHub { owner, repo, token } => {
self.download_tarball_from_github(owner, repo, token.as_deref(), sha)
.await
}
ResolvedSource::File { path } => std::fs::read(path).map_err(|e| {
format!("Failed to read tarball from {}: {e}", path.display())
}),
}
}
/// Download a tarball from GitHub for the given SHA.
async fn download_tarball(&self, sha: &str) -> Result<Vec<u8>, String> {
async fn download_tarball_from_github(
&self,
owner: &str,
repo: &str,
token: Option<&str>,
sha: &str,
) -> Result<Vec<u8>, String> {
let url = format!(
"https://api.github.com/repos/{}/{}/tarball/{}",
self.config.github.owner, self.config.github.repo, sha
owner, repo, sha
);
let mut request = self
@@ -171,7 +213,7 @@ impl RepoCode {
.header("User-Agent", "signal-gateway")
.header("X-GitHub-Api-Version", "2022-11-28");
if let Some(token) = &self.token {
if let Some(token) = token {
request = request.header("Authorization", format!("Bearer {}", token));
}
@@ -244,7 +286,7 @@ impl RepoCode {
let content = match String::from_utf8(contents) {
Ok(s) => s,
Err(e) => {
if self.config.include_non_utf8 {
if self.include_non_utf8 {
// Use lossy conversion if configured to include non-UTF-8
String::from_utf8_lossy(e.as_bytes()).into_owned()
} else {
signal-gateway-repo-code/tests/integration.rs
+25 -23
View File
@@ -3,7 +3,7 @@
//! These tests exercise the GitHub tarball download and file browsing functionality
//! against a real public repository at a pinned commit.
use signal_gateway_repo_code::{GitHubRepo, RepoCode, RepoCodeConfig, ShaCallback};
use signal_gateway_repo_code::{GitHubRepo, RepoCode, RepoCodeConfig, ShaCallback, Source};
use std::sync::Arc;
/// Test against cbeck88/ver-stub-rs at a known commit.
@@ -12,18 +12,20 @@ const TEST_OWNER: &str = "cbeck88";
const TEST_REPO: &str = "ver-stub-rs";
const TEST_SHA: &str = "79b98e25f27ae4f5dd73a5a3d8f37dad655a57e8";
fn create_test_app_code() -> RepoCode {
create_test_app_code_with_glob(vec![])
fn create_test_repo_code() -> RepoCode {
create_test_repo_code_with_glob(vec![])
}
fn create_test_app_code_with_glob(glob: Vec<String>) -> RepoCode {
fn create_test_repo_code_with_glob(glob: Vec<String>) -> RepoCode {
let config = RepoCodeConfig {
name: "test-app".to_string(),
github: GitHubRepo {
owner: TEST_OWNER.to_string(),
repo: TEST_REPO.to_string(),
source: Source::GitHub {
repo: GitHubRepo {
owner: TEST_OWNER.to_string(),
repo: TEST_REPO.to_string(),
},
token_file: None, // Public repo, no auth needed
},
token_file: None, // Public repo, no auth needed
glob,
include_non_utf8: false,
};
@@ -39,7 +41,7 @@ fn create_test_app_code_with_glob(glob: Vec<String>) -> RepoCode {
#[tokio::test]
async fn test_ls_root() {
let app = create_test_app_code();
let app = create_test_repo_code();
let result = app.ls(None).await.expect("ls failed");
@@ -59,7 +61,7 @@ async fn test_ls_root() {
#[tokio::test]
async fn test_ls_subdirectory() {
let app = create_test_app_code();
let app = create_test_repo_code();
let result = app.ls(Some("ver-stub")).await.expect("ls failed");
@@ -70,7 +72,7 @@ async fn test_ls_subdirectory() {
#[tokio::test]
async fn test_find_rust_files() {
let app = create_test_app_code();
let app = create_test_repo_code();
let result = app.find(Some("*.rs")).await.expect("find failed");
@@ -87,7 +89,7 @@ async fn test_find_rust_files() {
#[tokio::test]
async fn test_find_with_path_pattern() {
let app = create_test_app_code();
let app = create_test_repo_code();
let result = app
.find(Some("ver-stub-build/src/*.rs"))
@@ -103,7 +105,7 @@ async fn test_find_with_path_pattern() {
#[tokio::test]
async fn test_read_cargo_toml() {
let app = create_test_app_code();
let app = create_test_repo_code();
let result = app
.read("Cargo.toml", None, None)
@@ -121,7 +123,7 @@ async fn test_read_cargo_toml() {
#[tokio::test]
async fn test_read_with_line_range() {
let app = create_test_app_code();
let app = create_test_repo_code();
// Read just the first 5 lines
let result = app
@@ -142,7 +144,7 @@ async fn test_read_with_line_range() {
#[tokio::test]
async fn test_read_nonexistent_file() {
let app = create_test_app_code();
let app = create_test_repo_code();
let result = app.read("nonexistent-file.txt", None, None).await;
@@ -155,7 +157,7 @@ async fn test_read_nonexistent_file() {
#[tokio::test]
async fn test_search_simple() {
let app = create_test_app_code();
let app = create_test_repo_code();
let result = app
.search("workspace", 0, None)
@@ -171,7 +173,7 @@ async fn test_search_simple() {
#[tokio::test]
async fn test_search_with_context() {
let app = create_test_app_code();
let app = create_test_repo_code();
let result = app
.search("resolver", 2, None)
@@ -192,7 +194,7 @@ async fn test_search_with_context() {
#[tokio::test]
async fn test_search_with_path_prefix() {
let app = create_test_app_code();
let app = create_test_repo_code();
// Search only in ver-stub-build directory
let result = app
@@ -215,7 +217,7 @@ async fn test_search_with_path_prefix() {
#[tokio::test]
async fn test_search_no_matches() {
let app = create_test_app_code();
let app = create_test_repo_code();
let result = app
.search("xyzzy_unlikely_string_12345", 0, None)
@@ -228,7 +230,7 @@ async fn test_search_no_matches() {
#[tokio::test]
async fn test_glob_filter_rust_files_only() {
// Only include .rs files
let app = create_test_app_code_with_glob(vec!["**/*.rs".to_string()]);
let app = create_test_repo_code_with_glob(vec!["**/*.rs".to_string()]);
let result = app.find(Some("*")).await.expect("find failed");
@@ -250,7 +252,7 @@ async fn test_glob_filter_rust_files_only() {
#[tokio::test]
async fn test_glob_filter_specific_directory() {
// Only include files in ver-stub/src
let app = create_test_app_code_with_glob(vec!["ver-stub/src/**".to_string()]);
let app = create_test_repo_code_with_glob(vec!["ver-stub/src/**".to_string()]);
let result = app.find(Some("*")).await.expect("find failed");
@@ -274,7 +276,7 @@ async fn test_glob_filter_specific_directory() {
#[tokio::test]
async fn test_glob_filter_multiple_patterns() {
// Include both Cargo.toml files and shell scripts
let app = create_test_app_code_with_glob(vec!["**/Cargo.toml".to_string(), "*.sh".to_string()]);
let app = create_test_repo_code_with_glob(vec!["**/Cargo.toml".to_string(), "*.sh".to_string()]);
let result = app.find(Some("*")).await.expect("find failed");
@@ -295,7 +297,7 @@ async fn test_glob_filter_multiple_patterns() {
#[tokio::test]
async fn test_glob_filter_ls_shows_filtered_dirs() {
// Only include files in ver-stub directory
let app = create_test_app_code_with_glob(vec!["ver-stub/**".to_string()]);
let app = create_test_repo_code_with_glob(vec!["ver-stub/**".to_string()]);
let result = app.ls(None).await.expect("ls failed");