432 lines
18 KiB
Python
432 lines
18 KiB
Python
"""
|
|
Gitea Service Adapter
|
|
Handles Idea Dossier repository creation, full document tree synchronization,
|
|
organization management (under 'thinkstorm' org), and OAuth2 authentication.
|
|
Uses direct Gitea REST API v1 over HTTPS for fast, reliable, atomic synchronization.
|
|
"""
|
|
|
|
import json
|
|
import time
|
|
import re
|
|
import base64
|
|
import socket
|
|
import asyncio
|
|
import urllib3.util.connection as urllib3_cn
|
|
|
|
try:
|
|
urllib3_cn.allowed_gai_family = lambda: socket.AF_INET
|
|
except Exception:
|
|
pass
|
|
|
|
import requests
|
|
from pathlib import Path
|
|
from typing import Dict, Any, Optional, List
|
|
|
|
from .base import BaseServiceAdapter, ServiceHealth
|
|
from ..config import config, BASE_DIR
|
|
|
|
ARTIFACTS_DIR = BASE_DIR / "data" / "artifacts"
|
|
ARTIFACTS_DIR.mkdir(parents=True, exist_ok=True)
|
|
|
|
class GiteaAdapter(BaseServiceAdapter):
|
|
def __init__(self, endpoint: str = "https://git.labyricorn.com", api_token: str = ""):
|
|
super().__init__(service_id="gitea", endpoint=endpoint, api_key=api_token or config.services.gitea_api_token)
|
|
self.org_name = "thinkstorm"
|
|
|
|
def get_effective_token(self) -> str:
|
|
"""Retrieves configured API token from instance, DB configuration, or environment."""
|
|
if self.api_key:
|
|
return self.api_key
|
|
try:
|
|
from ..database import get_db
|
|
with get_db() as conn:
|
|
row = conn.execute("SELECT api_key_raw FROM service_configurations WHERE id = 'gitea'").fetchone()
|
|
if row and row["api_key_raw"]:
|
|
return row["api_key_raw"]
|
|
except Exception:
|
|
pass
|
|
return config.services.gitea_api_token or ""
|
|
|
|
def _get_headers(self, token: Optional[str] = None) -> Dict[str, str]:
|
|
t = token or self.get_effective_token()
|
|
headers = {
|
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",
|
|
"Content-Type": "application/json"
|
|
}
|
|
if t:
|
|
headers["Authorization"] = f"token {t}"
|
|
return headers
|
|
|
|
async def check_health(self) -> ServiceHealth:
|
|
start = time.time()
|
|
loop = asyncio.get_running_loop()
|
|
try:
|
|
url = f"{self.endpoint}/api/v1/version"
|
|
headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}
|
|
def fetch():
|
|
return requests.get(url, headers=headers, timeout=15.0)
|
|
resp = await loop.run_in_executor(None, fetch)
|
|
elapsed = int((time.time() - start) * 1000)
|
|
if resp.status_code == 200:
|
|
data = resp.json()
|
|
version = data.get("version", "unknown")
|
|
return ServiceHealth(
|
|
service_id=self.service_id,
|
|
healthy=True,
|
|
endpoint=self.endpoint,
|
|
message=f"Gitea online (v{version})",
|
|
response_time_ms=elapsed,
|
|
extra={"version": version}
|
|
)
|
|
return ServiceHealth(
|
|
service_id=self.service_id,
|
|
healthy=False,
|
|
endpoint=self.endpoint,
|
|
message=f"Gitea returned HTTP {resp.status_code}",
|
|
response_time_ms=elapsed
|
|
)
|
|
except Exception as e:
|
|
elapsed = int((time.time() - start) * 1000)
|
|
return ServiceHealth(
|
|
service_id=self.service_id,
|
|
healthy=False,
|
|
endpoint=self.endpoint,
|
|
message=f"Gitea connection error: {str(e)}",
|
|
response_time_ms=elapsed
|
|
)
|
|
|
|
def _ensure_org(self, session: requests.Session, token: str) -> None:
|
|
"""Ensures the 'thinkstorm' organization exists."""
|
|
headers = self._get_headers(token)
|
|
try:
|
|
r = session.get(f"{self.endpoint}/api/v1/orgs/{self.org_name}", headers=headers, timeout=60)
|
|
if r.status_code == 200:
|
|
return
|
|
if r.status_code == 404:
|
|
payload = {
|
|
"username": self.org_name,
|
|
"full_name": "ThinkStorm Idea Incubation",
|
|
"description": "Canonical dossiers, research, and project incubations generated by ThinkStorm",
|
|
"visibility": "public"
|
|
}
|
|
session.post(f"{self.endpoint}/api/v1/orgs", json=payload, headers=headers, timeout=60)
|
|
except Exception as e:
|
|
print(f"[Gitea] Org ensure notice: {e}")
|
|
|
|
def _ensure_repo(self, session: requests.Session, token: str, repo_name: str, description: str) -> bool:
|
|
"""Ensures the repository exists under the organization."""
|
|
headers = self._get_headers(token)
|
|
try:
|
|
r = session.get(f"{self.endpoint}/api/v1/repos/{self.org_name}/{repo_name}", headers=headers, timeout=60)
|
|
if r.status_code == 200:
|
|
return True
|
|
if r.status_code == 404:
|
|
payload = {
|
|
"name": repo_name,
|
|
"description": description,
|
|
"private": False,
|
|
"auto_init": True,
|
|
"default_branch": "main"
|
|
}
|
|
cr = session.post(f"{self.endpoint}/api/v1/orgs/{self.org_name}/repos", json=payload, headers=headers, timeout=60)
|
|
return cr.status_code in (200, 201)
|
|
except Exception as e:
|
|
print(f"[Gitea] Repo ensure error for {repo_name}: {e}")
|
|
return False
|
|
return False
|
|
|
|
def _grant_repo_write_access(
|
|
self,
|
|
session: requests.Session,
|
|
token: str,
|
|
repo_name: str,
|
|
username: str
|
|
) -> None:
|
|
"""Idempotently grants the verified claimant write access to a repository."""
|
|
safe_username = requests.utils.quote(username, safe="")
|
|
url = (
|
|
f"{self.endpoint}/api/v1/repos/{self.org_name}/{repo_name}"
|
|
f"/collaborators/{safe_username}"
|
|
)
|
|
response = session.put(
|
|
url,
|
|
json={"permission": "write"},
|
|
headers=self._get_headers(token),
|
|
timeout=60
|
|
)
|
|
if response.status_code not in (200, 201, 204):
|
|
raise RuntimeError(
|
|
f"Could not grant Gitea write access to '{username}' "
|
|
f"(HTTP {response.status_code})."
|
|
)
|
|
|
|
def _sync_files_via_api(self, token: str, repo_slug: str, files_dict: Dict[str, str]) -> None:
|
|
"""Commits or updates multiple files directly into the Gitea repository via Contents API."""
|
|
if not files_dict or not token:
|
|
return
|
|
|
|
headers = self._get_headers(token)
|
|
with requests.Session() as s:
|
|
for path, content in files_dict.items():
|
|
try:
|
|
# 1. Check if file already exists in repo to get current SHA
|
|
get_url = f"{self.endpoint}/api/v1/repos/{self.org_name}/{repo_slug}/contents/{path}"
|
|
r = s.get(get_url, headers=headers, timeout=60)
|
|
sha = None
|
|
if r.status_code == 200:
|
|
sha = r.json().get("sha")
|
|
|
|
b64_content = base64.b64encode(content.encode("utf-8")).decode("utf-8")
|
|
payload = {
|
|
"content": b64_content,
|
|
"message": f"Sync {path} into ThinkStorm dossier",
|
|
"branch": "main"
|
|
}
|
|
|
|
if sha:
|
|
payload["sha"] = sha
|
|
put_res = s.put(get_url, json=payload, headers=headers, timeout=60)
|
|
if put_res.status_code not in (200, 201):
|
|
print(f"[Gitea] Update notice for {path}: {put_res.status_code} {put_res.text[:100]}")
|
|
else:
|
|
post_res = s.post(get_url, json=payload, headers=headers, timeout=60)
|
|
if post_res.status_code not in (200, 201):
|
|
print(f"[Gitea] Create notice for {path}: {post_res.status_code} {post_res.text[:100]}")
|
|
except Exception as e:
|
|
print(f"[Gitea] File sync notice for {path}: {e}")
|
|
|
|
async def persist_idea_dossier_repo(
|
|
self,
|
|
idea_id: str,
|
|
title: str,
|
|
summary: str,
|
|
original_text: str,
|
|
categories: List[str],
|
|
tags: List[str],
|
|
lifecycle_state: str,
|
|
research_docs: Dict[str, str],
|
|
outputs: Dict[str, str],
|
|
provenance_runs: List[Dict[str, Any]],
|
|
existing_repo_url: Optional[str] = None,
|
|
submission_image: Optional[Dict[str, Any]] = None,
|
|
publish_remote: bool = True,
|
|
collaborator_username: Optional[str] = None
|
|
) -> Dict[str, Any]:
|
|
"""Persists a dossier locally and optionally publishes it to Gitea."""
|
|
# 1. Local disk directory
|
|
idea_dir = ARTIFACTS_DIR / idea_id
|
|
idea_dir.mkdir(parents=True, exist_ok=True)
|
|
(idea_dir / "research").mkdir(parents=True, exist_ok=True)
|
|
(idea_dir / "provenance").mkdir(parents=True, exist_ok=True)
|
|
(idea_dir / "outputs").mkdir(parents=True, exist_ok=True)
|
|
|
|
cat_str = ", ".join(categories) if categories else "General"
|
|
tag_str = ", ".join(f"#{t}" for t in tags) if tags else "None"
|
|
|
|
img_section = ""
|
|
if submission_image and submission_image.get("present"):
|
|
art_id = submission_image.get("artifact_id", "IMG-TS")
|
|
mime = submission_image.get("mime_type", "image/jpeg")
|
|
w = submission_image.get("width", 0)
|
|
h = submission_image.get("height", 0)
|
|
size_kb = round(submission_image.get("size_bytes", 0) / 1024, 1)
|
|
sha = submission_image.get("sha256", "")
|
|
img_section = (
|
|
f"## Reference Image\n\n"
|
|
f"- **Artifact ID:** `{art_id}`\n"
|
|
f"- **MIME Type:** `{mime}`\n"
|
|
f"- **Dimensions:** `{w}x{h}`\n"
|
|
f"- **File Size:** `{size_kb} KB`\n"
|
|
f"- **SHA-256:** `{sha}`\n\n"
|
|
)
|
|
|
|
ctx_section = ""
|
|
if "image-context.md" in research_docs:
|
|
ctx_section = f"## Image Context\n\n{research_docs['image-context.md']}\n\n"
|
|
|
|
# 2. Build README.md
|
|
readme_md = (
|
|
f"# {idea_id}: {title}\n\n"
|
|
f"**Lifecycle State:** `{lifecycle_state}` \n"
|
|
f"**Categories:** {cat_str} \n"
|
|
f"**Tags:** {tag_str} \n\n"
|
|
f"## Executive Summary\n{summary}\n\n"
|
|
f"## Original Submission Prompt\n> {original_text.strip()}\n\n"
|
|
f"{img_section}"
|
|
f"{ctx_section}"
|
|
f"## Project Structure\n"
|
|
f"- `research/`: Automated competitor analysis, prior art, deep research, and technical feasibility reports.\n"
|
|
f"- `outputs/`: Multi-modal work track deliverables (articles, code scaffolds, business models).\n"
|
|
f"- `provenance/`: Execution run telemetry, model audit trails, and token attribution logs.\n"
|
|
f"- `metadata.json`: Machine-readable metadata schema.\n"
|
|
)
|
|
(idea_dir / "README.md").write_text(readme_md, encoding="utf-8")
|
|
(idea_dir / "idea.md").write_text(readme_md, encoding="utf-8")
|
|
|
|
# 3. Build metadata.json
|
|
meta = {
|
|
"id": idea_id,
|
|
"title": title,
|
|
"summary": summary,
|
|
"categories": categories,
|
|
"tags": tags,
|
|
"lifecycle_state": lifecycle_state,
|
|
"updated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
|
"organization": self.org_name
|
|
}
|
|
meta_json_str = json.dumps(meta, indent=2)
|
|
(idea_dir / "metadata.json").write_text(meta_json_str, encoding="utf-8")
|
|
|
|
# 4. Research docs
|
|
files_to_sync = {
|
|
"README.md": readme_md,
|
|
"idea.md": readme_md,
|
|
"metadata.json": meta_json_str
|
|
}
|
|
|
|
for filename, content in research_docs.items():
|
|
safe_name = filename.replace("/", "_")
|
|
(idea_dir / "research" / safe_name).write_text(content, encoding="utf-8")
|
|
files_to_sync[f"research/{safe_name}"] = content
|
|
|
|
# 5. Outputs
|
|
for filename, content in outputs.items():
|
|
out_path = idea_dir / "outputs" / filename
|
|
out_path.parent.mkdir(parents=True, exist_ok=True)
|
|
out_path.write_text(content, encoding="utf-8")
|
|
files_to_sync[f"outputs/{filename}"] = content
|
|
|
|
# 6. Provenance
|
|
for run in provenance_runs:
|
|
run_id = run.get("id", f"run-{int(time.time())}")
|
|
run_json = json.dumps(run, indent=2)
|
|
(idea_dir / "provenance" / f"{run_id}.json").write_text(run_json, encoding="utf-8")
|
|
files_to_sync[f"provenance/{run_id}.json"] = run_json
|
|
|
|
# 7. Gitea Repository sync under 'thinkstorm' organization
|
|
clean_slug = re.sub(r'[^a-zA-Z0-9_-]', '-', idea_id.lower()).strip('-')
|
|
repo_name = f"ts-{clean_slug.replace('ts-', '')}"
|
|
repo_url = existing_repo_url or f"{self.endpoint}/{self.org_name}/{repo_name}"
|
|
clone_url = f"{self.endpoint}/{self.org_name}/{repo_name}.git"
|
|
ssh_url = f"ssh://[email protected]:22/{self.org_name}/{repo_name}.git"
|
|
|
|
token = self.get_effective_token() if publish_remote else ""
|
|
if publish_remote:
|
|
if not token:
|
|
raise RuntimeError("Gitea publishing is not configured with a service token.")
|
|
loop = asyncio.get_running_loop()
|
|
def sync_gitea():
|
|
with requests.Session() as s:
|
|
self._ensure_org(s, token)
|
|
if not self._ensure_repo(s, token, repo_name, f"[{idea_id}] {title}"):
|
|
raise RuntimeError(f"Could not create or access Gitea repository '{repo_name}'.")
|
|
if collaborator_username:
|
|
self._grant_repo_write_access(
|
|
s, token, repo_name, collaborator_username
|
|
)
|
|
self._sync_files_via_api(token, repo_name, files_to_sync)
|
|
return f"{self.endpoint}/{self.org_name}/{repo_name}"
|
|
|
|
actual_url = await loop.run_in_executor(None, sync_gitea)
|
|
repo_url = actual_url or repo_url
|
|
|
|
return {
|
|
"repo_name": f"{self.org_name}/{repo_name}",
|
|
"repo_url": repo_url,
|
|
"clone_url": clone_url,
|
|
"ssh_url": ssh_url,
|
|
"local_path": str(idea_dir),
|
|
"published": publish_remote
|
|
}
|
|
|
|
async def persist_work_track_outputs(
|
|
self,
|
|
idea_id: str,
|
|
track_name: str,
|
|
outputs: Dict[str, str],
|
|
publish_remote: bool = True
|
|
) -> Dict[str, Any]:
|
|
"""Persists Work Track deliverables locally and optionally pushes them."""
|
|
idea_dir = ARTIFACTS_DIR / idea_id
|
|
out_subdir = idea_dir / "outputs" / track_name.lower().replace(" ", "-")
|
|
out_subdir.mkdir(parents=True, exist_ok=True)
|
|
|
|
files_to_sync = {}
|
|
for filename, content in outputs.items():
|
|
file_path = out_subdir / filename
|
|
file_path.write_text(content, encoding="utf-8")
|
|
files_to_sync[f"outputs/{track_name.lower().replace(' ', '-')}/{filename}"] = content
|
|
|
|
clean_slug = re.sub(r'[^a-zA-Z0-9_-]', '-', idea_id.lower()).strip('-')
|
|
repo_name = f"ts-{clean_slug.replace('ts-', '')}"
|
|
repo_url = f"{self.endpoint}/{self.org_name}/{repo_name}"
|
|
|
|
token = self.get_effective_token() if publish_remote else ""
|
|
if token and files_to_sync:
|
|
loop = asyncio.get_running_loop()
|
|
try:
|
|
def do_sync():
|
|
with requests.Session() as s:
|
|
self._ensure_repo(s, token, repo_name, f"[{idea_id}] Project Dossier")
|
|
self._sync_files_via_api(token, repo_name, files_to_sync)
|
|
await loop.run_in_executor(None, do_sync)
|
|
except Exception as e:
|
|
print(f"[Gitea] Work track sync notice: {e}")
|
|
|
|
return {
|
|
"status": "SUCCESS" if publish_remote else "LOCAL_ONLY",
|
|
"path": str(out_subdir),
|
|
"repo_url": repo_url if publish_remote else ""
|
|
}
|
|
|
|
async def graduate_project(
|
|
self,
|
|
idea_id: str,
|
|
title: str,
|
|
summary: str,
|
|
repo_name: str = "",
|
|
collaborator_username: Optional[str] = None
|
|
) -> Dict[str, Any]:
|
|
"""Graduates an incubated Coding Project work track to a dedicated repository under 'thinkstorm'."""
|
|
clean_title = re.sub(r'[^a-zA-Z0-9_-]', '-', title.lower()).strip('-')[:30] or "project"
|
|
slug = repo_name or f"ts-{idea_id.lower()}-{clean_title}"
|
|
repo_url = f"{self.endpoint}/{self.org_name}/{slug}"
|
|
|
|
token = self.get_effective_token()
|
|
if not token:
|
|
raise RuntimeError("Gitea publishing is not configured with a service token.")
|
|
loop = asyncio.get_running_loop()
|
|
def create_remote():
|
|
with requests.Session() as s:
|
|
self._ensure_org(s, token)
|
|
if not self._ensure_repo(s, token, slug, f"[{idea_id}] {title}"):
|
|
raise RuntimeError(f"Could not create or access Gitea repository '{slug}'.")
|
|
if collaborator_username:
|
|
self._grant_repo_write_access(
|
|
s, token, slug, collaborator_username
|
|
)
|
|
return f"{self.endpoint}/{self.org_name}/{slug}"
|
|
actual_url = await loop.run_in_executor(None, create_remote)
|
|
repo_url = actual_url or repo_url
|
|
|
|
return {
|
|
"repo_name": f"{self.org_name}/{slug}",
|
|
"repo_url": repo_url,
|
|
"graduated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
|
}
|
|
|
|
async def verify_oauth_user(self, access_token: str) -> Optional[Dict[str, Any]]:
|
|
"""Fetches user profile details using Gitea OAuth access token."""
|
|
url = f"{self.endpoint}/api/v1/user"
|
|
loop = asyncio.get_running_loop()
|
|
def fetch():
|
|
return requests.get(url, headers=self._get_headers(access_token), timeout=8.0)
|
|
try:
|
|
resp = await loop.run_in_executor(None, fetch)
|
|
if resp.status_code == 200:
|
|
return resp.json()
|
|
return None
|
|
except Exception as e:
|
|
print(f"[Gitea] OAuth user verify error: {e}")
|
|
return None
|