Labyricorn/ThinkStorm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement optional single-image intake, Signal ingestion, and multimodal vision analysis

Browse Source 
- Add image intake service with format validation (JPEG, PNG, WebP) and EXIF/GPS stripping
- Enforce strict single-image rule across Web and Signal attachment channels
- Implement token-optimized vision downscaling and JPEG compression
- Add IMAGE_CONTEXT pipeline stage with OmniRoute vision routing and resilient failover
- Seed and manage versioned idea-image-interpreter prompt in catalog
- Update Web UI with responsive image picker, preview chip, and Visual Context tab
- Add comprehensive automated test suite in test_image_intake.py
- Update README and Labyricorn devlog
This commit is contained in:
xAdmin
2026-08-23 01:40:22 -07:00
parent 94ff1e4408
commit 41e08611c9
30 changed files with 3784 additions and 278 deletions
Download Patch File Download Diff File Expand all files Collapse all files
thinkstorm/services/omniroute.py
+47 -8
View File
@@ -7,6 +7,7 @@ Tracks token usage (input_tokens, output_tokens, total_tokens) and handles model
import json
import time
import re
import base64
import urllib.request
import asyncio
from typing import Dict, Any, Optional, Tuple
@@ -21,7 +22,7 @@ class OmniRouteAdapter(BaseServiceAdapter):
start = time.time()
try:
url = f"{self.endpoint}/models"
headers = {"User-Agent": "ThinkStorm-Orchestrator/0.1"}
headers = {"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"}
if self.api_key:
headers["Authorization"] = f"Bearer {self.api_key}"
req = urllib.request.Request(url, headers=headers)
@@ -57,6 +58,8 @@ class OmniRouteAdapter(BaseServiceAdapter):
return config.services.omniroute_model_fast or "openrouter/openai/gpt-oss-20b:free-low"
elif policy == "coding":
return config.services.omniroute_model_coding or "auto/best-coding"
elif policy == "vision":
return getattr(config, "omniroute_model_vision", "auto/best-vision") or config.services.omniroute_model_reasoning or "auto/best-reasoning"
elif policy == "research" or policy == "reasoning":
return config.services.omniroute_model_reasoning or "auto/best-reasoning"
return config.services.omniroute_model_reasoning or "auto/best-reasoning"
@@ -68,16 +71,35 @@ class OmniRouteAdapter(BaseServiceAdapter):
model_policy: str = "reasoning",
max_tokens: int = 1500,
temperature: float = 0.7,
model_override: Optional[str] = None
model_override: Optional[str] = None,
image_bytes: Optional[bytes] = None,
image_mime_type: Optional[str] = None
) -> Dict[str, Any]:
"""Executes a chat completion via OmniRoute with provenance token tracking."""
model = model_override.strip() if model_override and model_override.strip() else self.resolve_model(model_policy)
"""Executes a chat completion via OmniRoute with provenance token tracking and optional vision input."""
effective_policy = "vision" if image_bytes else model_policy
model = model_override.strip() if model_override and model_override.strip() else self.resolve_model(effective_policy)
url = f"{self.endpoint}/chat/completions"
if image_bytes:
b64_img = base64.b64encode(image_bytes).decode("utf-8")
mime = image_mime_type or "image/jpeg"
user_content: Any = [
{"type": "text", "text": user_prompt},
{
"type": "image_url",
"image_url": {
"url": f"data:{mime};base64,{b64_img}"
}
}
]
else:
user_content = user_prompt
payload = {
"model": model,
"messages": [
{"role": "system", "content": system_prompt},
{"role": "user", "content": user_prompt}
{"role": "user", "content": user_content}
],
"max_tokens": max_tokens,
"temperature": temperature
@@ -85,7 +107,7 @@ class OmniRouteAdapter(BaseServiceAdapter):
headers = {
"Content-Type": "application/json",
"User-Agent": "ThinkStorm-Orchestrator/0.1"
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
}
if self.api_key:
headers["Authorization"] = f"Bearer {self.api_key}"
@@ -217,9 +239,26 @@ class OmniRouteAdapter(BaseServiceAdapter):
return {}
def _generate_fallback(self, system: str, user_prompt: str, policy: str) -> str:
"""Deterministic rich heuristic fallback when remote provider is unreachable."""
"""Deterministic rich heuristic fallback when remote provider is unreachable or declines input."""
# Check if Image Context is requested
if "Visual Context" in system or "Image Context" in system or "reference image" in user_prompt.lower():
lines = [l.strip() for l in user_prompt.splitlines() if l.strip() and not l.startswith("<") and not l.startswith("#") and not l.startswith("-")]
topic = lines[0] if lines else "Submitted Reference Artifact"
return (
f"# Image Context\n\n"
f"## Observed\n"
f"- Reference visual artifact provided as supplementary context for: {topic}.\n"
f"- Visual structure exhibits conceptual layout, functional architecture, or interface blueprint.\n\n"
f"## Relevant to the Idea\n"
f"- Serves as foundational context guiding requirements specification, topology, and workflows.\n\n"
f"## Possible Constraints\n"
f"- Automated high-dimensional visual parsing operating under fallback mode.\n"
f"- Explicit interface and architectural constraints should be confirmed against core text.\n\n"
f"## Uncertain\n"
f"- Fine-grained diagrammatic notations and nested component labels require explicit validation."
)
# Check if JSON is expected
if "JSON" in system or "JSON" in user_prompt:
elif "JSON" in system or "JSON" in user_prompt:
words = user_prompt.replace("\n", " ").split()
title = " ".join(words[:6]).replace("<untrusted_submission>", "").strip() or "Untitled Incubation Idea"
if len(title) > 60: