Implement optional single-image intake, Signal ingestion, and multimodal vision analysis
- Add image intake service with format validation (JPEG, PNG, WebP) and EXIF/GPS stripping - Enforce strict single-image rule across Web and Signal attachment channels - Implement token-optimized vision downscaling and JPEG compression - Add IMAGE_CONTEXT pipeline stage with OmniRoute vision routing and resilient failover - Seed and manage versioned idea-image-interpreter prompt in catalog - Update Web UI with responsive image picker, preview chip, and Visual Context tab - Add comprehensive automated test suite in test_image_intake.py - Update README and Labyricorn devlog
This commit is contained in:
@@ -21,6 +21,10 @@ from ..services.perplexica import PerplexicaAdapter
|
||||
from ..services.opengist import OpenGistAdapter
|
||||
from ..services.gitea import GiteaAdapter
|
||||
from ..services.virustotal import VirusTotalAdapter
|
||||
from ..services.image_handler import (
|
||||
get_image_artifact_path,
|
||||
create_token_optimized_vision_payload
|
||||
)
|
||||
|
||||
omniroute_svc = OmniRouteAdapter()
|
||||
searxng_svc = SearXNGAdapter()
|
||||
@@ -118,6 +122,135 @@ async def process_url_safety(idea_id: str, submission_text: str) -> Tuple[List[I
|
||||
))
|
||||
return url_records, quarantine_required
|
||||
|
||||
# -------------------------------------------------------------
|
||||
# Processor 1.5: Visual Reference & Image Context
|
||||
# -------------------------------------------------------------
|
||||
async def process_image_context(
|
||||
idea_id: str,
|
||||
original_text: str,
|
||||
submission_image: Optional[Dict[str, Any]]
|
||||
) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""
|
||||
Analyzes submitted reference image strictly as supporting context for the idea.
|
||||
Employs token-optimized downscaling and resilient failover.
|
||||
Returns:
|
||||
Tuple[Optional[str], Optional[str]]: (image_context_report, processor_run_id)
|
||||
"""
|
||||
if not submission_image or not submission_image.get("present"):
|
||||
return None, None
|
||||
|
||||
img_path = get_image_artifact_path(idea_id, submission_image)
|
||||
if not img_path or not img_path.exists():
|
||||
return None, None
|
||||
|
||||
start_time = get_utc_now()
|
||||
prompt_info = get_prompt_version("idea-image-interpreter", is_admin=True)
|
||||
if not prompt_info:
|
||||
system_prompt = (
|
||||
"You are ThinkStorm's Visual Context & Reference Image Interpreter. "
|
||||
"Analyze the submitted image strictly as supporting context for the submitted idea.\n\n"
|
||||
"Output format:\n# Image Context\n\n## Observed\n- ...\n\n## Relevant to the Idea\n- ...\n\n## Possible Constraints\n- ...\n\n## Uncertain\n- ..."
|
||||
)
|
||||
user_prompt_template = "Idea Submission:\n<untrusted_submission>\n{{submission_text}}\n</untrusted_submission>\n\nAnalyze the provided reference image and deliver the structured Image Context report in Markdown."
|
||||
prompt_version = 1
|
||||
prompt_hash = ""
|
||||
else:
|
||||
system_prompt = prompt_info["system_prompt"]
|
||||
user_prompt_template = prompt_info["user_prompt_template"]
|
||||
prompt_version = prompt_info["version"]
|
||||
prompt_hash = prompt_info.get("prompt_hash", "")
|
||||
|
||||
user_prompt = (
|
||||
user_prompt_template
|
||||
.replace("{{submission_text}}", original_text)
|
||||
.replace("{{mime_type}}", str(submission_image.get("mime_type", "image/jpeg")))
|
||||
.replace("{{dimensions}}", f"{submission_image.get('width', 0)}x{submission_image.get('height', 0)}")
|
||||
.replace("{{original_filename}}", str(submission_image.get("original_filename", "submission_image")))
|
||||
)
|
||||
|
||||
# Token-efficient downscaling / encoding to prevent unnecessary vision token bloat
|
||||
raw_img_bytes = img_path.read_bytes()
|
||||
opt_bytes, opt_mime = create_token_optimized_vision_payload(
|
||||
raw_img_bytes,
|
||||
submission_image.get("mime_type", "image/jpeg")
|
||||
)
|
||||
|
||||
try:
|
||||
llm_resp = await omniroute_svc.chat_completion(
|
||||
system_prompt=system_prompt,
|
||||
user_prompt=user_prompt,
|
||||
model_policy="vision",
|
||||
max_tokens=1500,
|
||||
image_bytes=opt_bytes,
|
||||
image_mime_type=opt_mime
|
||||
)
|
||||
image_context_report = llm_resp.get("text", "").strip()
|
||||
end_time = get_utc_now()
|
||||
|
||||
run_id = await record_processor_run(
|
||||
idea_id=idea_id,
|
||||
processor_name="IdeaImageInterpreter",
|
||||
stage="IMAGE_CONTEXT",
|
||||
prompt_id="idea-image-interpreter",
|
||||
prompt_version=prompt_version,
|
||||
prompt_hash=prompt_hash,
|
||||
model_policy="vision",
|
||||
resolved_provider=llm_resp.get("resolved_provider", "OmniRoute"),
|
||||
resolved_model=llm_resp.get("resolved_model", "auto/best-vision"),
|
||||
input_tokens=llm_resp.get("input_tokens", 0),
|
||||
output_tokens=llm_resp.get("output_tokens", 0),
|
||||
total_tokens=llm_resp.get("total_tokens", 0),
|
||||
started_at=start_time,
|
||||
completed_at=end_time,
|
||||
duration_ms=llm_resp.get("duration_ms", 0),
|
||||
output_artifact="image-context.md",
|
||||
output_data={"content": image_context_report},
|
||||
status=ProcessorStatus.COMPLETED
|
||||
)
|
||||
|
||||
submission_image["vision_analysis_run_id"] = run_id
|
||||
with get_db() as conn:
|
||||
conn.execute(
|
||||
"UPDATE ideas SET submission_image = ? WHERE id = ?",
|
||||
(json.dumps(submission_image), idea_id)
|
||||
)
|
||||
|
||||
return image_context_report, run_id
|
||||
|
||||
except Exception as e:
|
||||
end_time = get_utc_now()
|
||||
print(f"[IMAGE_CONTEXT] Notice: Vision processor encountered exception: {e}. Executing graceful failover.")
|
||||
run_id = await record_processor_run(
|
||||
idea_id=idea_id,
|
||||
processor_name="IdeaImageInterpreter",
|
||||
stage="IMAGE_CONTEXT",
|
||||
prompt_id="idea-image-interpreter",
|
||||
prompt_version=prompt_version,
|
||||
prompt_hash=prompt_hash,
|
||||
model_policy="vision",
|
||||
resolved_provider="OmniRoute",
|
||||
resolved_model="auto/best-vision",
|
||||
input_tokens=0,
|
||||
output_tokens=0,
|
||||
total_tokens=0,
|
||||
started_at=start_time,
|
||||
completed_at=end_time,
|
||||
duration_ms=0,
|
||||
output_artifact=None,
|
||||
output_data={},
|
||||
status=ProcessorStatus.FAILED,
|
||||
error_message=str(e)
|
||||
)
|
||||
|
||||
submission_image["vision_analysis_run_id"] = run_id
|
||||
with get_db() as conn:
|
||||
conn.execute(
|
||||
"UPDATE ideas SET submission_image = ? WHERE id = ?",
|
||||
(json.dumps(submission_image), idea_id)
|
||||
)
|
||||
|
||||
return None, run_id
|
||||
|
||||
# -------------------------------------------------------------
|
||||
# Processor 2: Normalization & Classification
|
||||
# -------------------------------------------------------------
|
||||
@@ -431,7 +564,7 @@ async def process_feasibility_critique(idea_id: str, title: str, summary: str, r
|
||||
# -------------------------------------------------------------
|
||||
# Complete Intake Pipeline Orchestrator
|
||||
# -------------------------------------------------------------
|
||||
async def execute_intake_pipeline(idea_id: str):
|
||||
async def execute_intake_pipeline(idea_id: str, bypass_duplicate_check: bool = False):
|
||||
"""Orchestrates end-to-end idea intake pipeline from SUBMITTED to AVAILABLE."""
|
||||
with get_db() as conn:
|
||||
idea_row = conn.execute("SELECT * FROM ideas WHERE id = ?", (idea_id,)).fetchone()
|
||||
@@ -448,25 +581,41 @@ async def execute_intake_pipeline(idea_id: str):
|
||||
conn.execute("UPDATE ideas SET lifecycle_state = 'QUARANTINED', processing_state = 'IDLE' WHERE id = ?", (idea_id,))
|
||||
return
|
||||
|
||||
# Step 2: Normalization
|
||||
norm = await process_normalization(idea_id, original_text)
|
||||
# Step 1.5: Image Context (if reference image is present)
|
||||
image_context = None
|
||||
submission_image = None
|
||||
raw_img_json = idea_row["submission_image"] if "submission_image" in idea_row.keys() else None
|
||||
if raw_img_json:
|
||||
try:
|
||||
submission_image = json.loads(raw_img_json) if isinstance(raw_img_json, str) else raw_img_json
|
||||
except Exception:
|
||||
submission_image = None
|
||||
|
||||
if submission_image and submission_image.get("present"):
|
||||
image_context, _ = await process_image_context(idea_id, original_text, submission_image)
|
||||
|
||||
# Step 2: Normalization (incorporating visual context if available)
|
||||
norm_text = f"{original_text}\n\n[Visual Reference Context]:\n{image_context}" if image_context else original_text
|
||||
norm = await process_normalization(idea_id, norm_text)
|
||||
title = norm["title"]
|
||||
summary = norm["summary"]
|
||||
categories = norm["categories"]
|
||||
tags = norm["tags"]
|
||||
|
||||
# Step 3: Duplicate Check
|
||||
dup_info = await process_duplicate_detection(idea_id, title, summary)
|
||||
if dup_info.get("is_duplicate"):
|
||||
with get_db() as conn:
|
||||
conn.execute("UPDATE ideas SET processing_state = 'IDLE' WHERE id = ?", (idea_id,))
|
||||
return
|
||||
if not bypass_duplicate_check:
|
||||
dup_info = await process_duplicate_detection(idea_id, title, summary)
|
||||
if dup_info.get("is_duplicate"):
|
||||
with get_db() as conn:
|
||||
conn.execute("UPDATE ideas SET processing_state = 'IDLE' WHERE id = ?", (idea_id,))
|
||||
return
|
||||
|
||||
# Step 4: Prior Art
|
||||
prior_art = await process_prior_art(idea_id, title, summary, original_text, tags)
|
||||
|
||||
# Step 5: Research Synthesis
|
||||
research = await process_research_synthesis(idea_id, title, original_text, prior_art)
|
||||
# Step 5: Research Synthesis (incorporating visual context)
|
||||
synthesis_context = f"{prior_art}\n\n## Visual Context Findings\n{image_context}" if image_context else prior_art
|
||||
research = await process_research_synthesis(idea_id, title, original_text, synthesis_context)
|
||||
|
||||
# Step 6: Feasibility & Critique
|
||||
feasibility = await process_feasibility_critique(idea_id, title, summary, research)
|
||||
@@ -477,6 +626,8 @@ async def execute_intake_pipeline(idea_id: str):
|
||||
"analysis.md": research,
|
||||
"feasibility.md": feasibility
|
||||
}
|
||||
if image_context:
|
||||
research_docs["image-context.md"] = image_context
|
||||
with get_db() as conn:
|
||||
runs = [dict(r) for r in conn.execute("SELECT * FROM processor_runs WHERE idea_id = ?", (idea_id,)).fetchall()]
|
||||
|
||||
@@ -491,7 +642,8 @@ async def execute_intake_pipeline(idea_id: str):
|
||||
lifecycle_state="AVAILABLE",
|
||||
research_docs=research_docs,
|
||||
outputs={},
|
||||
provenance_runs=runs
|
||||
provenance_runs=runs,
|
||||
submission_image=submission_image
|
||||
)
|
||||
|
||||
# 2. Secondary: Persist local durable files & OpenGist
|
||||
@@ -505,7 +657,8 @@ async def execute_intake_pipeline(idea_id: str):
|
||||
lifecycle_state="AVAILABLE",
|
||||
research_docs=research_docs,
|
||||
outputs={},
|
||||
provenance_runs=runs
|
||||
provenance_runs=runs,
|
||||
submission_image=submission_image
|
||||
)
|
||||
|
||||
# Finalize Idea to AVAILABLE and record repository links
|
||||
@@ -542,8 +695,17 @@ async def execute_intake_pipeline(idea_id: str):
|
||||
# -------------------------------------------------------------
|
||||
# Work Track Execution (Claimed Ideas)
|
||||
# -------------------------------------------------------------
|
||||
def _extract_delimited_section(content: str, section: str) -> str:
|
||||
"""Extract one Markdown artifact from a delimited multi-artifact response."""
|
||||
start_marker = f"<!-- {section}_START -->"
|
||||
end_marker = f"<!-- {section}_END -->"
|
||||
if start_marker in content and end_marker in content:
|
||||
return content.split(start_marker, 1)[1].split(end_marker, 1)[0].strip()
|
||||
return content.strip()
|
||||
|
||||
|
||||
async def execute_work_track_workflow(work_track_id: str, model_override: Optional[str] = None):
|
||||
"""Executes generative workflow for a specific work track (Article or Coding Project)."""
|
||||
"""Executes the generative workflow for an Article, Coding, or YouTube work track."""
|
||||
with get_db() as conn:
|
||||
track = conn.execute("SELECT * FROM work_tracks WHERE id = ?", (work_track_id,)).fetchone()
|
||||
if not track:
|
||||
@@ -615,6 +777,30 @@ async def execute_work_track_workflow(work_track_id: str, model_override: Option
|
||||
generated_outputs["architecture.md"] = f"# System Architecture Blueprint: {track_name}\n\n" + spec_text
|
||||
generated_outputs["requirements.md"] = f"# Core Requirements & Stories: {track_name}\n\n" + spec_text
|
||||
|
||||
elif work_type == "YOUTUBE_VIDEO":
|
||||
prompt_info = get_prompt_version("youtube-video-generator", is_admin=True)
|
||||
user_prompt = (
|
||||
prompt_info["user_prompt_template"]
|
||||
.replace("{{title}}", idea["title"])
|
||||
.replace("{{track_name}}", track_name)
|
||||
.replace("{{summary}}", idea["summary"])
|
||||
.replace("{{research_context}}", combined_research)
|
||||
)
|
||||
llm_resp = await omniroute_svc.chat_completion(
|
||||
system_prompt=prompt_info["system_prompt"],
|
||||
user_prompt=user_prompt,
|
||||
model_policy="reasoning",
|
||||
max_tokens=5000,
|
||||
model_override=effective_model
|
||||
)
|
||||
package_text = llm_resp["text"]
|
||||
generated_outputs["video-outline.md"] = _extract_delimited_section(package_text, "OUTLINE")
|
||||
generated_outputs["video-script.md"] = _extract_delimited_section(package_text, "SCRIPT")
|
||||
generated_outputs["promotion-plan.md"] = _extract_delimited_section(package_text, "PROMOTION")
|
||||
|
||||
else:
|
||||
raise ValueError(f"Unsupported work type: {work_type}")
|
||||
|
||||
# Persist outputs in DB & OpenGist with versioning
|
||||
end_time = get_utc_now()
|
||||
resolved_model = llm_resp.get("resolved_model", effective_model)
|
||||
|
||||
Reference in New Issue
Block a user