Skjermklipp-input: paste screenshot i chat → CAS → media-node (oppgave 29.1)
Frontend:
- ChatInput: paste-handler detekterer bilder fra clipboard (ClipboardEvent),
laster opp til CAS via uploadMedia med metadata_extra { source: "screenshot" }
- Chat-side: viser bildenoder inline med AI-beskrivelse når tilgjengelig
- api.ts: uploadMedia støtter nå metadata_extra for ekstra node-metadata
Backend (maskinrommet):
- upload_media: nytt metadata_extra multipart-felt som merges inn i
media-nodens metadata (f.eks. source, description)
- describe_image: ny jobbtype — enqueuues automatisk for screenshot-uploads,
kaller synops-ai med --image for AI-beskrivelse av bildet
- Beskrivelsen lagres tilbake i media-nodens metadata.description
synops-ai:
- Nytt --image flag for multimodal LLM-kall (vision) via LiteLLM
- Sender bilde som base64 data-URL i OpenAI-kompatibelt format
- Brukes av describe_image-jobben for bildbeskrivelse
This commit is contained in:
parent
09ed49d7d7
commit
1d8ebf259b
11 changed files with 403 additions and 50 deletions
|
|
@ -239,6 +239,8 @@ export interface UploadMediaRequest {
|
||||||
source_id?: string;
|
source_id?: string;
|
||||||
visibility?: string;
|
visibility?: string;
|
||||||
title?: string;
|
title?: string;
|
||||||
|
/** Ekstra metadata som merges inn i media-nodens metadata (f.eks. { source: "screenshot" }) */
|
||||||
|
metadata_extra?: Record<string, unknown>;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface UploadMediaResponse {
|
export interface UploadMediaResponse {
|
||||||
|
|
@ -258,6 +260,7 @@ export async function uploadMedia(
|
||||||
if (data.source_id) form.append('source_id', data.source_id);
|
if (data.source_id) form.append('source_id', data.source_id);
|
||||||
if (data.visibility) form.append('visibility', data.visibility);
|
if (data.visibility) form.append('visibility', data.visibility);
|
||||||
if (data.title) form.append('title', data.title);
|
if (data.title) form.append('title', data.title);
|
||||||
|
if (data.metadata_extra) form.append('metadata_extra', JSON.stringify(data.metadata_extra));
|
||||||
|
|
||||||
const res = await fetch(`${BASE_URL}/intentions/upload_media`, {
|
const res = await fetch(`${BASE_URL}/intentions/upload_media`, {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,6 @@
|
||||||
<script lang="ts">
|
<script lang="ts">
|
||||||
import VoiceRecorder from './VoiceRecorder.svelte';
|
import VoiceRecorder from './VoiceRecorder.svelte';
|
||||||
|
import { uploadMedia, casUrl } from '$lib/api';
|
||||||
|
|
||||||
interface Props {
|
interface Props {
|
||||||
onsubmit: (content: string) => Promise<void>;
|
onsubmit: (content: string) => Promise<void>;
|
||||||
|
|
@ -14,9 +15,13 @@
|
||||||
let content = $state('');
|
let content = $state('');
|
||||||
let submitting = $state(false);
|
let submitting = $state(false);
|
||||||
let error = $state('');
|
let error = $state('');
|
||||||
|
let uploading = $state(0);
|
||||||
let textareaEl: HTMLTextAreaElement | undefined = $state();
|
let textareaEl: HTMLTextAreaElement | undefined = $state();
|
||||||
|
|
||||||
const isEmpty = $derived(!content.trim());
|
/** Preview for pasted image before upload completes */
|
||||||
|
let imagePreview: { url: string; name: string } | undefined = $state();
|
||||||
|
|
||||||
|
const isEmpty = $derived(!content.trim() && uploading === 0);
|
||||||
|
|
||||||
async function handleSubmit() {
|
async function handleSubmit() {
|
||||||
if (isEmpty || submitting || disabled) return;
|
if (isEmpty || submitting || disabled) return;
|
||||||
|
|
@ -49,45 +54,104 @@
|
||||||
el.style.height = 'auto';
|
el.style.height = 'auto';
|
||||||
el.style.height = Math.min(el.scrollHeight, 160) + 'px';
|
el.style.height = Math.min(el.scrollHeight, 160) + 'px';
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Handle paste event — detect images from clipboard */
|
||||||
|
function handlePaste(e: ClipboardEvent) {
|
||||||
|
const items = e.clipboardData?.items;
|
||||||
|
if (!items) return;
|
||||||
|
|
||||||
|
for (const item of items) {
|
||||||
|
if (item.type.startsWith('image/')) {
|
||||||
|
e.preventDefault();
|
||||||
|
const file = item.getAsFile();
|
||||||
|
if (file) handleImageUpload(file);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Upload a pasted/dropped image as a screenshot media node */
|
||||||
|
async function handleImageUpload(file: File) {
|
||||||
|
if (!accessToken) {
|
||||||
|
error = 'Ikke innlogget — kan ikke laste opp bilder';
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
uploading++;
|
||||||
|
|
||||||
|
// Show preview
|
||||||
|
const previewUrl = URL.createObjectURL(file);
|
||||||
|
imagePreview = { url: previewUrl, name: file.name || 'Skjermklipp' };
|
||||||
|
|
||||||
|
try {
|
||||||
|
await uploadMedia(accessToken, {
|
||||||
|
file,
|
||||||
|
source_id: contextId,
|
||||||
|
title: file.name || 'Skjermklipp',
|
||||||
|
visibility: 'hidden',
|
||||||
|
metadata_extra: { source: 'screenshot' },
|
||||||
|
});
|
||||||
|
} catch (e) {
|
||||||
|
error = e instanceof Error ? e.message : 'Feil ved bildeopplasting';
|
||||||
|
} finally {
|
||||||
|
uploading--;
|
||||||
|
if (imagePreview) {
|
||||||
|
URL.revokeObjectURL(imagePreview.url);
|
||||||
|
imagePreview = undefined;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
<div class="flex items-end gap-2">
|
<div class="flex flex-col gap-2">
|
||||||
<div class="relative flex-1">
|
{#if imagePreview}
|
||||||
<textarea
|
<div class="flex items-center gap-2 rounded-lg border border-blue-200 bg-blue-50 px-3 py-2">
|
||||||
bind:this={textareaEl}
|
<img src={imagePreview.url} alt="Forhåndsvisning" class="h-12 w-12 rounded object-cover" />
|
||||||
bind:value={content}
|
<span class="text-xs text-blue-700">Laster opp {imagePreview.name}…</span>
|
||||||
onkeydown={handleKeydown}
|
</div>
|
||||||
oninput={autoResize}
|
{/if}
|
||||||
placeholder="Skriv en melding…"
|
{#if uploading > 0 && !imagePreview}
|
||||||
|
<p class="text-xs text-blue-600">Laster opp bilde…</p>
|
||||||
|
{/if}
|
||||||
|
<div class="flex items-end gap-2">
|
||||||
|
<div class="relative flex-1">
|
||||||
|
<textarea
|
||||||
|
bind:this={textareaEl}
|
||||||
|
bind:value={content}
|
||||||
|
onkeydown={handleKeydown}
|
||||||
|
oninput={autoResize}
|
||||||
|
onpaste={handlePaste}
|
||||||
|
placeholder="Skriv en melding… (lim inn bilde med Ctrl+V)"
|
||||||
|
disabled={disabled || submitting}
|
||||||
|
rows={1}
|
||||||
|
class="w-full resize-none rounded-xl border border-gray-200 bg-gray-50 px-4 py-2.5 text-sm text-gray-900 placeholder:text-gray-400 focus:border-blue-300 focus:bg-white focus:outline-none focus:ring-1 focus:ring-blue-300"
|
||||||
|
></textarea>
|
||||||
|
{#if error}
|
||||||
|
<p class="absolute -top-6 left-0 text-xs text-red-600">{error}</p>
|
||||||
|
{/if}
|
||||||
|
</div>
|
||||||
|
<VoiceRecorder
|
||||||
|
{accessToken}
|
||||||
|
sourceId={contextId}
|
||||||
disabled={disabled || submitting}
|
disabled={disabled || submitting}
|
||||||
rows={1}
|
onerror={(msg) => { error = msg; }}
|
||||||
class="w-full resize-none rounded-xl border border-gray-200 bg-gray-50 px-4 py-2.5 text-sm text-gray-900 placeholder:text-gray-400 focus:border-blue-300 focus:bg-white focus:outline-none focus:ring-1 focus:ring-blue-300"
|
/>
|
||||||
></textarea>
|
<button
|
||||||
{#if error}
|
onclick={handleSubmit}
|
||||||
<p class="absolute -top-6 left-0 text-xs text-red-600">{error}</p>
|
disabled={isEmpty || submitting || disabled}
|
||||||
{/if}
|
class="flex h-10 w-10 shrink-0 items-center justify-center rounded-full bg-blue-600 text-white transition-colors hover:bg-blue-700 disabled:cursor-not-allowed disabled:opacity-40"
|
||||||
|
aria-label="Send melding"
|
||||||
|
>
|
||||||
|
{#if submitting}
|
||||||
|
<svg class="h-4 w-4 animate-spin" fill="none" viewBox="0 0 24 24">
|
||||||
|
<circle class="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" stroke-width="4"></circle>
|
||||||
|
<path class="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4z"></path>
|
||||||
|
</svg>
|
||||||
|
{:else}
|
||||||
|
<svg class="h-4 w-4" fill="none" viewBox="0 0 24 24" stroke="currentColor" stroke-width="2">
|
||||||
|
<path stroke-linecap="round" stroke-linejoin="round" d="M5 12h14M12 5l7 7-7 7" />
|
||||||
|
</svg>
|
||||||
|
{/if}
|
||||||
|
</button>
|
||||||
</div>
|
</div>
|
||||||
<VoiceRecorder
|
|
||||||
{accessToken}
|
|
||||||
sourceId={contextId}
|
|
||||||
disabled={disabled || submitting}
|
|
||||||
onerror={(msg) => { error = msg; }}
|
|
||||||
/>
|
|
||||||
<button
|
|
||||||
onclick={handleSubmit}
|
|
||||||
disabled={isEmpty || submitting || disabled}
|
|
||||||
class="flex h-10 w-10 shrink-0 items-center justify-center rounded-full bg-blue-600 text-white transition-colors hover:bg-blue-700 disabled:cursor-not-allowed disabled:opacity-40"
|
|
||||||
aria-label="Send melding"
|
|
||||||
>
|
|
||||||
{#if submitting}
|
|
||||||
<svg class="h-4 w-4 animate-spin" fill="none" viewBox="0 0 24 24">
|
|
||||||
<circle class="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" stroke-width="4"></circle>
|
|
||||||
<path class="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4z"></path>
|
|
||||||
</svg>
|
|
||||||
{:else}
|
|
||||||
<svg class="h-4 w-4" fill="none" viewBox="0 0 24 24" stroke="currentColor" stroke-width="2">
|
|
||||||
<path stroke-linecap="round" stroke-linejoin="round" d="M5 12h14M12 5l7 7-7 7" />
|
|
||||||
</svg>
|
|
||||||
{/if}
|
|
||||||
</button>
|
|
||||||
</div>
|
</div>
|
||||||
|
|
|
||||||
|
|
@ -154,6 +154,31 @@
|
||||||
} catch { return false; }
|
} catch { return false; }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Check if this node is an image media node */
|
||||||
|
function isImageNode(node: Node): boolean {
|
||||||
|
if (node.nodeKind !== 'media') return false;
|
||||||
|
try {
|
||||||
|
const meta = JSON.parse(node.metadata ?? '{}');
|
||||||
|
return typeof meta.mime === 'string' && meta.mime.startsWith('image/');
|
||||||
|
} catch { return false; }
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Get CAS image URL for a media node */
|
||||||
|
function imageSrc(node: Node): string {
|
||||||
|
try {
|
||||||
|
const meta = JSON.parse(node.metadata ?? '{}');
|
||||||
|
return casUrl(meta.cas_hash);
|
||||||
|
} catch { return ''; }
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Get AI-generated description for an image node */
|
||||||
|
function imageDescription(node: Node): string | undefined {
|
||||||
|
try {
|
||||||
|
const meta = JSON.parse(node.metadata ?? '{}');
|
||||||
|
return meta.description;
|
||||||
|
} catch { return undefined; }
|
||||||
|
}
|
||||||
|
|
||||||
/** Get CAS audio URL for a media node */
|
/** Get CAS audio URL for a media node */
|
||||||
function audioSrc(node: Node): string {
|
function audioSrc(node: Node): string {
|
||||||
try {
|
try {
|
||||||
|
|
@ -234,9 +259,10 @@
|
||||||
{#each messages as msg (msg.id)}
|
{#each messages as msg (msg.id)}
|
||||||
{@const own = isOwnMessage(msg)}
|
{@const own = isOwnMessage(msg)}
|
||||||
{@const audio = isAudioNode(msg)}
|
{@const audio = isAudioNode(msg)}
|
||||||
|
{@const image = isImageNode(msg)}
|
||||||
{@const bot = isAgentMessage(msg)}
|
{@const bot = isAgentMessage(msg)}
|
||||||
<div class="flex {own ? 'justify-end' : 'justify-start'}">
|
<div class="flex {own ? 'justify-end' : 'justify-start'}">
|
||||||
<div class="max-w-[75%] {own ? (audio ? 'bg-blue-50 border border-blue-200 text-gray-900' : 'bg-blue-600 text-white') : bot ? 'bg-amber-50 border border-amber-200 text-gray-900' : 'bg-white border border-gray-200 text-gray-900'} rounded-2xl px-4 py-2 shadow-sm">
|
<div class="max-w-[75%] {own ? (audio || image ? 'bg-blue-50 border border-blue-200 text-gray-900' : 'bg-blue-600 text-white') : bot ? 'bg-amber-50 border border-amber-200 text-gray-900' : 'bg-white border border-gray-200 text-gray-900'} rounded-2xl px-4 py-2 shadow-sm">
|
||||||
{#if !own}
|
{#if !own}
|
||||||
<p class="mb-0.5 text-xs font-medium {bot ? 'text-amber-700' : 'text-blue-600'}">
|
<p class="mb-0.5 text-xs font-medium {bot ? 'text-amber-700' : 'text-blue-600'}">
|
||||||
{#if bot}<span title="AI-agent">🤖 </span>{/if}{senderName(msg)}
|
{#if bot}<span title="AI-agent">🤖 </span>{/if}{senderName(msg)}
|
||||||
|
|
@ -258,12 +284,24 @@
|
||||||
accessToken={hasSegments(msg) ? accessToken : undefined}
|
accessToken={hasSegments(msg) ? accessToken : undefined}
|
||||||
compact
|
compact
|
||||||
/>
|
/>
|
||||||
|
{:else if image}
|
||||||
|
<div class="my-1">
|
||||||
|
<img
|
||||||
|
src={imageSrc(msg)}
|
||||||
|
alt={imageDescription(msg) || msg.title || 'Skjermklipp'}
|
||||||
|
class="max-w-full rounded-lg"
|
||||||
|
loading="lazy"
|
||||||
|
/>
|
||||||
|
{#if imageDescription(msg)}
|
||||||
|
<p class="mt-1 text-xs text-gray-500 italic">{imageDescription(msg)}</p>
|
||||||
|
{/if}
|
||||||
|
</div>
|
||||||
{:else}
|
{:else}
|
||||||
<p class="text-sm whitespace-pre-wrap break-words">
|
<p class="text-sm whitespace-pre-wrap break-words">
|
||||||
{msg.content || ''}
|
{msg.content || ''}
|
||||||
</p>
|
</p>
|
||||||
{/if}
|
{/if}
|
||||||
<p class="mt-1 text-right text-[10px] {own && !audio ? 'text-blue-200' : 'text-gray-400'}">
|
<p class="mt-1 text-right text-[10px] {own && !audio && !image ? 'text-blue-200' : 'text-gray-400'}">
|
||||||
{formatTime(msg)}
|
{formatTime(msg)}
|
||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
|
|
|
||||||
108
maskinrommet/src/describe_image.rs
Normal file
108
maskinrommet/src/describe_image.rs
Normal file
|
|
@ -0,0 +1,108 @@
|
||||||
|
// describe_image — AI-beskrivelse av bilder via synops-ai CLI.
|
||||||
|
//
|
||||||
|
// Jobbtype: "describe_image"
|
||||||
|
// Payload: { "media_node_id", "cas_hash", "mime" }
|
||||||
|
//
|
||||||
|
// Flyten:
|
||||||
|
// 1. Sender bildefilen (CAS-sti) til synops-ai prompt --image
|
||||||
|
// 2. synops-ai sender multimodal melding til LLM via LiteLLM
|
||||||
|
// 3. Oppdaterer media-nodens metadata med description-feltet
|
||||||
|
//
|
||||||
|
// Ref: docs/retninger/unix_filosofi.md, oppgave 29.1
|
||||||
|
|
||||||
|
use sqlx::PgPool;
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
use crate::cas::CasStore;
|
||||||
|
use crate::jobs::JobRow;
|
||||||
|
|
||||||
|
/// Håndterer describe_image-jobb.
|
||||||
|
///
|
||||||
|
/// Sender bildet til synops-ai for beskrivelse via vision-modell,
|
||||||
|
/// og lagrer resultatet i media-nodens metadata.
|
||||||
|
pub async fn handle_describe_image(
|
||||||
|
job: &JobRow,
|
||||||
|
db: &PgPool,
|
||||||
|
cas: &CasStore,
|
||||||
|
) -> Result<serde_json::Value, String> {
|
||||||
|
let media_node_id: Uuid = job.payload["media_node_id"]
|
||||||
|
.as_str()
|
||||||
|
.and_then(|s| s.parse().ok())
|
||||||
|
.ok_or("Mangler gyldig 'media_node_id' i payload")?;
|
||||||
|
|
||||||
|
let cas_hash = job.payload["cas_hash"]
|
||||||
|
.as_str()
|
||||||
|
.ok_or("Mangler 'cas_hash' i payload")?;
|
||||||
|
|
||||||
|
let mime = job.payload["mime"]
|
||||||
|
.as_str()
|
||||||
|
.unwrap_or("image/png");
|
||||||
|
|
||||||
|
// Verifiser at bildet finnes i CAS
|
||||||
|
let cas_path = cas.path_for(cas_hash);
|
||||||
|
if !cas_path.exists() {
|
||||||
|
return Err(format!("Bilde ikke funnet i CAS: {cas_hash}"));
|
||||||
|
}
|
||||||
|
|
||||||
|
let bin = std::env::var("SYNOPS_AI_BIN")
|
||||||
|
.unwrap_or_else(|_| "synops-ai".to_string());
|
||||||
|
|
||||||
|
tracing::info!(
|
||||||
|
media_node_id = %media_node_id,
|
||||||
|
cas_hash,
|
||||||
|
"Sender bilde til AI for beskrivelse"
|
||||||
|
);
|
||||||
|
|
||||||
|
let output = tokio::process::Command::new(&bin)
|
||||||
|
.arg("prompt")
|
||||||
|
.arg("--prompt")
|
||||||
|
.arg("Beskriv dette bildet kort og konsist på norsk. Fokuser på innholdet, ikke formatet. Maks 2-3 setninger.")
|
||||||
|
.arg("--image")
|
||||||
|
.arg(cas_path.to_str().unwrap_or(""))
|
||||||
|
.arg("--image-mime")
|
||||||
|
.arg(mime)
|
||||||
|
.arg("--job-type")
|
||||||
|
.arg("describe_image")
|
||||||
|
.arg("--temperature")
|
||||||
|
.arg("0.3")
|
||||||
|
.output()
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("Kunne ikke starte {bin}: {e}"))?;
|
||||||
|
|
||||||
|
if !output.status.success() {
|
||||||
|
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||||
|
return Err(format!("synops-ai feilet: {stderr}"));
|
||||||
|
}
|
||||||
|
|
||||||
|
let description = String::from_utf8_lossy(&output.stdout).trim().to_string();
|
||||||
|
|
||||||
|
if description.is_empty() {
|
||||||
|
return Err("synops-ai returnerte tom beskrivelse".into());
|
||||||
|
}
|
||||||
|
|
||||||
|
tracing::info!(
|
||||||
|
media_node_id = %media_node_id,
|
||||||
|
description_len = description.len(),
|
||||||
|
"Bildebeskrivelse mottatt fra AI"
|
||||||
|
);
|
||||||
|
|
||||||
|
// Oppdater media-nodens metadata med description
|
||||||
|
sqlx::query(
|
||||||
|
r#"UPDATE nodes
|
||||||
|
SET metadata = metadata || jsonb_build_object('description', $2::text),
|
||||||
|
updated_at = now()
|
||||||
|
WHERE id = $1"#,
|
||||||
|
)
|
||||||
|
.bind(media_node_id)
|
||||||
|
.bind(&description)
|
||||||
|
.execute(db)
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("Kunne ikke oppdatere media-node med beskrivelse: {e}"))?;
|
||||||
|
|
||||||
|
// PG NOTIFY-trigger på nodes-tabellen sender sanntidsoppdatering automatisk.
|
||||||
|
|
||||||
|
Ok(serde_json::json!({
|
||||||
|
"media_node_id": media_node_id.to_string(),
|
||||||
|
"description": description,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
@ -2154,6 +2154,8 @@ pub struct UploadMediaResponse {
|
||||||
/// - `source_id` (valgfritt): Node-ID å koble media til via `has_media`-edge.
|
/// - `source_id` (valgfritt): Node-ID å koble media til via `has_media`-edge.
|
||||||
/// - `visibility` (valgfritt): Synlighet for media-noden. Default: "hidden".
|
/// - `visibility` (valgfritt): Synlighet for media-noden. Default: "hidden".
|
||||||
/// - `title` (valgfritt): Tittel for media-noden (default: filnavn).
|
/// - `title` (valgfritt): Tittel for media-noden (default: filnavn).
|
||||||
|
/// - `metadata_extra` (valgfritt): JSON-objekt med ekstra metadata som merges
|
||||||
|
/// inn i media-nodens metadata (f.eks. `{"source":"screenshot"}`).
|
||||||
///
|
///
|
||||||
/// Ref: docs/primitiver/nodes.md (media), docs/retninger/universell_input.md
|
/// Ref: docs/primitiver/nodes.md (media), docs/retninger/universell_input.md
|
||||||
pub async fn upload_media(
|
pub async fn upload_media(
|
||||||
|
|
@ -2167,6 +2169,7 @@ pub async fn upload_media(
|
||||||
let mut source_id: Option<Uuid> = None;
|
let mut source_id: Option<Uuid> = None;
|
||||||
let mut visibility = "hidden".to_string();
|
let mut visibility = "hidden".to_string();
|
||||||
let mut title: Option<String> = None;
|
let mut title: Option<String> = None;
|
||||||
|
let mut metadata_extra: Option<serde_json::Value> = None;
|
||||||
|
|
||||||
// -- Parse multipart-felter --
|
// -- Parse multipart-felter --
|
||||||
while let Some(field) = multipart.next_field().await.map_err(|e| {
|
while let Some(field) = multipart.next_field().await.map_err(|e| {
|
||||||
|
|
@ -2219,6 +2222,18 @@ pub async fn upload_media(
|
||||||
})?;
|
})?;
|
||||||
title = Some(text);
|
title = Some(text);
|
||||||
}
|
}
|
||||||
|
"metadata_extra" => {
|
||||||
|
let text = field.text().await.map_err(|e| {
|
||||||
|
bad_request(&format!("Kunne ikke lese metadata_extra: {e}"))
|
||||||
|
})?;
|
||||||
|
let parsed: serde_json::Value = serde_json::from_str(&text).map_err(|e| {
|
||||||
|
bad_request(&format!("Ugyldig JSON i metadata_extra: {e}"))
|
||||||
|
})?;
|
||||||
|
if !parsed.is_object() {
|
||||||
|
return Err(bad_request("metadata_extra må være et JSON-objekt"));
|
||||||
|
}
|
||||||
|
metadata_extra = Some(parsed);
|
||||||
|
}
|
||||||
_ => {
|
_ => {
|
||||||
// Ignorer ukjente felter
|
// Ignorer ukjente felter
|
||||||
}
|
}
|
||||||
|
|
@ -2249,12 +2264,21 @@ pub async fn upload_media(
|
||||||
let mime = content_type.unwrap_or_else(|| "application/octet-stream".to_string());
|
let mime = content_type.unwrap_or_else(|| "application/octet-stream".to_string());
|
||||||
let node_title = title.unwrap_or_else(|| file_name.unwrap_or_default());
|
let node_title = title.unwrap_or_else(|| file_name.unwrap_or_default());
|
||||||
|
|
||||||
let metadata = serde_json::json!({
|
let mut metadata = serde_json::json!({
|
||||||
"cas_hash": cas_result.hash,
|
"cas_hash": cas_result.hash,
|
||||||
"mime": mime,
|
"mime": mime,
|
||||||
"size_bytes": cas_result.size,
|
"size_bytes": cas_result.size,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// Merge ekstra metadata fra klienten (f.eks. source: "screenshot")
|
||||||
|
if let Some(extra) = &metadata_extra {
|
||||||
|
if let Some(obj) = extra.as_object() {
|
||||||
|
for (k, v) in obj {
|
||||||
|
metadata[k] = v.clone();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Skriv media-node til PG
|
// Skriv media-node til PG
|
||||||
sqlx::query(
|
sqlx::query(
|
||||||
r#"INSERT INTO nodes (id, node_kind, title, content, visibility, metadata, created_by)
|
r#"INSERT INTO nodes (id, node_kind, title, content, visibility, metadata, created_by)
|
||||||
|
|
@ -2339,6 +2363,45 @@ pub async fn upload_media(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// -- Enqueue AI-beskrivelse for skjermklipp --
|
||||||
|
let is_screenshot = metadata_extra
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|v| v.get("source"))
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
== Some("screenshot");
|
||||||
|
let is_image = mime.starts_with("image/");
|
||||||
|
|
||||||
|
if is_screenshot && is_image {
|
||||||
|
let payload = serde_json::json!({
|
||||||
|
"media_node_id": media_node_id,
|
||||||
|
"cas_hash": cas_result.hash,
|
||||||
|
"mime": mime,
|
||||||
|
});
|
||||||
|
|
||||||
|
let collection_id = if let Some(src_id) = source_id {
|
||||||
|
find_collection_for_node(&state.db, src_id).await.ok().flatten()
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
|
||||||
|
match crate::jobs::enqueue(&state.db, "describe_image", payload, collection_id, 5).await {
|
||||||
|
Ok(job_id) => {
|
||||||
|
tracing::info!(
|
||||||
|
job_id = %job_id,
|
||||||
|
media_node_id = %media_node_id,
|
||||||
|
"describe_image-jobb opprettet for skjermklipp"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
tracing::error!(
|
||||||
|
media_node_id = %media_node_id,
|
||||||
|
error = %e,
|
||||||
|
"Kunne ikke opprette describe_image-jobb"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// -- Enqueue transkripsjons-jobb for lydfiler --
|
// -- Enqueue transkripsjons-jobb for lydfiler --
|
||||||
if is_audio_mime(&mime) {
|
if is_audio_mime(&mime) {
|
||||||
let payload = serde_json::json!({
|
let payload = serde_json::json!({
|
||||||
|
|
|
||||||
|
|
@ -223,6 +223,9 @@ async fn dispatch(
|
||||||
"clip_url" => {
|
"clip_url" => {
|
||||||
clip::handle_clip_url(job, db).await
|
clip::handle_clip_url(job, db).await
|
||||||
}
|
}
|
||||||
|
"describe_image" => {
|
||||||
|
crate::describe_image::handle_describe_image(job, db, cas).await
|
||||||
|
}
|
||||||
// Orchestration: trigger-evaluering har lagt jobben i kø.
|
// Orchestration: trigger-evaluering har lagt jobben i kø.
|
||||||
// Kompilatoren parser scriptet og validerer det.
|
// Kompilatoren parser scriptet og validerer det.
|
||||||
// Utførelse av kompilert script kommer i oppgave 24.5.
|
// Utførelse av kompilert script kommer i oppgave 24.5.
|
||||||
|
|
|
||||||
|
|
@ -10,6 +10,7 @@ pub mod cas;
|
||||||
pub mod cli_dispatch;
|
pub mod cli_dispatch;
|
||||||
pub mod clip;
|
pub mod clip;
|
||||||
mod custom_domain;
|
mod custom_domain;
|
||||||
|
pub mod describe_image;
|
||||||
mod intentions;
|
mod intentions;
|
||||||
pub mod jobs;
|
pub mod jobs;
|
||||||
pub mod livekit;
|
pub mod livekit;
|
||||||
|
|
|
||||||
3
tasks.md
3
tasks.md
|
|
@ -390,8 +390,7 @@ alle relevante modaliteter. Alt ender som noder. Modaliteten er transport,
|
||||||
noden er det som lever videre.
|
noden er det som lever videre.
|
||||||
|
|
||||||
### Skjermklipp
|
### Skjermklipp
|
||||||
- [~] 29.1 Skjermklipp-input: paste screenshot fra clipboard i chat/editor → upload til CAS → media-node. Frontend detekterer bilde-paste (ClipboardEvent). Valgfri AI-beskrivelse via synops-ai ("beskriv dette bildet"). Metadata: `{ "source": "screenshot", "description": "..." }`.
|
- [x] 29.1 Skjermklipp-input: paste screenshot fra clipboard i chat/editor → upload til CAS → media-node. Frontend detekterer bilde-paste (ClipboardEvent). Valgfri AI-beskrivelse via synops-ai ("beskriv dette bildet"). Metadata: `{ "source": "screenshot", "description": "..." }`.
|
||||||
> Påbegynt: 2026-03-18T20:55
|
|
||||||
|
|
||||||
### RSS/Feed-abonnement
|
### RSS/Feed-abonnement
|
||||||
- [ ] 29.2 `synops-feed` CLI: abonner på RSS/Atom-feed. Input: `--url <feed-url> --collection-id <uuid> [--interval 30m]`. Poller feed, oppretter `content`-node for nye entries med `metadata.source_url` og `tagged`-edge "feed". AI-oppsummering valgfritt. Paywall-deteksjon gjenbrukt fra synops-clip.
|
- [ ] 29.2 `synops-feed` CLI: abonner på RSS/Atom-feed. Input: `--url <feed-url> --collection-id <uuid> [--interval 30m]`. Poller feed, oppretter `content`-node for nye entries med `metadata.source_url` og `tagged`-edge "feed". AI-oppsummering valgfritt. Paywall-deteksjon gjenbrukt fra synops-clip.
|
||||||
|
|
|
||||||
1
tools/synops-ai/Cargo.lock
generated
1
tools/synops-ai/Cargo.lock
generated
|
|
@ -1909,6 +1909,7 @@ dependencies = [
|
||||||
name = "synops-ai"
|
name = "synops-ai"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
|
"base64",
|
||||||
"chrono",
|
"chrono",
|
||||||
"clap",
|
"clap",
|
||||||
"reqwest",
|
"reqwest",
|
||||||
|
|
|
||||||
|
|
@ -18,4 +18,5 @@ chrono = { version = "0.4", features = ["serde"] }
|
||||||
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "json"] }
|
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "json"] }
|
||||||
tracing = "0.1"
|
tracing = "0.1"
|
||||||
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
||||||
|
base64 = "0.22"
|
||||||
synops-common = { path = "../synops-common" }
|
synops-common = { path = "../synops-common" }
|
||||||
|
|
|
||||||
|
|
@ -44,6 +44,15 @@ struct PromptArgs {
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
system: Option<String>,
|
system: Option<String>,
|
||||||
|
|
||||||
|
/// Bilde-fil å inkludere som visuell kontekst (multimodal/vision).
|
||||||
|
/// Bildet sendes som base64 data-URL til LLM.
|
||||||
|
#[arg(long)]
|
||||||
|
image: Option<String>,
|
||||||
|
|
||||||
|
/// MIME-type for bildet (default: image/png)
|
||||||
|
#[arg(long, default_value = "image/png")]
|
||||||
|
image_mime: String,
|
||||||
|
|
||||||
/// Job-type for modelloppslag og logging (default: "simple_prompt")
|
/// Job-type for modelloppslag og logging (default: "simple_prompt")
|
||||||
#[arg(long, default_value = "simple_prompt")]
|
#[arg(long, default_value = "simple_prompt")]
|
||||||
job_type: String,
|
job_type: String,
|
||||||
|
|
@ -104,7 +113,40 @@ struct ChatRequest {
|
||||||
#[derive(Serialize, Clone)]
|
#[derive(Serialize, Clone)]
|
||||||
struct ChatMessage {
|
struct ChatMessage {
|
||||||
role: String,
|
role: String,
|
||||||
content: String,
|
content: MessageContent,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Meldingsinnhold: enten en enkel streng eller multimodal (tekst + bilde).
|
||||||
|
/// OpenAI-kompatibelt format: string | array<{type, text?, image_url?}>
|
||||||
|
#[derive(Clone)]
|
||||||
|
enum MessageContent {
|
||||||
|
Text(String),
|
||||||
|
Multimodal(Vec<ContentPart>),
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize, Clone)]
|
||||||
|
#[serde(tag = "type")]
|
||||||
|
enum ContentPart {
|
||||||
|
#[serde(rename = "text")]
|
||||||
|
Text { text: String },
|
||||||
|
#[serde(rename = "image_url")]
|
||||||
|
ImageUrl { image_url: ImageUrlDetail },
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize, Clone)]
|
||||||
|
struct ImageUrlDetail {
|
||||||
|
url: String,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
detail: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Serialize for MessageContent {
|
||||||
|
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
|
||||||
|
match self {
|
||||||
|
MessageContent::Text(s) => serializer.serialize_str(s),
|
||||||
|
MessageContent::Multimodal(parts) => parts.serialize(serializer),
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Deserialize)]
|
#[derive(Deserialize)]
|
||||||
|
|
@ -126,11 +168,11 @@ struct UsageInfo {
|
||||||
|
|
||||||
#[derive(Deserialize)]
|
#[derive(Deserialize)]
|
||||||
struct Choice {
|
struct Choice {
|
||||||
message: MessageContent,
|
message: ResponseMessage,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Deserialize)]
|
#[derive(Deserialize)]
|
||||||
struct MessageContent {
|
struct ResponseMessage {
|
||||||
content: Option<String>,
|
content: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -191,12 +233,42 @@ async fn run_prompt(args: PromptArgs) -> Result<(), String> {
|
||||||
if let Some(ref sys) = args.system {
|
if let Some(ref sys) = args.system {
|
||||||
messages.push(ChatMessage {
|
messages.push(ChatMessage {
|
||||||
role: "system".to_string(),
|
role: "system".to_string(),
|
||||||
content: sys.clone(),
|
content: MessageContent::Text(sys.clone()),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Multimodal melding hvis --image er oppgitt
|
||||||
|
let user_content = if let Some(ref image_path) = args.image {
|
||||||
|
let image_data = std::fs::read(image_path)
|
||||||
|
.map_err(|e| format!("Kunne ikke lese bildefil '{}': {e}", image_path))?;
|
||||||
|
|
||||||
|
use base64::Engine;
|
||||||
|
let b64 = base64::engine::general_purpose::STANDARD.encode(&image_data);
|
||||||
|
let data_url = format!("data:{};base64,{}", args.image_mime, b64);
|
||||||
|
|
||||||
|
tracing::info!(
|
||||||
|
image_path,
|
||||||
|
image_size = image_data.len(),
|
||||||
|
mime = %args.image_mime,
|
||||||
|
"Inkluderer bilde i multimodal melding"
|
||||||
|
);
|
||||||
|
|
||||||
|
MessageContent::Multimodal(vec![
|
||||||
|
ContentPart::Text { text: args.prompt.clone() },
|
||||||
|
ContentPart::ImageUrl {
|
||||||
|
image_url: ImageUrlDetail {
|
||||||
|
url: data_url,
|
||||||
|
detail: Some("auto".to_string()),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
])
|
||||||
|
} else {
|
||||||
|
MessageContent::Text(args.prompt.clone())
|
||||||
|
};
|
||||||
|
|
||||||
messages.push(ChatMessage {
|
messages.push(ChatMessage {
|
||||||
role: "user".to_string(),
|
role: "user".to_string(),
|
||||||
content: args.prompt.clone(),
|
content: user_content,
|
||||||
});
|
});
|
||||||
|
|
||||||
let (content, usage, actual_model) =
|
let (content, usage, actual_model) =
|
||||||
|
|
@ -711,11 +783,11 @@ async fn run_script(args: ScriptArgs) -> Result<(), String> {
|
||||||
let messages = vec![
|
let messages = vec![
|
||||||
ChatMessage {
|
ChatMessage {
|
||||||
role: "system".to_string(),
|
role: "system".to_string(),
|
||||||
content: system_prompt,
|
content: MessageContent::Text(system_prompt),
|
||||||
},
|
},
|
||||||
ChatMessage {
|
ChatMessage {
|
||||||
role: "user".to_string(),
|
role: "user".to_string(),
|
||||||
content: user_content,
|
content: MessageContent::Text(user_content),
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue