vegard
1d8ebf259b
Frontend:
- ChatInput: paste-handler detekterer bilder fra clipboard (ClipboardEvent),
laster opp til CAS via uploadMedia med metadata_extra { source: "screenshot" }
- Chat-side: viser bildenoder inline med AI-beskrivelse når tilgjengelig
- api.ts: uploadMedia støtter nå metadata_extra for ekstra node-metadata
Backend (maskinrommet):
- upload_media: nytt metadata_extra multipart-felt som merges inn i
media-nodens metadata (f.eks. source, description)
- describe_image: ny jobbtype — enqueuues automatisk for screenshot-uploads,
kaller synops-ai med --image for AI-beskrivelse av bildet
- Beskrivelsen lagres tilbake i media-nodens metadata.description
synops-ai:
- Nytt --image flag for multimodal LLM-kall (vision) via LiteLLM
- Sender bilde som base64 data-URL i OpenAI-kompatibelt format
- Brukes av describe_image-jobben for bildbeskrivelse
323 lines
11 KiB
Svelte
323 lines
11 KiB
Svelte
<script lang="ts">
|
|
import { page } from '$app/stores';
|
|
import { connectionState, nodeStore, edgeStore, nodeVisibility } from '$lib/realtime';
|
|
import type { Node } from '$lib/realtime';
|
|
import { createNode, casUrl } from '$lib/api';
|
|
import ChatInput from '$lib/components/ChatInput.svelte';
|
|
import AudioPlayer from '$lib/components/AudioPlayer.svelte';
|
|
import { tick } from 'svelte';
|
|
|
|
const session = $derived($page.data.session as Record<string, unknown> | undefined);
|
|
const nodeId = $derived(session?.nodeId as string | undefined);
|
|
const accessToken = $derived(session?.accessToken as string | undefined);
|
|
const connected = $derived(connectionState.current === 'connected');
|
|
const communicationId = $derived($page.params.id ?? '');
|
|
|
|
/** The communication node itself */
|
|
const communicationNode = $derived(connected ? nodeStore.get(communicationId) : undefined);
|
|
|
|
/**
|
|
* Messages in this chat: all nodes with a belongs_to edge pointing
|
|
* to this communication node, sorted by created_at ascending.
|
|
*
|
|
* Edge direction: message (source) --belongs_to--> communication (target)
|
|
* So we look at edges targeting the communication node with type belongs_to.
|
|
*/
|
|
const messages = $derived.by(() => {
|
|
if (!connected || !communicationId) return [];
|
|
|
|
const seen = new Set<string>();
|
|
const nodes: Node[] = [];
|
|
|
|
// Text messages: belongs_to edges pointing to this communication node
|
|
// Edge direction: message (source) --belongs_to--> communication (target)
|
|
for (const edge of edgeStore.byTarget(communicationId)) {
|
|
if (edge.edgeType !== 'belongs_to') continue;
|
|
const node = nodeStore.get(edge.sourceId);
|
|
if (node && nodeVisibility(node, nodeId) !== 'hidden' && !seen.has(node.id)) {
|
|
seen.add(node.id);
|
|
nodes.push(node);
|
|
}
|
|
}
|
|
|
|
// Media nodes: has_media edges from this communication node
|
|
// Edge direction: communication (source) --has_media--> media (target)
|
|
for (const edge of edgeStore.bySource(communicationId)) {
|
|
if (edge.edgeType !== 'has_media') continue;
|
|
const node = nodeStore.get(edge.targetId);
|
|
if (node && nodeVisibility(node, nodeId) !== 'hidden' && !seen.has(node.id)) {
|
|
seen.add(node.id);
|
|
nodes.push(node);
|
|
}
|
|
}
|
|
|
|
// Sort by created_at ascending (oldest first, like a chat)
|
|
nodes.sort((a, b) => {
|
|
const ta = a.createdAt ?? 0;
|
|
const tb = b.createdAt ?? 0;
|
|
return ta > tb ? 1 : ta < tb ? -1 : 0;
|
|
});
|
|
|
|
return nodes;
|
|
});
|
|
|
|
/** Participants: nodes with owner/member_of edges to this communication node */
|
|
const participants = $derived.by(() => {
|
|
if (!connected || !communicationId) return [];
|
|
|
|
const edges = edgeStore.byTarget(communicationId)
|
|
.filter(e => e.edgeType === 'owner' || e.edgeType === 'member_of');
|
|
|
|
const nodes: Node[] = [];
|
|
for (const edge of edges) {
|
|
const node = nodeStore.get(edge.sourceId);
|
|
if (node) nodes.push(node);
|
|
}
|
|
return nodes;
|
|
});
|
|
|
|
/** For 1:1 chats, show the other person's name instead of the generic title */
|
|
const chatTitle = $derived.by(() => {
|
|
if (participants.length === 2 && nodeId) {
|
|
const other = participants.find(p => p.id !== nodeId);
|
|
if (other?.title) return other.title;
|
|
}
|
|
return communicationNode?.title || 'Samtale';
|
|
});
|
|
|
|
// Auto-scroll to bottom when new messages arrive
|
|
let messagesContainer: HTMLDivElement | undefined = $state();
|
|
let prevMessageCount = $state(0);
|
|
|
|
$effect(() => {
|
|
const count = messages.length;
|
|
if (count > prevMessageCount && messagesContainer) {
|
|
tick().then(() => {
|
|
messagesContainer?.scrollTo({ top: messagesContainer.scrollHeight, behavior: 'smooth' });
|
|
});
|
|
}
|
|
prevMessageCount = count;
|
|
});
|
|
|
|
/** Send a message in this communication node */
|
|
async function handleSendMessage(content: string) {
|
|
if (!accessToken) throw new Error('Ikke innlogget');
|
|
|
|
await createNode(accessToken, {
|
|
node_kind: 'content',
|
|
content,
|
|
visibility: 'hidden',
|
|
context_id: communicationId,
|
|
});
|
|
}
|
|
|
|
/** Format timestamp for display */
|
|
function formatTime(node: Node): string {
|
|
if (!node.createdAt) return '';
|
|
const ms = Math.floor(node.createdAt / 1000);
|
|
const date = new Date(ms);
|
|
const now = new Date();
|
|
const isToday = date.toDateString() === now.toDateString();
|
|
if (isToday) {
|
|
return date.toLocaleTimeString('nb-NO', { hour: '2-digit', minute: '2-digit' });
|
|
}
|
|
return date.toLocaleDateString('nb-NO', {
|
|
day: 'numeric', month: 'short', hour: '2-digit', minute: '2-digit'
|
|
});
|
|
}
|
|
|
|
/** Get display name for message sender */
|
|
function senderName(node: Node): string {
|
|
if (!node.createdBy) return 'Ukjent';
|
|
const sender = nodeStore.get(node.createdBy);
|
|
return sender?.title || sender?.nodeKind || 'Ukjent';
|
|
}
|
|
|
|
/** Check if message sender is an agent (bot) */
|
|
function isAgentMessage(node: Node): boolean {
|
|
if (!node.createdBy) return false;
|
|
const sender = nodeStore.get(node.createdBy);
|
|
return sender?.nodeKind === 'agent';
|
|
}
|
|
|
|
/** Check if this message is from the current user */
|
|
function isOwnMessage(node: Node): boolean {
|
|
return !!nodeId && node.createdBy === nodeId;
|
|
}
|
|
|
|
/** Check if this node is an audio media node */
|
|
function isAudioNode(node: Node): boolean {
|
|
if (node.nodeKind !== 'media') return false;
|
|
try {
|
|
const meta = JSON.parse(node.metadata ?? '{}');
|
|
return typeof meta.mime === 'string' && meta.mime.startsWith('audio/');
|
|
} catch { return false; }
|
|
}
|
|
|
|
/** Check if this node is an image media node */
|
|
function isImageNode(node: Node): boolean {
|
|
if (node.nodeKind !== 'media') return false;
|
|
try {
|
|
const meta = JSON.parse(node.metadata ?? '{}');
|
|
return typeof meta.mime === 'string' && meta.mime.startsWith('image/');
|
|
} catch { return false; }
|
|
}
|
|
|
|
/** Get CAS image URL for a media node */
|
|
function imageSrc(node: Node): string {
|
|
try {
|
|
const meta = JSON.parse(node.metadata ?? '{}');
|
|
return casUrl(meta.cas_hash);
|
|
} catch { return ''; }
|
|
}
|
|
|
|
/** Get AI-generated description for an image node */
|
|
function imageDescription(node: Node): string | undefined {
|
|
try {
|
|
const meta = JSON.parse(node.metadata ?? '{}');
|
|
return meta.description;
|
|
} catch { return undefined; }
|
|
}
|
|
|
|
/** Get CAS audio URL for a media node */
|
|
function audioSrc(node: Node): string {
|
|
try {
|
|
const meta = JSON.parse(node.metadata ?? '{}');
|
|
return casUrl(meta.cas_hash);
|
|
} catch { return ''; }
|
|
}
|
|
|
|
/** Get transcription duration from metadata */
|
|
function audioDuration(node: Node): number | undefined {
|
|
try {
|
|
const meta = JSON.parse(node.metadata ?? '{}');
|
|
const dms = meta.transcription?.duration_ms;
|
|
return dms ? dms / 1000 : undefined;
|
|
} catch { return undefined; }
|
|
}
|
|
|
|
/** Check if an audio node has transcription segments */
|
|
function hasSegments(node: Node): boolean {
|
|
try {
|
|
const meta = JSON.parse(node.metadata ?? '{}');
|
|
return (meta.transcription?.segment_count ?? 0) > 0;
|
|
} catch { return false; }
|
|
}
|
|
</script>
|
|
|
|
<div class="flex h-screen flex-col bg-gray-50">
|
|
<!-- Header -->
|
|
<header class="shrink-0 border-b border-gray-200 bg-white">
|
|
<div class="mx-auto flex max-w-3xl items-center gap-3 px-4 py-3">
|
|
<a
|
|
href="/"
|
|
class="rounded p-1 text-gray-400 hover:bg-gray-100 hover:text-gray-600"
|
|
aria-label="Tilbake"
|
|
>
|
|
<svg class="h-5 w-5" fill="none" viewBox="0 0 24 24" stroke="currentColor" stroke-width="2">
|
|
<path stroke-linecap="round" stroke-linejoin="round" d="M15 19l-7-7 7-7" />
|
|
</svg>
|
|
</a>
|
|
<div class="min-w-0 flex-1">
|
|
<h1 class="truncate text-lg font-semibold text-gray-900">
|
|
{chatTitle}
|
|
</h1>
|
|
{#if participants.length > 0}
|
|
<p class="truncate text-xs text-gray-500">
|
|
{participants.map(p => p.title || 'Ukjent').join(', ')}
|
|
</p>
|
|
{/if}
|
|
</div>
|
|
{#if connected}
|
|
<span class="text-xs text-green-600">Tilkoblet</span>
|
|
{:else}
|
|
<span class="text-xs text-gray-400">{connectionState.current}</span>
|
|
{/if}
|
|
</div>
|
|
</header>
|
|
|
|
<!-- Messages -->
|
|
<div
|
|
bind:this={messagesContainer}
|
|
class="flex-1 overflow-y-auto"
|
|
>
|
|
<div class="mx-auto max-w-3xl px-4 py-4">
|
|
{#if !connected}
|
|
<p class="text-center text-sm text-gray-400">Venter på tilkobling…</p>
|
|
{:else if !communicationNode}
|
|
<div class="rounded-lg border border-yellow-200 bg-yellow-50 p-4 text-sm text-yellow-800">
|
|
<p class="font-medium">Samtale ikke funnet</p>
|
|
<p class="mt-1">Kommunikasjonsnoden med ID {communicationId} finnes ikke eller er ikke tilgjengelig.</p>
|
|
<a href="/" class="mt-2 inline-block text-blue-600 hover:underline">Tilbake til mottak</a>
|
|
</div>
|
|
{:else if messages.length === 0}
|
|
<p class="text-center text-sm text-gray-400">
|
|
Ingen meldinger ennå. Skriv den første!
|
|
</p>
|
|
{:else}
|
|
<div class="space-y-3">
|
|
{#each messages as msg (msg.id)}
|
|
{@const own = isOwnMessage(msg)}
|
|
{@const audio = isAudioNode(msg)}
|
|
{@const image = isImageNode(msg)}
|
|
{@const bot = isAgentMessage(msg)}
|
|
<div class="flex {own ? 'justify-end' : 'justify-start'}">
|
|
<div class="max-w-[75%] {own ? (audio || image ? 'bg-blue-50 border border-blue-200 text-gray-900' : 'bg-blue-600 text-white') : bot ? 'bg-amber-50 border border-amber-200 text-gray-900' : 'bg-white border border-gray-200 text-gray-900'} rounded-2xl px-4 py-2 shadow-sm">
|
|
{#if !own}
|
|
<p class="mb-0.5 text-xs font-medium {bot ? 'text-amber-700' : 'text-blue-600'}">
|
|
{#if bot}<span title="AI-agent">🤖 </span>{/if}{senderName(msg)}
|
|
</p>
|
|
{/if}
|
|
{#if audio}
|
|
<div class="flex items-center gap-1.5 mb-1">
|
|
<svg class="h-3.5 w-3.5 text-blue-500 shrink-0" fill="none" viewBox="0 0 24 24" stroke="currentColor" stroke-width="2">
|
|
<path stroke-linecap="round" stroke-linejoin="round" d="M12 1a3 3 0 00-3 3v8a3 3 0 006 0V4a3 3 0 00-3-3z" />
|
|
<path stroke-linecap="round" stroke-linejoin="round" d="M19 10v2a7 7 0 01-14 0v-2" />
|
|
</svg>
|
|
<span class="text-[11px] font-medium text-blue-600">Talenotat</span>
|
|
</div>
|
|
<AudioPlayer
|
|
src={audioSrc(msg)}
|
|
duration={audioDuration(msg)}
|
|
transcript={!hasSegments(msg) ? (msg.content || undefined) : undefined}
|
|
nodeId={hasSegments(msg) ? msg.id : undefined}
|
|
accessToken={hasSegments(msg) ? accessToken : undefined}
|
|
compact
|
|
/>
|
|
{:else if image}
|
|
<div class="my-1">
|
|
<img
|
|
src={imageSrc(msg)}
|
|
alt={imageDescription(msg) || msg.title || 'Skjermklipp'}
|
|
class="max-w-full rounded-lg"
|
|
loading="lazy"
|
|
/>
|
|
{#if imageDescription(msg)}
|
|
<p class="mt-1 text-xs text-gray-500 italic">{imageDescription(msg)}</p>
|
|
{/if}
|
|
</div>
|
|
{:else}
|
|
<p class="text-sm whitespace-pre-wrap break-words">
|
|
{msg.content || ''}
|
|
</p>
|
|
{/if}
|
|
<p class="mt-1 text-right text-[10px] {own && !audio && !image ? 'text-blue-200' : 'text-gray-400'}">
|
|
{formatTime(msg)}
|
|
</p>
|
|
</div>
|
|
</div>
|
|
{/each}
|
|
</div>
|
|
{/if}
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Input -->
|
|
{#if connected && accessToken && communicationNode}
|
|
<div class="shrink-0 border-t border-gray-200 bg-white">
|
|
<div class="mx-auto max-w-3xl px-4 py-3">
|
|
<ChatInput onsubmit={handleSendMessage} disabled={!connected} {accessToken} contextId={communicationId} />
|
|
</div>
|
|
</div>
|
|
{/if}
|
|
</div>
|