// synops-feed — Abonner på RSS/Atom-feed og opprett content-noder. // // Input: --url --collection-id --created-by // [--interval 30m] [--payload-json ] // // Henter feed, parser med feed-rs, oppretter content-node for nye entries // med metadata.source_url, tagged-edge "feed", og belongs_to-edge til samling. // Dedupliserer på entry-URL (sjekker metadata.source_url i eksisterende noder). // Paywall-deteksjon gjenbrukt fra synops-clip. // // Designvalg: Verktøyet kjøres én gang per poll. Maskinrommet orkestrerer // periodisk kjøring via jobbkø. --interval lagres som metadata for // orkestreringsoppsettet, men selve pollinga skjer ikke i denne prosessen. // // Miljøvariabler: // DATABASE_URL — PostgreSQL-tilkobling (påkrevd) // RUST_LOG — Loggnivå (default: synops_feed=info) // // Ref: docs/retninger/unix_filosofi.md use chrono::{DateTime, Utc}; use clap::Parser; use feed_rs::parser; use serde::{Deserialize, Serialize}; use std::process; use tracing::{debug, info, warn}; use uuid::Uuid; // ============================================================================= // CLI // ============================================================================= /// Abonner på RSS/Atom-feed og opprett content-noder. #[derive(Parser)] #[command(name = "synops-feed", about = "Abonner på RSS/Atom-feed")] struct Cli { /// Feed-URL (RSS eller Atom) #[arg(long)] url: Option, /// Samlings-ID å legge nye entries i #[arg(long)] collection_id: Option, /// Bruker-ID som oppretter nodene #[arg(long)] created_by: Option, /// Poll-intervall (f.eks. "30m", "1h") — lagres som metadata, brukes av orkestrator #[arg(long, default_value = "30m")] interval: String, /// Jobb-payload som JSON (for maskinrommet/jobbkø) #[arg(long)] payload_json: Option, } /// Payload fra jobbkø. #[derive(Deserialize)] struct JobPayload { url: String, collection_id: Uuid, created_by: Uuid, #[serde(default = "default_interval")] interval: String, } fn default_interval() -> String { "30m".to_string() } /// Resultat-output som JSON til stdout. #[derive(Serialize)] struct FeedResult { status: String, feed_url: String, feed_title: Option, items_found: usize, nodes_created: usize, items_skipped: usize, errors: Vec, } // ============================================================================= // Main // ============================================================================= #[tokio::main] async fn main() { synops_common::logging::init("synops_feed"); let cli = Cli::parse(); // Løs opp parametre: payload-json overskriver CLI-args let (url, collection_id, created_by, _interval) = if let Some(ref json) = cli.payload_json { match serde_json::from_str::(json) { Ok(p) => (p.url, p.collection_id, p.created_by, p.interval), Err(e) => { eprintln!("Ugyldig --payload-json: {e}"); process::exit(1); } } } else { let url = cli.url.unwrap_or_else(|| { eprintln!("--url er påkrevd"); process::exit(1); }); let collection_id = cli.collection_id.unwrap_or_else(|| { eprintln!("--collection-id er påkrevd"); process::exit(1); }); let created_by = cli.created_by.unwrap_or_else(|| { eprintln!("--created-by er påkrevd"); process::exit(1); }); (url, collection_id, created_by, cli.interval) }; match fetch_and_process(&url, collection_id, created_by).await { Ok(result) => { let json = serde_json::to_string_pretty(&result).unwrap(); println!("{json}"); if result.status == "error" { process::exit(1); } } Err(e) => { eprintln!("Feil: {e}"); process::exit(1); } } } // ============================================================================= // Feed-henting og -prosessering // ============================================================================= async fn fetch_and_process( url: &str, collection_id: Uuid, created_by: Uuid, ) -> Result { // 1. Hent feed-XML info!(url, "Henter feed"); let client = reqwest::Client::builder() .user_agent("synops-feed/0.1") .timeout(std::time::Duration::from_secs(30)) .build() .map_err(|e| format!("HTTP-klient feilet: {e}"))?; let response = client .get(url) .send() .await .map_err(|e| format!("Kunne ikke hente feed: {e}"))?; if !response.status().is_success() { return Err(format!("HTTP {}: {}", response.status(), url)); } let body = response .bytes() .await .map_err(|e| format!("Kunne ikke lese respons: {e}"))?; // 2. Parse feed (RSS 2.0, Atom 1.0, etc.) let feed = parser::parse(&body[..]).map_err(|e| format!("Kunne ikke parse feed: {e}"))?; let feed_title = feed.title.map(|t| t.content); info!( feed_title = feed_title.as_deref().unwrap_or("(uten tittel)"), entries = feed.entries.len(), "Feed parset" ); // 3. Koble til database let db = synops_common::db::connect().await?; // 4. Hent eksisterende source_url-er for denne samlingen for deduplisering let existing_urls: Vec = sqlx::query_scalar( r#" SELECT n.metadata->>'source_url' FROM nodes n INNER JOIN edges e ON e.source_id = n.id AND e.target_id = $1 AND e.edge_type = 'belongs_to' WHERE n.node_kind = 'content' AND n.metadata->>'source_url' IS NOT NULL "#, ) .bind(collection_id) .fetch_all(&db) .await .map_err(|e| format!("PG-feil ved dedupliseringssjekk: {e}"))?; debug!(existing = existing_urls.len(), "Eksisterende feed-entries i samling"); // 5. Prosesser nye entries let mut nodes_created = 0usize; let mut items_skipped = 0usize; let mut errors = Vec::new(); for entry in &feed.entries { // Finn entry-URL let entry_url = entry .links .first() .map(|l| l.href.clone()) .or_else(|| entry.id.clone().into()) .unwrap_or_default(); if entry_url.is_empty() { debug!("Entry uten URL, hopper over"); items_skipped += 1; continue; } // Dedupliser if existing_urls.iter().any(|u| u == &entry_url) { debug!(url = %entry_url, "Allerede importert, hopper over"); items_skipped += 1; continue; } // Opprett node match create_feed_entry_node(&db, entry, &entry_url, collection_id, created_by).await { Ok(node_id) => { info!(node_id = %node_id, url = %entry_url, "Feed-entry opprettet"); nodes_created += 1; } Err(e) => { warn!(url = %entry_url, error = %e, "Kunne ikke opprette entry"); errors.push(format!("{entry_url}: {e}")); } } } info!( nodes_created, items_skipped, errors = errors.len(), "Feed-prosessering ferdig" ); Ok(FeedResult { status: if errors.is_empty() { "completed".to_string() } else { "partial".to_string() }, feed_url: url.to_string(), feed_title, items_found: feed.entries.len(), nodes_created, items_skipped, errors, }) } // ============================================================================= // Node-opprettelse // ============================================================================= async fn create_feed_entry_node( db: &sqlx::PgPool, entry: &feed_rs::model::Entry, entry_url: &str, collection_id: Uuid, created_by: Uuid, ) -> Result { let node_id = Uuid::now_v7(); // Tittel let title = entry .title .as_ref() .map(|t| t.content.clone()) .unwrap_or_else(|| truncate(entry_url, 80).to_string()); // Innhold: bruk summary eller content fra feed let content_text = entry .summary .as_ref() .map(|s| s.content.clone()) .or_else(|| { entry .content .as_ref() .and_then(|c| c.body.as_ref().map(|b| strip_html_tags(b))) }) .unwrap_or_default(); // Forfatter let author = entry.authors.first().map(|a| a.name.clone()); // Publiseringsdato let published: Option> = entry.published.or(entry.updated); // Paywall-sjekk på innholdet let raw_html = entry .content .as_ref() .and_then(|c| c.body.clone()) .unwrap_or_default(); let paywall = detect_paywall(&content_text, &raw_html); if paywall { debug!(url = %entry_url, "Betalingsmur detektert"); } // Metadata let metadata = serde_json::json!({ "source_url": entry_url, "original_author": author, "published_at": published.map(|d| d.to_rfc3339()), "paywall": paywall, "feed_entry_id": entry.id, }); // 1. Opprett content-node sqlx::query( r#" INSERT INTO nodes (id, node_kind, title, content, visibility, metadata, created_by) VALUES ($1, 'content', $2, $3, 'hidden'::visibility, $4, $5) "#, ) .bind(node_id) .bind(&title) .bind(&content_text) .bind(&metadata) .bind(created_by) .execute(db) .await .map_err(|e| format!("INSERT node feilet: {e}"))?; // 2. tagged-edge "feed" (selv-edge) sqlx::query( r#" INSERT INTO edges (id, source_id, target_id, edge_type, metadata, system, created_by) VALUES ($1, $2, $2, 'tagged', $3, false, $4) ON CONFLICT (source_id, target_id, edge_type) DO NOTHING "#, ) .bind(Uuid::now_v7()) .bind(node_id) .bind(serde_json::json!({"tag": "feed"})) .bind(created_by) .execute(db) .await .map_err(|e| format!("INSERT tagged-edge feilet: {e}"))?; // 3. belongs_to-edge (content → samling) sqlx::query( r#" INSERT INTO edges (id, source_id, target_id, edge_type, metadata, system, created_by) VALUES ($1, $2, $3, 'belongs_to', '{}', false, $4) ON CONFLICT (source_id, target_id, edge_type) DO NOTHING "#, ) .bind(Uuid::now_v7()) .bind(node_id) .bind(collection_id) .bind(created_by) .execute(db) .await .map_err(|e| format!("INSERT belongs_to-edge feilet: {e}"))?; Ok(node_id) } // ============================================================================= // Paywall-deteksjon (gjenbrukt fra synops-clip) // ============================================================================= /// Detekter betalingsmur basert på innhold og HTML-mønstre. fn detect_paywall(content: &str, html: &str) -> bool { let content_lower = content.to_lowercase(); let html_lower = html.to_lowercase(); // 1. Veldig kort innhold (under 200 tegn) tyder på betalingsmur let trimmed_len = content.trim().len(); if trimmed_len > 0 && trimmed_len < 200 { debug!(trimmed_len, "Kort innhold — mulig betalingsmur"); return true; } // 2. Kjente betalingsmur-mønstre i innholdet let paywall_content_patterns = [ "logg inn for å lese", "log in to read", "sign in to continue", "subscribe to continue", "abonner for å lese", "kun for abonnenter", "for subscribers only", "du må være innlogget", "denne artikkelen er for abonnenter", "kjøp tilgang", "buy access", "unlock this article", "lås opp artikkelen", "registrer deg for å lese", "create an account to read", "meld deg inn", "bli abonnent", "allerede abonnent", "already a subscriber", ]; for pattern in &paywall_content_patterns { if content_lower.contains(pattern) { debug!(pattern, "Betalingsmur-mønster funnet i innhold"); return true; } } // 3. Kjente betalingsmur-klasser/elementer i HTML let paywall_html_patterns = [ "class=\"paywall", "class=\"pw-", "id=\"paywall", "data-paywall", "class=\"subscriber-only", "class=\"premium-content", "class=\"locked-content", "class=\"article-paywall", "class=\"metered-", "class=\"piano-", "id=\"piano-", "class=\"tp-modal", "data-piano", "class=\"ns-paywall", "class=\"nettavisen-paywall", "class=\"schibsted-paywall", "class=\"amedia-paywall", ]; for pattern in &paywall_html_patterns { if html_lower.contains(pattern) { debug!(pattern, "Betalingsmur-mønster funnet i HTML"); return true; } } // 4. Sjekk for meta-tagger som indikerer betalt innhold if html_lower.contains(r#"content="locked""#) || html_lower.contains(r#"content="premium""#) || html_lower.contains("isaccessibleforfree") && html_lower.contains(r#""false""#) { debug!("Betalingsmur-meta-tagg funnet"); return true; } false } // ============================================================================= // Hjelpefunksjoner // ============================================================================= /// Enkel HTML-tag-stripping for feed-innhold. fn strip_html_tags(html: &str) -> String { let mut result = String::with_capacity(html.len()); let mut in_tag = false; for ch in html.chars() { match ch { '<' => in_tag = true, '>' => in_tag = false, _ if !in_tag => result.push(ch), _ => {} } } // Normaliser whitespace result .split_whitespace() .collect::>() .join(" ") .trim() .to_string() } fn truncate(s: &str, max: usize) -> &str { if s.len() <= max { s } else { &s[..s.floor_char_boundary(max)] } }