vegard/synops
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Benchmark: lås til git-ref, advar ved dirty tree

Browse source 
Alle modeller testes mot nøyaktig samme kildekode.
Ref og tidsstempel logges i CSV-header.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
vegard 2026-03-18 15:15:36 +00:00
parent a836be6992
commit 67bc564f92
1 changed files with 16 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
scripts/benchmark-models.sh
View file

@ -9,6 +9,19 @@
set -euo pipefail set -euo pipefail
ROOT="$(cd "$(dirname "$0")/.." && pwd)" ROOT="$(cd "$(dirname "$0")/.." && pwd)"
# Lås til en bestemt git-ref for å sikre at alle modeller
# testes mot nøyaktig samme kildekode. Sett via env eller
# bruk HEAD ved kjøretidspunkt.
BENCHMARK_REF="${BENCHMARK_REF:-$(git -C "$ROOT" rev-parse HEAD)}"
echo "=== Benchmark kjører mot git ref: $BENCHMARK_REF ==="
echo "=== Tidspunkt: $(date -Iseconds) ==="
# Verifiser at working tree matcher ref (advar hvis dirty)
if ! git -C "$ROOT" diff --quiet HEAD 2>/dev/null; then
echo "ADVARSEL: working tree har ucommittede endringer."
echo "Resultatene kan avvike fra ref $BENCHMARK_REF."
fi
SIMPLE_FILE="tools/synops-transcribe/src/main.rs" SIMPLE_FILE="tools/synops-transcribe/src/main.rs"
COMPLEX_FILE="tools/synops-audio/src/main.rs" COMPLEX_FILE="tools/synops-audio/src/main.rs"
PROMPT_BASE="Les %s nøye. Gi en vurdering: 1) Hva gjør verktøyet? 2) Er det bugs? 3) Hva kan forbedres? Maks 300 ord." PROMPT_BASE="Les %s nøye. Gi en vurdering: 1) Hva gjør verktøyet? 2) Er det bugs? 3) Hva kan forbedres? Maks 300 ord."
@ -28,7 +41,9 @@ declare -A EFFORT_PREFIX=(
OUTDIR="/tmp/benchmark-$(date +%Y%m%d-%H%M)" OUTDIR="/tmp/benchmark-$(date +%Y%m%d-%H%M)"
mkdir -p "$OUTDIR" mkdir -p "$OUTDIR"
echo "modell,effort,fil,tid_sek,ord,bugs_nøkkelord" > "$OUTDIR/results.csv" echo "# git_ref: $BENCHMARK_REF" > "$OUTDIR/results.csv"
echo "# tidspunkt: $(date -Iseconds)" >> "$OUTDIR/results.csv"
echo "modell,effort,fil,tid_sek,ord,bugs_nøkkelord" >> "$OUTDIR/results.csv"
for file in "$SIMPLE_FILE" "$COMPLEX_FILE"; do for file in "$SIMPLE_FILE" "$COMPLEX_FILE"; do
file_label=$(basename "$file" .rs) file_label=$(basename "$file" .rs)