initial commit: live meeting transcription pipeline
Go orchestrator: spawns audiotee for capture (system audio + mic, two tracks), segments with an energy-threshold VAD, transcribes each segment via qwen-asr (subprocess per segment, vendored as a pinned git submodule in third_party/qwen-asr), and serves the live transcript over SSE while also writing it durably to a text file. Includes a glossary/prompt-leakage guard (internal/asr/leak.go) that discards a segment if the model echoes the biasing prompt instead of transcribing. cmd/transcriptor-ai is the main orchestrator; cmd/transcript-tail is a minimal terminal SSE client. See CLAUDE.md for the full architecture and the reasoning behind each choice (Go over Python/Swift/Rust, SSE over WebSocket, why the VAD is a hand-rolled heuristic, etc). Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
sttlab-tech
@@ -0,0 +1,77 @@
|
||||
// Package asr transcribes bounded PCM segments via qwen-asr
|
||||
// (https://github.com/antirez/qwen-asr), invoked as a one-shot subprocess
|
||||
// per segment rather than a long-lived daemon.
|
||||
//
|
||||
// This was verified empirically, not assumed: `qwen_asr --stdin --stream
|
||||
// --silent` does not print incremental partial text as it processes a
|
||||
// single stdin stream — it prints only the final transcription once done,
|
||||
// even in --stream mode (that flag changes its internal chunked-encoding
|
||||
// strategy, not what gets written to stdout). Measured latency including
|
||||
// process startup and model load, on the 0.6B model, was ~0.15-0.2x
|
||||
// realtime for a ~6s clip — comfortably fast enough to spawn fresh per VAD
|
||||
// segment instead of keeping a persistent process fed over a pipe.
|
||||
// Our own VAD segment boundaries (internal/vad) are therefore what stands
|
||||
// in for "live" here: each vad.Segment.IsFinal maps directly to the
|
||||
// SSE message's is_final field described in CLAUDE.md.
|
||||
package asr
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Config points at a built qwen_asr binary and a downloaded model
|
||||
// directory. Neither has an established default location yet (unlike
|
||||
// audiotee's ~/bin convention), so both are required.
|
||||
type Config struct {
|
||||
BinaryPath string
|
||||
ModelDir string
|
||||
// Prompt biases transcription toward glossary terms, e.g.
|
||||
// "Preserve spelling: Kubernetes, PostgreSQL". Optional.
|
||||
Prompt string
|
||||
// Threads sets qwen_asr's -t flag. Zero means "let it pick" (all CPUs).
|
||||
Threads int
|
||||
}
|
||||
|
||||
type Transcriber struct {
|
||||
cfg Config
|
||||
run runFunc // seam for testing without a real binary
|
||||
}
|
||||
|
||||
func New(cfg Config) *Transcriber {
|
||||
return &Transcriber{cfg: cfg, run: runCommand}
|
||||
}
|
||||
|
||||
// Transcribe runs qwen_asr once against pcm (raw s16le, 16kHz, mono — the
|
||||
// format both audiotee and internal/vad produce) and returns the
|
||||
// transcribed text, trimmed.
|
||||
func (t *Transcriber) Transcribe(ctx context.Context, pcm []byte) (string, error) {
|
||||
args := []string{"-d", t.cfg.ModelDir, "--stdin", "--stream", "--silent"}
|
||||
if t.cfg.Prompt != "" {
|
||||
args = append(args, "--prompt", t.cfg.Prompt)
|
||||
}
|
||||
if t.cfg.Threads > 0 {
|
||||
args = append(args, "-t", strconv.Itoa(t.cfg.Threads))
|
||||
}
|
||||
|
||||
stdout, stderr, err := t.run(ctx, t.cfg.BinaryPath, args, pcm)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("qwen_asr: %w (stderr: %s)", err, bytes.TrimSpace(stderr))
|
||||
}
|
||||
text := strings.TrimSpace(string(stdout))
|
||||
|
||||
if looksLikePromptLeak(text, t.cfg.Prompt) {
|
||||
// Discard rather than return it as a genuine transcription — see
|
||||
// leak.go. Logged here (not just silently dropped) since this
|
||||
// package has no structured logger of its own; matches the plain
|
||||
// stderr diagnostics the CLI already prints elsewhere.
|
||||
fmt.Fprintf(os.Stderr, "asr: discarding segment, looks like a --prompt leak, not a transcription: %q\n", text)
|
||||
return "", nil
|
||||
}
|
||||
|
||||
return text, nil
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
package asr
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// looksLikePromptLeak reports whether text appears to be the model echoing
|
||||
// the --prompt biasing content back verbatim instead of genuinely
|
||||
// transcribing the segment's audio — a known failure mode of
|
||||
// prompt-conditioned generation, confirmed via real testing against
|
||||
// qwen_asr (see transcriptor-ui/CLAUDE.md, "Glossary prompt-leakage bug"):
|
||||
// a whole segment came back as literally the glossary terms, twice, on two
|
||||
// separate 5-minute test runs, even after switching to the tool's
|
||||
// recommended "Preserve spelling: ..." prompt framing (which reduces but
|
||||
// does not eliminate the risk).
|
||||
//
|
||||
// Heuristic: if most of text's words also appear among prompt's words, and
|
||||
// there are enough of them to rule out a legitimate short mention of one
|
||||
// or two glossary terms in real speech, treat it as a leak. Both observed
|
||||
// real leaks were near-total reproductions of the prompt's term list, so a
|
||||
// strict threshold (80% word overlap, at least 4 words) catches them
|
||||
// without plausibly false-flagging a normal sentence that happens to
|
||||
// mention "Claude" or "Mistral" once — normal speech has enough other
|
||||
// words to keep the overlap ratio well below that.
|
||||
func looksLikePromptLeak(text, prompt string) bool {
|
||||
if prompt == "" || text == "" {
|
||||
return false
|
||||
}
|
||||
textWords := normalizeWords(text)
|
||||
if len(textWords) < 4 {
|
||||
return false
|
||||
}
|
||||
promptWords := wordSet(normalizeWords(prompt))
|
||||
|
||||
matched := 0
|
||||
for _, w := range textWords {
|
||||
if promptWords[w] {
|
||||
matched++
|
||||
}
|
||||
}
|
||||
return float64(matched)/float64(len(textWords)) >= 0.8
|
||||
}
|
||||
|
||||
// normalizeWords lowercases and splits on anything that isn't a letter or
|
||||
// digit — unicode.IsLetter (not an ASCII-only check) so accented
|
||||
// characters (French: é, è, ï, ç, ...) stay part of their word instead of
|
||||
// being split into fragments.
|
||||
func normalizeWords(s string) []string {
|
||||
return strings.FieldsFunc(strings.ToLower(s), func(r rune) bool {
|
||||
return !unicode.IsLetter(r) && !unicode.IsDigit(r)
|
||||
})
|
||||
}
|
||||
|
||||
func wordSet(words []string) map[string]bool {
|
||||
set := make(map[string]bool, len(words))
|
||||
for _, w := range words {
|
||||
set[w] = true
|
||||
}
|
||||
return set
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
package asr
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"os/exec"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
// runFunc executes binary with args, feeding stdin, and returns
|
||||
// stdout/stderr. It's a seam so tests can substitute a fake process instead
|
||||
// of requiring the real qwen_asr binary and a multi-GB model on disk.
|
||||
type runFunc func(ctx context.Context, binary string, args []string, stdin []byte) (stdout, stderr []byte, err error)
|
||||
|
||||
func runCommand(ctx context.Context, binary string, args []string, stdin []byte) ([]byte, []byte, error) {
|
||||
cmd := exec.CommandContext(ctx, binary, args...)
|
||||
cmd.Stdin = bytes.NewReader(stdin)
|
||||
var stdout, stderr bytes.Buffer
|
||||
cmd.Stdout = &stdout
|
||||
cmd.Stderr = &stderr
|
||||
// Run qwen_asr in its own process group so the terminal's Ctrl+C
|
||||
// (SIGINT to the whole foreground process group) doesn't kill an
|
||||
// in-flight transcription directly. It's still cancelable through ctx,
|
||||
// same as before — this only stops the shell from also signaling it.
|
||||
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
|
||||
err := cmd.Run()
|
||||
return stdout.Bytes(), stderr.Bytes(), err
|
||||
}
|
||||
@@ -0,0 +1,238 @@
|
||||
// Package capture wraps the audiotee subprocess and turns its two audio
|
||||
// outputs (system audio on stdout, microphone on a FIFO) into a single
|
||||
// stream of tagged chunks that downstream VAD/ASR stages can consume.
|
||||
package capture
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Track identifies which audio source a Chunk came from.
|
||||
type Track string
|
||||
|
||||
const (
|
||||
TrackSystem Track = "system"
|
||||
TrackMic Track = "mic"
|
||||
)
|
||||
|
||||
// Chunk is a slice of raw PCM audio (16-bit signed, little-endian, mono, at
|
||||
// Config.SampleRate) read off one of audiotee's two outputs. Timestamp is
|
||||
// when this process read the chunk, not when audiotee captured it — audiotee
|
||||
// does not emit per-chunk timestamps (see transcriptor-ai/CLAUDE.md), so this
|
||||
// is an approximation good enough for live display, not sample-accurate
|
||||
// realignment.
|
||||
type Chunk struct {
|
||||
Track Track
|
||||
Data []byte
|
||||
Timestamp time.Time
|
||||
}
|
||||
|
||||
// Config controls how the audiotee subprocess is launched.
|
||||
type Config struct {
|
||||
// AudioteePath is the path to the audiotee binary. If empty, it's
|
||||
// resolved via PATH and then $HOME/bin/audiotee.
|
||||
AudioteePath string
|
||||
// SampleRate is passed to audiotee's --sample-rate flag.
|
||||
SampleRate int
|
||||
// ChunkBytes is the read buffer size used per Chunk. Must be even
|
||||
// (16-bit samples). Defaults to 3200 bytes (~100ms at 16kHz mono).
|
||||
ChunkBytes int
|
||||
// CaptureMic controls whether audiotee is asked to also capture the
|
||||
// mic track (--capture-mic --mic-output). audiotee has no equivalent
|
||||
// flag to skip the system tap — it's always captured — so there is no
|
||||
// "mic only" at this layer; a caller that only wants mic output must
|
||||
// still receive (and can simply ignore) TrackSystem chunks. See
|
||||
// CLAUDE.md for why this is a known, accepted limitation rather than
|
||||
// something fixed here.
|
||||
CaptureMic bool
|
||||
}
|
||||
|
||||
// Capturer runs audiotee and streams both its tracks as Chunks.
|
||||
type Capturer struct {
|
||||
cfg Config
|
||||
cmd *exec.Cmd
|
||||
fifoPath string
|
||||
micFile *os.File
|
||||
|
||||
chunks chan Chunk
|
||||
errs chan error
|
||||
|
||||
wg sync.WaitGroup
|
||||
done chan struct{} // closed once both track readers have hit EOF
|
||||
}
|
||||
|
||||
// New creates a Capturer. Call Start to launch audiotee.
|
||||
func New(cfg Config) *Capturer {
|
||||
if cfg.ChunkBytes <= 0 {
|
||||
cfg.ChunkBytes = 3200
|
||||
}
|
||||
if cfg.ChunkBytes%2 != 0 {
|
||||
cfg.ChunkBytes++
|
||||
}
|
||||
return &Capturer{
|
||||
cfg: cfg,
|
||||
chunks: make(chan Chunk, 64),
|
||||
errs: make(chan error, 4),
|
||||
}
|
||||
}
|
||||
|
||||
// Chunks returns the channel both tracks' audio is delivered on. Closed once
|
||||
// both tracks have stopped delivering data (audiotee exited or Stop was
|
||||
// called).
|
||||
func (c *Capturer) Chunks() <-chan Chunk { return c.chunks }
|
||||
|
||||
// Errs returns non-fatal errors encountered while reading either track.
|
||||
func (c *Capturer) Errs() <-chan error { return c.errs }
|
||||
|
||||
// Start resolves the audiotee binary, launches it with system+mic capture
|
||||
// enabled, and begins streaming both tracks. It returns once audiotee has
|
||||
// been launched; reading happens in background goroutines.
|
||||
func (c *Capturer) Start(ctx context.Context) error {
|
||||
audioteePath, err := resolveAudioteePath(c.cfg.AudioteePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
args := []string{"--sample-rate", fmt.Sprintf("%d", c.cfg.SampleRate)}
|
||||
|
||||
var fifoPath string
|
||||
if c.cfg.CaptureMic {
|
||||
fifoPath = filepath.Join(os.TempDir(), fmt.Sprintf("transcriptor-ai-mic-%d.fifo", os.Getpid()))
|
||||
if err := syscall.Mkfifo(fifoPath, 0o600); err != nil {
|
||||
return fmt.Errorf("create mic fifo: %w", err)
|
||||
}
|
||||
c.fifoPath = fifoPath
|
||||
args = append(args, "--capture-mic", "--mic-output", fifoPath)
|
||||
}
|
||||
|
||||
cmd := exec.CommandContext(ctx, audioteePath, args...)
|
||||
cmd.Stderr = os.Stderr // audiotee's JSON logs, useful as-is while iterating
|
||||
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
c.cleanupFifo()
|
||||
return fmt.Errorf("attach stdout pipe: %w", err)
|
||||
}
|
||||
|
||||
// Open the FIFO for reading in the background before starting audiotee:
|
||||
// opening a FIFO blocks until the other end is opened too, so this must
|
||||
// not block Start() itself, and audiotee (the writer) hasn't been
|
||||
// spawned yet at this point.
|
||||
var micOpened chan struct{}
|
||||
var micFile *os.File
|
||||
var micOpenErr error
|
||||
if c.cfg.CaptureMic {
|
||||
micOpened = make(chan struct{})
|
||||
go func() {
|
||||
defer close(micOpened)
|
||||
micFile, micOpenErr = os.OpenFile(fifoPath, os.O_RDONLY, 0)
|
||||
}()
|
||||
}
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
c.cleanupFifo()
|
||||
return fmt.Errorf("start audiotee: %w", err)
|
||||
}
|
||||
c.cmd = cmd
|
||||
|
||||
if c.cfg.CaptureMic {
|
||||
<-micOpened
|
||||
if micOpenErr != nil {
|
||||
return fmt.Errorf("open mic fifo: %w", micOpenErr)
|
||||
}
|
||||
c.micFile = micFile
|
||||
}
|
||||
|
||||
c.done = make(chan struct{})
|
||||
c.wg.Add(1)
|
||||
go c.readTrack(TrackSystem, stdout)
|
||||
|
||||
if c.cfg.CaptureMic {
|
||||
c.wg.Add(1)
|
||||
go c.readTrack(TrackMic, micFile)
|
||||
}
|
||||
|
||||
go func() {
|
||||
c.wg.Wait()
|
||||
close(c.chunks)
|
||||
close(c.done)
|
||||
}()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Capturer) readTrack(track Track, r io.Reader) {
|
||||
defer c.wg.Done()
|
||||
buf := make([]byte, c.cfg.ChunkBytes)
|
||||
for {
|
||||
n, err := io.ReadFull(r, buf)
|
||||
if n > 0 {
|
||||
data := make([]byte, n)
|
||||
copy(data, buf[:n])
|
||||
c.chunks <- Chunk{Track: track, Data: data, Timestamp: time.Now()}
|
||||
}
|
||||
if err != nil {
|
||||
if err != io.EOF && err != io.ErrUnexpectedEOF {
|
||||
select {
|
||||
case c.errs <- fmt.Errorf("%s track: %w", track, err):
|
||||
default:
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Stop terminates audiotee gracefully (SIGINT, matching the fix in audiotee
|
||||
// itself so shutdown doesn't hang) and cleans up the FIFO.
|
||||
func (c *Capturer) Stop() error {
|
||||
if c.cmd != nil && c.cmd.Process != nil {
|
||||
_ = c.cmd.Process.Signal(syscall.SIGINT)
|
||||
}
|
||||
if c.done != nil {
|
||||
// audiotee exiting closes its stdout pipe and the mic FIFO write
|
||||
// end, which is what makes readTrack see EOF — wait for that before
|
||||
// reaping the process. Calling cmd.Wait() first can close the pipe
|
||||
// out from under an in-progress read (see os/exec's StdoutPipe doc).
|
||||
<-c.done
|
||||
}
|
||||
if c.cmd != nil && c.cmd.Process != nil {
|
||||
_ = c.cmd.Wait()
|
||||
}
|
||||
if c.micFile != nil {
|
||||
_ = c.micFile.Close()
|
||||
}
|
||||
c.cleanupFifo()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Capturer) cleanupFifo() {
|
||||
if c.fifoPath != "" {
|
||||
_ = os.Remove(c.fifoPath)
|
||||
}
|
||||
}
|
||||
|
||||
func resolveAudioteePath(configured string) (string, error) {
|
||||
if configured != "" {
|
||||
return configured, nil
|
||||
}
|
||||
if p, err := exec.LookPath("audiotee"); err == nil {
|
||||
return p, nil
|
||||
}
|
||||
home, err := os.UserHomeDir()
|
||||
if err == nil {
|
||||
candidate := filepath.Join(home, "bin", "audiotee")
|
||||
if info, statErr := os.Stat(candidate); statErr == nil && !info.IsDir() {
|
||||
return candidate, nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("audiotee binary not found on PATH or in ~/bin — build it via audiotee/scripts/build-signed.sh")
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
// Package transcript defines the live transcript message format (see
|
||||
// CLAUDE.md) and a Broadcaster that fans a single stream of messages out to
|
||||
// any number of SSE clients — the merge stage's output contract.
|
||||
//
|
||||
// Ordering note: messages are broadcast in the order transcribeTrack
|
||||
// produces them, not re-sorted by Timestamp. With two tracks processed
|
||||
// independently, a slower track could in principle publish slightly out of
|
||||
// chronological order relative to the other. Measured ASR latency
|
||||
// (~0.15-0.2x realtime) is small relative to segment spacing (3-5s), so
|
||||
// this hasn't been observed in practice — add a small reordering buffer
|
||||
// here if it ever becomes visible, not before.
|
||||
package transcript
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/stephanetailland/transcriptor-ai/internal/capture"
|
||||
)
|
||||
|
||||
// Message is one transcript segment, matching the SSE JSON format
|
||||
// documented in CLAUDE.md.
|
||||
type Message struct {
|
||||
Track capture.Track `json:"track"`
|
||||
Text string `json:"text"`
|
||||
IsFinal bool `json:"is_final"`
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
}
|
||||
|
||||
// clientBuffer bounds how many undelivered messages a slow SSE client can
|
||||
// accumulate before new messages are dropped for it, so one stalled client
|
||||
// can't back-pressure the whole pipeline.
|
||||
const clientBuffer = 32
|
||||
|
||||
// Broadcaster fans Published messages out to every currently-connected SSE
|
||||
// client and doubles as the http.Handler for the SSE endpoint.
|
||||
type Broadcaster struct {
|
||||
mu sync.Mutex
|
||||
clients map[chan Message]struct{}
|
||||
}
|
||||
|
||||
func NewBroadcaster() *Broadcaster {
|
||||
return &Broadcaster{clients: make(map[chan Message]struct{})}
|
||||
}
|
||||
|
||||
// Publish sends msg to every connected client. Non-blocking: a client whose
|
||||
// buffer is full misses the message rather than stalling the sender.
|
||||
func (b *Broadcaster) Publish(msg Message) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
for ch := range b.clients {
|
||||
select {
|
||||
case ch <- msg:
|
||||
default:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (b *Broadcaster) subscribe() chan Message {
|
||||
ch := make(chan Message, clientBuffer)
|
||||
b.mu.Lock()
|
||||
b.clients[ch] = struct{}{}
|
||||
b.mu.Unlock()
|
||||
return ch
|
||||
}
|
||||
|
||||
func (b *Broadcaster) unsubscribe(ch chan Message) {
|
||||
b.mu.Lock()
|
||||
delete(b.clients, ch)
|
||||
b.mu.Unlock()
|
||||
}
|
||||
|
||||
// ServeHTTP streams messages to the client as Server-Sent Events until the
|
||||
// client disconnects. See CLAUDE.md for why SSE over WebSocket: the flow is
|
||||
// one-directional, and this needs no client library beyond EventSource in a
|
||||
// browser or a plain HTTP client in a terminal (e.g. `curl -N`).
|
||||
func (b *Broadcaster) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
flusher, ok := w.(http.Flusher)
|
||||
if !ok {
|
||||
http.Error(w, "streaming unsupported", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
w.Header().Set("Cache-Control", "no-cache")
|
||||
w.Header().Set("Connection", "keep-alive")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
flusher.Flush()
|
||||
|
||||
ch := b.subscribe()
|
||||
defer b.unsubscribe(ch)
|
||||
|
||||
for {
|
||||
select {
|
||||
case msg, ok := <-ch:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
data, err := json.Marshal(msg)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(w, "data: %s\n\n", data)
|
||||
flusher.Flush()
|
||||
case <-r.Context().Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package transcript
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// FileWriter appends each Message to a text file as a human-readable line,
|
||||
// flushed to disk immediately. Unlike Broadcaster.Publish (which drops
|
||||
// messages for a slow/stalled SSE client on purpose), FileWriter never
|
||||
// drops — this is the durable record a crash shouldn't lose data from, so
|
||||
// every Write is synchronous and fsync'd before returning.
|
||||
type FileWriter struct {
|
||||
mu sync.Mutex
|
||||
f *os.File
|
||||
}
|
||||
|
||||
// NewFileWriter opens path for appending, creating it if needed. Appending
|
||||
// (not truncating) means restarting transcriptor-ai against the same path
|
||||
// continues the same transcript file rather than discarding it.
|
||||
func NewFileWriter(path string) (*FileWriter, error) {
|
||||
f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open transcript file: %w", err)
|
||||
}
|
||||
return &FileWriter{f: f}, nil
|
||||
}
|
||||
|
||||
// Write appends one line for msg and fsyncs before returning.
|
||||
func (fw *FileWriter) Write(msg Message) error {
|
||||
fw.mu.Lock()
|
||||
defer fw.mu.Unlock()
|
||||
line := fmt.Sprintf("[%s] [%s] %s\n", msg.Timestamp.Format("15:04:05"), msg.Track, msg.Text)
|
||||
if _, err := io.WriteString(fw.f, line); err != nil {
|
||||
return fmt.Errorf("write transcript line: %w", err)
|
||||
}
|
||||
if err := fw.f.Sync(); err != nil {
|
||||
return fmt.Errorf("sync transcript file: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (fw *FileWriter) Close() error {
|
||||
fw.mu.Lock()
|
||||
defer fw.mu.Unlock()
|
||||
return fw.f.Close()
|
||||
}
|
||||
@@ -0,0 +1,217 @@
|
||||
// Package vad turns a stream of raw audio chunks into bounded speech
|
||||
// segments using a simple energy-threshold heuristic — no ML model. See
|
||||
// transcriptor-ai/CLAUDE.md for why: no Go/Swift ML ecosystem was mature
|
||||
// enough to justify the integration cost for this. Revisit only if
|
||||
// segmentation quality proves to be the actual bottleneck.
|
||||
package vad
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"math"
|
||||
"time"
|
||||
|
||||
"github.com/stephanetailland/transcriptor-ai/internal/capture"
|
||||
)
|
||||
|
||||
// Segment is a bounded span of speech audio ready for ASR.
|
||||
type Segment struct {
|
||||
Track capture.Track
|
||||
Data []byte // raw PCM, concatenation of the contributing chunks
|
||||
Start time.Time
|
||||
End time.Time
|
||||
// IsFinal is true when silence closed the utterance, false when the
|
||||
// segment was force-cut at MaxSegmentDuration and more audio for the
|
||||
// same utterance follows in the next segment (which starts with
|
||||
// Config.Overlap worth of audio repeated for ASR context continuity).
|
||||
IsFinal bool
|
||||
}
|
||||
|
||||
// Config controls the energy-threshold heuristic. All durations are
|
||||
// expressed in wall-clock time; SampleRate converts them to byte counts.
|
||||
type Config struct {
|
||||
SampleRate int
|
||||
|
||||
// SilenceThresholdDB is the RMS level (dBFS, 0 = full scale) below which
|
||||
// a chunk is considered silence. Real speech in testing sits around -40
|
||||
// to -20 dBFS; captured silence sits around -90 dBFS. Needs empirical
|
||||
// tuning against real meeting audio — this default is a starting point,
|
||||
// not a validated value.
|
||||
SilenceThresholdDB float64
|
||||
|
||||
// SilenceHangover is how long silence must persist after speech before
|
||||
// the segment is closed as final (IsFinal: true).
|
||||
SilenceHangover time.Duration
|
||||
|
||||
// MinSpeechDuration is the minimum contiguous voiced audio required
|
||||
// before a segment starts accumulating, to filter transient noise
|
||||
// (clicks, pops) rather than opening a segment on every blip.
|
||||
MinSpeechDuration time.Duration
|
||||
|
||||
// MaxSegmentDuration force-cuts a segment even mid-speech, so a long
|
||||
// monologue still produces incremental, timely output instead of one
|
||||
// unbounded segment.
|
||||
MaxSegmentDuration time.Duration
|
||||
|
||||
// Overlap is how much trailing audio is carried into the next segment
|
||||
// when force-cut, for ASR context continuity across the cut. Tradeoff:
|
||||
// the carried-over audio gets transcribed again, and both copies are
|
||||
// published — a nonzero Overlap produces visible duplicate text at
|
||||
// every force-cut boundary (confirmed in real testing: this is worse
|
||||
// than the word occasionally getting cut in half at a hard boundary,
|
||||
// which is why DefaultConfig uses 0). Only raise this if paired with
|
||||
// dedup logic on the published text — never on its own.
|
||||
Overlap time.Duration
|
||||
}
|
||||
|
||||
// DefaultConfig returns starting-point thresholds; tune against real audio.
|
||||
func DefaultConfig(sampleRate int) Config {
|
||||
return Config{
|
||||
SampleRate: sampleRate,
|
||||
SilenceThresholdDB: -40,
|
||||
SilenceHangover: 600 * time.Millisecond,
|
||||
MinSpeechDuration: 200 * time.Millisecond,
|
||||
MaxSegmentDuration: 4 * time.Second,
|
||||
Overlap: 0,
|
||||
}
|
||||
}
|
||||
|
||||
func (c Config) bytesForDuration(d time.Duration) int {
|
||||
// 16-bit mono samples: 2 bytes/sample.
|
||||
samples := int(float64(c.SampleRate) * d.Seconds())
|
||||
n := samples * 2
|
||||
if n%2 != 0 {
|
||||
n++
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// Detector runs the heuristic independently per track (system and mic have
|
||||
// separate state) over a shared, tagged chunk stream.
|
||||
type Detector struct {
|
||||
cfg Config
|
||||
states map[capture.Track]*trackState
|
||||
}
|
||||
|
||||
func New(cfg Config) *Detector {
|
||||
return &Detector{cfg: cfg, states: make(map[capture.Track]*trackState)}
|
||||
}
|
||||
|
||||
type trackState struct {
|
||||
inSpeech bool
|
||||
buffer []byte
|
||||
segStart time.Time
|
||||
|
||||
// candidate holds voiced audio not yet promoted to a real segment,
|
||||
// while we wait to confirm MinSpeechDuration.
|
||||
candidate []byte
|
||||
candidateStart time.Time
|
||||
|
||||
silenceSince time.Time // zero value means "not currently in a silence run"
|
||||
lastChunkAt time.Time // timestamp of the most recently processed chunk
|
||||
}
|
||||
|
||||
// Run consumes chunks until the channel closes, emitting Segments on the
|
||||
// returned channel. Any in-progress segment is flushed (as final, since
|
||||
// capture ended) before the output channel closes.
|
||||
func (d *Detector) Run(chunks <-chan capture.Chunk) <-chan Segment {
|
||||
out := make(chan Segment, 16)
|
||||
go func() {
|
||||
defer close(out)
|
||||
for chunk := range chunks {
|
||||
d.process(chunk, out)
|
||||
}
|
||||
for track, st := range d.states {
|
||||
if st.inSpeech && len(st.buffer) > 0 {
|
||||
out <- Segment{Track: track, Data: st.buffer, Start: st.segStart, End: st.lastChunkAt, IsFinal: true}
|
||||
}
|
||||
}
|
||||
}()
|
||||
return out
|
||||
}
|
||||
|
||||
func (d *Detector) process(chunk capture.Chunk, out chan<- Segment) {
|
||||
st, ok := d.states[chunk.Track]
|
||||
if !ok {
|
||||
st = &trackState{}
|
||||
d.states[chunk.Track] = st
|
||||
}
|
||||
|
||||
speech := dbFS(chunk.Data) >= d.cfg.SilenceThresholdDB
|
||||
st.lastChunkAt = chunk.Timestamp
|
||||
|
||||
switch {
|
||||
case speech && !st.inSpeech:
|
||||
// Possible start of an utterance — buffer as a candidate until
|
||||
// MinSpeechDuration is confirmed, to filter transient noise.
|
||||
if len(st.candidate) == 0 {
|
||||
st.candidateStart = chunk.Timestamp
|
||||
}
|
||||
st.candidate = append(st.candidate, chunk.Data...)
|
||||
if chunk.Timestamp.Sub(st.candidateStart) >= d.cfg.MinSpeechDuration {
|
||||
st.inSpeech = true
|
||||
st.segStart = st.candidateStart
|
||||
st.buffer = st.candidate
|
||||
st.candidate = nil
|
||||
st.silenceSince = time.Time{}
|
||||
}
|
||||
|
||||
case speech && st.inSpeech:
|
||||
st.buffer = append(st.buffer, chunk.Data...)
|
||||
st.silenceSince = time.Time{} // speech resumed, cancel any pending hangover
|
||||
if chunk.Timestamp.Sub(st.segStart) >= d.cfg.MaxSegmentDuration {
|
||||
out <- Segment{Track: chunk.Track, Data: st.buffer, Start: st.segStart, End: chunk.Timestamp, IsFinal: false}
|
||||
st.buffer = tail(st.buffer, d.cfg.bytesForDuration(d.cfg.Overlap))
|
||||
st.segStart = chunk.Timestamp.Add(-d.cfg.Overlap)
|
||||
}
|
||||
|
||||
case !speech && st.inSpeech:
|
||||
if st.silenceSince.IsZero() {
|
||||
st.silenceSince = chunk.Timestamp
|
||||
}
|
||||
st.buffer = append(st.buffer, chunk.Data...) // keep brief trailing silence for natural cutoff
|
||||
if chunk.Timestamp.Sub(st.silenceSince) >= d.cfg.SilenceHangover {
|
||||
out <- Segment{Track: chunk.Track, Data: st.buffer, Start: st.segStart, End: chunk.Timestamp, IsFinal: true}
|
||||
st.inSpeech = false
|
||||
st.buffer = nil
|
||||
st.silenceSince = time.Time{}
|
||||
}
|
||||
|
||||
default: // !speech && !st.inSpeech
|
||||
st.candidate = nil
|
||||
}
|
||||
}
|
||||
|
||||
// dbFS returns the RMS level of a 16-bit little-endian PCM buffer in dBFS
|
||||
// (0 = full scale for int16). Silence (all-zero input) returns -infinity,
|
||||
// which compares correctly against any real threshold.
|
||||
func dbFS(data []byte) float64 {
|
||||
n := len(data) / 2
|
||||
if n == 0 {
|
||||
return math.Inf(-1)
|
||||
}
|
||||
var sumSquares float64
|
||||
for i := 0; i+1 < len(data); i += 2 {
|
||||
s := int16(binary.LittleEndian.Uint16(data[i : i+2]))
|
||||
v := float64(s)
|
||||
sumSquares += v * v
|
||||
}
|
||||
rms := math.Sqrt(sumSquares / float64(n))
|
||||
if rms == 0 {
|
||||
return math.Inf(-1)
|
||||
}
|
||||
return 20 * math.Log10(rms/32768)
|
||||
}
|
||||
|
||||
func tail(data []byte, n int) []byte {
|
||||
if n <= 0 {
|
||||
return nil
|
||||
}
|
||||
if n >= len(data) {
|
||||
out := make([]byte, len(data))
|
||||
copy(out, data)
|
||||
return out
|
||||
}
|
||||
out := make([]byte, n)
|
||||
copy(out, data[len(data)-n:])
|
||||
return out
|
||||
}
|
||||
Reference in New Issue
Block a user