From b54194798972c7736605abd912c644f69f894e86 Mon Sep 17 00:00:00 2001 From: nuxflix <57486719+nuxflix@users.noreply.github.com> Date: Sat, 29 Aug 2026 18:33:55 -0400 Subject: [PATCH] feat: add RMS, silence trim, and last user text Give sessions a way to measure PCM energy, strip leading and trailing silence, and read the most recent spoken user turn. Co-authored-by: Cursor --- CHANGELOG.md | 9 ++++++ audio/utils.go | 55 ++++++++++++++++++++++++++++++++ audio/utils_test.go | 62 ++++++++++++++++++++++++++++++++++++ docs/concepts/llm-context.md | 2 +- docs/guides/audio.md | 4 +++ frames/llm_context.go | 16 ++++++++++ frames/llm_context_test.go | 22 +++++++++++++ processor/trim.go | 40 +++++++++++++++++++++++ processor/trim_test.go | 60 ++++++++++++++++++++++++++++++++++ 9 files changed, 269 insertions(+), 1 deletion(-) create mode 100644 processor/trim.go create mode 100644 processor/trim_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a3e04fb..5279f17d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,15 @@ and this project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.ht ### Added +- **RMS energy and silence trimming.** `audio.RMS` reports the root-mean-square + amplitude of a 16-bit PCM chunk. `audio.TrimSilence` drops leading and + trailing silence, and `processor.NewAudioTrim` applies that to every audio + frame on the pipeline. + +- **The last spoken user turn.** `frames.LLMContext.LastUserText` returns the + text of the most recent user message, skipping tool-result placeholders that + share the user role. + - **A user turn processor.** `turns.NewUserTurnProcessor` decides the user's turn in a processor of its own, so the decision can be made once and shared by several aggregators, or placed at a particular point in the pipeline. The diff --git a/audio/utils.go b/audio/utils.go index 3e6a34e9..732c5761 100644 --- a/audio/utils.go +++ b/audio/utils.go @@ -3,6 +3,7 @@ package audio import ( "bytes" "encoding/binary" + "math" "github.com/nuxflix/voxigo/audio/g711" "github.com/nuxflix/voxigo/audio/resample" @@ -35,6 +36,60 @@ func IsSilence(pcm []byte) bool { return true } +// RMS is the root-mean-square amplitude of a chunk of 16-bit signed PCM. It is +// the usual energy measure for a buffer: a silent chunk is near zero, speech is +// typically a few hundred to a few thousand, and a full-scale square wave is +// 32767. An empty buffer, or one that does not complete a sample, is 0. +func RMS(pcm []byte) float64 { + n := 0 + var sum float64 + for i := 0; i+1 < len(pcm); i += 2 { + s := float64(int16(binary.LittleEndian.Uint16(pcm[i:]))) + sum += s * s + n++ + } + if n == 0 { + return 0 + } + return math.Sqrt(sum / float64(n)) +} + +// TrimSilence drops leading and trailing silent samples from a chunk of 16-bit +// signed PCM, using the same threshold as IsSilence. What remains is a copy, so +// a later write to the input cannot reach it. An all-silent or empty buffer +// comes back empty. A trailing odd byte is ignored, as it is on IsSilence. +func TrimSilence(pcm []byte) []byte { + n := len(pcm) - len(pcm)%2 + start := 0 + for start+1 < n { + if absSample(pcm, start) > speakingThreshold { + break + } + start += 2 + } + end := n + for end >= start+2 { + if absSample(pcm, end-2) > speakingThreshold { + break + } + end -= 2 + } + if start >= end { + return nil + } + out := make([]byte, end-start) + copy(out, pcm[start:end]) + return out +} + +func absSample(pcm []byte, i int) int { + sample := int(int16(binary.LittleEndian.Uint16(pcm[i:]))) + if sample < 0 { + sample = -sample + } + return sample +} + // MixAudio sums two streams of 16-bit signed PCM sample by sample, clipping the // result to the 16-bit range. The streams need not be the same length: the // shorter one is treated as though it were padded with silence, so the result is diff --git a/audio/utils_test.go b/audio/utils_test.go index 9860d491..a1f0c30e 100644 --- a/audio/utils_test.go +++ b/audio/utils_test.go @@ -64,6 +64,68 @@ func equal(got []int16, want ...int16) bool { return true } +func TestRMS(t *testing.T) { + tests := []struct { + name string + in []byte + want float64 + }{ + {"empty", nil, 0}, + {"silence", pcm(0, 0, 0), 0}, + {"the 3-4-5 triangle", pcm(3, 4), 5}, + {"constant amplitude", pcm(100, -100), 100}, + {"odd trailing byte is ignored", append(pcm(3, 4), 0xFF), 5}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := audio.RMS(tt.in) + if mathAbs(got-tt.want) > 1e-9 { + t.Errorf("RMS() = %v, want %v", got, tt.want) + } + }) + } +} + +func mathAbs(v float64) float64 { + if v < 0 { + return -v + } + return v +} + +func TestTrimSilence(t *testing.T) { + tests := []struct { + name string + in []byte + want []int16 + }{ + {"empty", nil, nil}, + {"all silence", pcm(0, 0, 10), nil}, + {"leading and trailing drop away", pcm(0, 100, 200, 0), []int16{100, 200}}, + {"at the threshold is still silence", pcm(20, 100, -20), []int16{100}}, + {"speech only is unchanged", pcm(100, -200), []int16{100, -200}}, + {"odd trailing byte is ignored", append(pcm(0, 100, 0), 0xFF), []int16{100}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := samples(audio.TrimSilence(tt.in)) + if !equal(got, tt.want...) { + t.Errorf("TrimSilence() = %v, want %v", got, tt.want) + } + }) + } + + t.Run("returns a copy", func(t *testing.T) { + in := pcm(0, 100, 0) + out := audio.TrimSilence(in) + in[2] = 0 + in[3] = 0 + if got := samples(out); !equal(got, 100) { + t.Errorf("mutating the input reached the result: %v", got) + } + }) +} + func TestMixAudio(t *testing.T) { tests := []struct { name string diff --git a/docs/concepts/llm-context.md b/docs/concepts/llm-context.md index d89f2622..2b74cc9f 100644 --- a/docs/concepts/llm-context.md +++ b/docs/concepts/llm-context.md @@ -24,7 +24,7 @@ long-lived aggregate shared between the aggregators and the LLM service, and it convo.AddUserMessage("What's the weather?") convo.AddAssistantMessage("Sunny and 22 degrees.") msgs := convo.Messages() // a copy -n := convo.EstimatedTokens() +last := convo.LastUserText() // "What's the weather?" convo.SetSystem("You are terse.") // swap the prompt mid-conversation convo.SetTools(tools) // change advertised tools diff --git a/docs/guides/audio.md b/docs/guides/audio.md index e4008b58..e5f3d46a 100644 --- a/docs/guides/audio.md +++ b/docs/guides/audio.md @@ -150,6 +150,10 @@ buffer holds audio in memory. ## Other pieces +- **`audio.RMS`**: the root-mean-square amplitude of a 16-bit PCM chunk, for an + energy gate or a level meter that should not depend on a single peak sample. +- **`audio.TrimSilence`** / **`processor.NewAudioTrim`**: drop leading and + trailing silence from a buffer or from every audio frame on the pipeline. - **`audio/onset`**: finds the first audible sample in a PCM stream, so time-to-first-audio metrics measure real speech rather than leading silence. - **`audio/chain.go`**: composes several `audio.Filter`s into one. diff --git a/frames/llm_context.go b/frames/llm_context.go index 1ce7c194..b7c28026 100644 --- a/frames/llm_context.go +++ b/frames/llm_context.go @@ -465,6 +465,22 @@ func (c *LLMContext) Messages() []Message { return cloneMessages(c.messages) } +// LastUserText returns the text of the most recent user turn, skipping tool +// result messages (those are written as the user role so they sit next to the +// call they answer). An empty string means no spoken user turn is in the +// conversation yet. +func (c *LLMContext) LastUserText() string { + c.mu.Lock() + defer c.mu.Unlock() + for i := len(c.messages) - 1; i >= 0; i-- { + m := c.messages[i] + if m.Role == RoleUser && len(m.ToolResults) == 0 { + return m.Text + } + } + return "" +} + // MessagesFor returns the messages to send to the named provider: every // universal one, plus the provider's own, and none written for a different // provider. It is what an adapter reads rather than Messages, so a conversation diff --git a/frames/llm_context_test.go b/frames/llm_context_test.go index b9dee0bf..a8e69553 100644 --- a/frames/llm_context_test.go +++ b/frames/llm_context_test.go @@ -244,3 +244,25 @@ func TestSetMessagesDoesNotAliasTheCaller(t *testing.T) { t.Errorf("context result = %q, want it untouched by the caller's slice", got) } } + +// TestLastUserText returns the most recent spoken user turn and skips the user +// role that only carries a tool result. +func TestLastUserText(t *testing.T) { + c := frames.NewLLMContext("system") + if got := c.LastUserText(); got != "" { + t.Errorf("LastUserText() = %q, want empty on a new context", got) + } + + c.AddUserMessage("first") + c.AddAssistantMessage("reply") + c.AddUserMessage("second") + if got := c.LastUserText(); got != "second" { + t.Errorf("LastUserText() = %q, want the later user turn", got) + } + + c.AddAssistantToolCall(frames.ToolCall{ID: "c1", Name: "get_weather"}) + c.AddToolResult(frames.ToolResult{ID: "c1", Name: "get_weather", Content: "sunny"}) + if got := c.LastUserText(); got != "second" { + t.Errorf("LastUserText() = %q, want the spoken turn, not the tool result", got) + } +} diff --git a/processor/trim.go b/processor/trim.go new file mode 100644 index 00000000..c7a887f2 --- /dev/null +++ b/processor/trim.go @@ -0,0 +1,40 @@ +package processor + +import ( + "context" + + "github.com/nuxflix/voxigo/audio" + "github.com/nuxflix/voxigo/frames" +) + +// AudioTrim drops leading and trailing silence from every audio frame that +// passes through it. A frame that is silent end to end keeps its shape as an +// empty payload rather than being dropped, so timing and frame order stay +// intact while the samples themselves go away. +type AudioTrim struct { + *Base +} + +// NewAudioTrim builds a processor that strips silence from the edges of each +// audio frame. +func NewAudioTrim() *AudioTrim { + p := &AudioTrim{} + p.Base = New("AudioTrim", p) + return p +} + +// ProcessFrame trims audio frames and forwards everything else. +func (p *AudioTrim) ProcessFrame(ctx context.Context, f frames.Frame, dir Direction) error { + if err := p.Base.ProcessFrame(ctx, f, dir); err != nil { + return err + } + if af, ok := f.(frames.AudioFrame); ok { + data := af.AudioData() + if trimmed := audio.TrimSilence(data.Audio); trimmed != nil { + data.Audio = trimmed + } else { + data.Audio = data.Audio[:0] + } + } + return p.PushFrame(ctx, f, dir) +} diff --git a/processor/trim_test.go b/processor/trim_test.go new file mode 100644 index 00000000..fbf8d7df --- /dev/null +++ b/processor/trim_test.go @@ -0,0 +1,60 @@ +package processor_test + +import ( + "context" + "encoding/binary" + "testing" + + "github.com/nuxflix/voxigo/clock" + "github.com/nuxflix/voxigo/frames" + "github.com/nuxflix/voxigo/processor" +) + +func pcm16(samples ...int16) []byte { + b := make([]byte, len(samples)*2) + for i, s := range samples { + binary.LittleEndian.PutUint16(b[i*2:], uint16(s)) + } + return b +} + +func TestAudioTrimStripsSilenceAndPassesOtherFrames(t *testing.T) { + p := processor.NewAudioTrim() + c := newCapture() + p.Link(c) + + ctx := context.Background() + setup := processor.Setup{Clock: clock.NewSystem()} + if err := p.Setup(ctx, setup); err != nil { + t.Fatal(err) + } + if err := c.Setup(ctx, setup); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _ = p.Cleanup(ctx) + _ = c.Cleanup(ctx) + }) + + _ = p.QueueFrame(ctx, frames.NewStartFrame(), processor.Downstream) + mustReceive[*frames.StartFrame](t, c.got, "StartFrame") + + in := frames.NewInputAudioRawFrame(pcm16(0, 100, 200, 0), 16000, 1) + _ = p.QueueFrame(ctx, in, processor.Downstream) + got := mustReceive[*frames.InputAudioRawFrame](t, c.got, "InputAudioRawFrame") + if len(got.Audio) != 4 { + t.Fatalf("trimmed length = %d, want 4", len(got.Audio)) + } + s0 := int16(binary.LittleEndian.Uint16(got.Audio[0:])) + s1 := int16(binary.LittleEndian.Uint16(got.Audio[2:])) + if s0 != 100 || s1 != 200 { + t.Fatalf("trimmed samples = %d, %d, want 100, 200", s0, s1) + } + + text := frames.NewTextFrame("leave me") + _ = p.QueueFrame(ctx, text, processor.Downstream) + out := mustReceive[*frames.TextFrame](t, c.got, "TextFrame") + if out.Text != "leave me" { + t.Fatalf("Text = %q", out.Text) + } +}