rogee/go-sip
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: normalize Bailian TTS streaming audio

Browse Source
This commit is contained in:
rogee
2026-10-13 07:32:51 +08:00
parent c2db4b3a0a
commit 9bf20cfbba
13 changed files with 257 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/ai/bailian_tts.go
+4
View File
@@ -18,5 +18,9 @@ func bailianFailureSummary(err error) (string, int) {
if errors.Is(err, context.Canceled) {
return "canceled", 0
}
var decodeErr *bailianTTSAudioDecodeError
if errors.As(err, &decodeErr) {
return "audio_decode", 0
}
return "task_websocket", 0
}
internal/ai/bailian_tts_formats_test.go
+43
View File
@@ -5,6 +5,7 @@ import (
"context"
"encoding/binary"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"strings"
@@ -47,6 +48,37 @@ func TestCosyVoiceWAVTaskReturnsNormalizedPCM16(t *testing.T) {
}
}
func TestCosyVoiceWAVRepairsObservedStreamingLengthSentinels(t *testing.T) {
samples := []int16{1000, -1000, 2000, -2000}
encoded := testTTSWAV(8000, samples)
binary.LittleEndian.PutUint32(encoded[4:8], 0x7fffffbf)
binary.LittleEndian.PutUint32(encoded[40:44], 0x7fffff9b)
got, sourceRate, err := normalizeBailianTTSOutput(TTSConfig{OutputFormat: "wav", OutputSampleRate: 8000}, encoded)
if err != nil {
t.Fatal(err)
}
if sourceRate != 8000 {
t.Fatalf("WAV source sample rate = %d, want 8000", sourceRate)
}
want := audio.ResamplePCM16(testTTSPCM16(samples), 8000, internalPCM16SampleRate)
if !bytes.Equal(got, want) {
t.Fatalf("normalized streaming WAV = %v, want %v", got, want)
}
}
func TestCosyVoiceWAVDoesNotRepairUnknownLengthMismatch(t *testing.T) {
encoded := testTTSWAV(8000, []int16{1000, -1000})
binary.LittleEndian.PutUint32(encoded[4:8], uint32(len(encoded)-8+1024))
_, _, err := normalizeBailianTTSOutput(TTSConfig{OutputFormat: "wav", OutputSampleRate: 8000}, encoded)
if err == nil {
t.Fatal("unknown WAV length mismatch was repaired")
}
if !strings.Contains(err.Error(), "RIFF declared") {
t.Fatalf("WAV length diagnostics were lost: %v", err)
}
}
func TestCosyVoiceWAVDerivesRateFromContainerWhenUnset(t *testing.T) {
binding := bailianFixtureWithTTSParams(t, map[string]any{"format": "wav"})
samples := []int16{1000, -1000}
@@ -70,6 +102,17 @@ func TestNormalizeBailianWAVRejectsConfiguredSampleRateMismatch(t *testing.T) {
}
}
func TestBailianFailureSummaryClassifiesTTSDecodeErrors(t *testing.T) {
for _, format := range []string{"MP3", "WAV"} {
decodeErr := &bailianTTSAudioDecodeError{format: strings.ToLower(format), err: fmt.Errorf("decoder rejected audio")}
err := fmt.Errorf("approved TTS synthesis failed: %w", decodeErr)
category, status := bailianFailureSummary(err)
if category != "audio_decode" || status != 0 {
t.Errorf("%s decode failure summary = (%q, %d), want (audio_decode, 0)", format, category, status)
}
}
}
func runMockCosyVoiceTTS(t *testing.T, binding Binding, encoded []byte, wantParameters map[string]any) []byte {
t.Helper()
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
internal/ai/tts_audio.go
+55 -3
View File
@@ -1,9 +1,11 @@
package ai
import (
"encoding/binary"
"encoding/json"
"errors"
"fmt"
"log/slog"
"git.ipao.vip/rogee/go-sip/internal/audio"
)
@@ -12,8 +14,22 @@ const (
ttsOutputFormatPCM = "pcm"
ttsOutputFormatMP3 = "mp3"
ttsOutputFormatWAV = "wav"
bailianStreamingWAVRIFFSizeSentinel = uint32(0x7fffffbf)
bailianStreamingWAVDataSizeSentinel = uint32(0x7fffff9b)
)
type bailianTTSAudioDecodeError struct {
format string
err error
}
func (e *bailianTTSAudioDecodeError) Error() string {
return fmt.Sprintf("decode TTS %s: %v", e.format, e.err)
}
func (e *bailianTTSAudioDecodeError) Unwrap() error { return e.err }
func parseBailianTTSOutputSettings(params map[string]json.RawMessage) (string, int, error) {
format := ttsOutputFormatPCM
if rawFormat, configured := params["format"]; configured {
@@ -39,6 +55,32 @@ func parseBailianTTSOutputSettings(params map[string]json.RawMessage) (string, i
return format, rate, nil
}
// repairObservedBailianStreamingWAVLengths corrects only the exact RIFF/data
// size sentinels observed in a completed Bailian streaming WAV response. The
// existing WAV decoder still validates the resulting PCM container; all other
// length mismatches remain errors.
func repairObservedBailianStreamingWAVLengths(encoded []byte) ([]byte, bool) {
const canonicalWAVHeaderBytes = 44
if len(encoded) < canonicalWAVHeaderBytes ||
string(encoded[0:4]) != "RIFF" ||
binary.LittleEndian.Uint32(encoded[4:8]) != bailianStreamingWAVRIFFSizeSentinel ||
string(encoded[8:12]) != "WAVE" ||
string(encoded[12:16]) != "fmt " ||
binary.LittleEndian.Uint32(encoded[16:20]) != 16 ||
string(encoded[36:40]) != "data" ||
binary.LittleEndian.Uint32(encoded[40:44]) != bailianStreamingWAVDataSizeSentinel {
return nil, false
}
actualDataBytes := len(encoded) - canonicalWAVHeaderBytes
if actualDataBytes <= 0 {
return nil, false
}
corrected := append([]byte(nil), encoded...)
binary.LittleEndian.PutUint32(corrected[4:8], uint32(len(corrected)-8))
binary.LittleEndian.PutUint32(corrected[40:44], uint32(actualDataBytes))
return corrected, true
}
func normalizeBailianTTSOutput(approved TTSConfig, encoded []byte) ([]byte, int, error) {
if len(encoded) == 0 {
return nil, 0, errors.New("TTS provider returned empty audio")
@@ -65,7 +107,7 @@ func normalizeBailianTTSOutput(approved TTSConfig, encoded []byte) ([]byte, int,
case ttsOutputFormatMP3:
pcm16, sourceRate, err := audio.DecodeMP3ToPCM16Mono(encoded, internalPCM16SampleRate, maxBailianPCMBytes)
if err != nil {
return nil, 0, fmt.Errorf("decode TTS MP3: %w", err)
return nil, 0, &bailianTTSAudioDecodeError{format: ttsOutputFormatMP3, err: err}
}
if approved.OutputSampleRate != 0 && approved.OutputSampleRate != sourceRate {
return nil, 0, fmt.Errorf("TTS MP3 sample rate %d does not match requested %d Hz", sourceRate, approved.OutputSampleRate)
@@ -73,9 +115,19 @@ func normalizeBailianTTSOutput(approved TTSConfig, encoded []byte) ([]byte, int,
return pcm16, sourceRate, nil
case ttsOutputFormatWAV:
pcm16, sourceRate, err := audio.DecodeWAVToPCM16Mono(encoded, internalPCM16SampleRate, maxBailianPCMBytes)
wavInput := encoded
if repaired, ok := repairObservedBailianStreamingWAVLengths(encoded); ok {
wavInput = repaired
slog.Info("Bailian TTS streaming WAV length sentinels normalized",
"original_riff_declared_bytes", uint64(bailianStreamingWAVRIFFSizeSentinel)+8,
"riff_received_bytes", len(encoded),
"original_data_declared_bytes", bailianStreamingWAVDataSizeSentinel,
"data_available_bytes", len(encoded)-44,
)
}
pcm16, sourceRate, err := audio.DecodeWAVToPCM16Mono(wavInput, internalPCM16SampleRate, maxBailianPCMBytes)
if err != nil {
return nil, 0, fmt.Errorf("decode TTS WAV: %w", err)
return nil, 0, &bailianTTSAudioDecodeError{format: ttsOutputFormatWAV, err: err}
}
if approved.OutputSampleRate != 0 && approved.OutputSampleRate != sourceRate {
return nil, 0, fmt.Errorf("TTS WAV sample rate %d does not match requested %d Hz", sourceRate, approved.OutputSampleRate)
internal/audio/mp3.go
+3 -2
View File
@@ -4,9 +4,10 @@ import (
"bytes"
"encoding/binary"
"errors"
"fmt"
"io"
"github.com/hajimehoshi/go-mp3"
"github.com/hujm2023/go-mp3/v2"
)
// DecodeMP3ToPCM16Mono decodes MP3 to signed little-endian mono PCM16 at the
@@ -25,7 +26,7 @@ func DecodeMP3ToPCM16Mono(encoded []byte, targetSampleRate, maxOutputBytes int)
decoder, err := mp3.NewDecoder(bytes.NewReader(encoded))
if err != nil {
return nil, 0, errors.New("MP3 stream could not be decoded")
return nil, 0, fmt.Errorf("MP3 stream could not be decoded: %w", err)
}
sourceSampleRate := decoder.SampleRate()
if sourceSampleRate < 8000 || sourceSampleRate > 48000 {
internal/audio/mp3_test.go
+26
View File
@@ -1,6 +1,8 @@
package audio
import (
"bytes"
"errors"
"os"
"testing"
)
@@ -34,9 +36,33 @@ func TestDecodeMP3ToPCM16Mono(t *testing.T) {
}
}
func TestDecodeMP3ToPCM16MonoSupportsMPEG25At8k(t *testing.T) {
// Fixture is a 1-second 440-Hz mono MP3 generated with FFmpeg/libmp3lame at 8 kHz, including ID3v2 metadata.
encoded, err := os.ReadFile("testdata/sine_8k_mono_mpeg25.mp3")
if err != nil {
t.Fatal(err)
}
if !bytes.HasPrefix(encoded, []byte("ID3")) {
t.Fatal("fixture must include the provider-style ID3v2 header")
}
pcm16, sourceRate, err := DecodeMP3ToPCM16Mono(encoded, 16000, 1<<20)
if err != nil {
t.Fatal(err)
}
if sourceRate != 8000 {
t.Fatalf("source sample rate = %d, want 8000", sourceRate)
}
if len(pcm16) == 0 || len(pcm16)%2 != 0 {
t.Fatalf("decoded PCM16 length = %d, want nonempty complete samples", len(pcm16))
}
}
func TestDecodeMP3ToPCM16MonoRejectsInvalidAndOversizedAudio(t *testing.T) {
if _, _, err := DecodeMP3ToPCM16Mono([]byte("not an MP3"), 16000, 1<<20); err == nil {
t.Fatal("invalid MP3 was accepted")
} else if errors.Unwrap(err) == nil {
t.Fatalf("decoder failure cause was hidden: %v", err)
}
encoded, err := os.ReadFile("testdata/sine_16k_mono.mp3")
if err != nil {
internal/audio/testdata/sine_8k_mono_mpeg25.mp3
Vendored
BIN
View File
Binary file not shown.
internal/audio/wav.go
+51 -1
View File
@@ -158,14 +158,64 @@ func DecodeWAVToPCM16Mono(encoded []byte, targetRate, maxAudioBytes int) ([]byte
return pcm16, int(layout.sampleRate), nil
}
type wavLengthMismatchError struct {
riffDeclaredBytes uint64
receivedBytes uint64
dataChunkDeclaredBytes uint64
dataChunkAvailableBytes uint64
hasDataChunk bool
}
func (e *wavLengthMismatchError) Error() string {
if e.hasDataChunk {
return fmt.Sprintf("RIFF/WAVE length invalid or truncated: RIFF declared %d bytes, received %d bytes; data chunk declared %d bytes, available %d bytes", e.riffDeclaredBytes, e.receivedBytes, e.dataChunkDeclaredBytes, e.dataChunkAvailableBytes)
}
return fmt.Sprintf("RIFF/WAVE length invalid or truncated: RIFF declared %d bytes, received %d bytes; data chunk size unavailable", e.riffDeclaredBytes, e.receivedBytes)
}
func wavLengthMismatch(encoded []byte, riffDeclaredBytes uint64) error {
mismatch := &wavLengthMismatchError{
riffDeclaredBytes: riffDeclaredBytes,
receivedBytes: uint64(len(encoded)),
}
for offset := 12; offset+8 <= len(encoded); {
chunkID := string(encoded[offset : offset+4])
chunkSize := uint64(binary.LittleEndian.Uint32(encoded[offset+4 : offset+8]))
chunkStart := uint64(offset + 8)
if chunkID == "data" {
mismatch.hasDataChunk = true
mismatch.dataChunkDeclaredBytes = chunkSize
available := uint64(len(encoded)) - chunkStart
if available > chunkSize {
available = chunkSize
}
mismatch.dataChunkAvailableBytes = available
return mismatch
}
chunkEnd := chunkStart + chunkSize
if chunkEnd < chunkStart || chunkEnd > uint64(len(encoded)) {
return mismatch
}
paddedEnd := chunkEnd + (chunkSize & 1)
if paddedEnd > uint64(len(encoded)) {
return mismatch
}
offset = int(paddedEnd)
}
return mismatch
}
func inspectWAVPCMLayout(encoded []byte) (wavPCMLayout, error) {
if len(encoded) < 12 || string(encoded[:4]) != "RIFF" || string(encoded[8:12]) != "WAVE" {
return wavPCMLayout{}, errors.New("audio is not a RIFF/WAVE file")
}
riffEnd64 := uint64(binary.LittleEndian.Uint32(encoded[4:8])) + 8
if riffEnd64 < 12 || riffEnd64 > uint64(len(encoded)) {
if riffEnd64 < 12 {
return wavPCMLayout{}, errors.New("RIFF/WAVE length is invalid or truncated")
}
if riffEnd64 > uint64(len(encoded)) {
return wavPCMLayout{}, wavLengthMismatch(encoded, riffEnd64)
}
layout := wavPCMLayout{dataOffset: -1, riffEnd: int(riffEnd64)}
var sawFormat bool
for offset := 12; offset < layout.riffEnd; {
internal/audio/wav_test.go
+15
View File
@@ -62,6 +62,21 @@ func TestDecodeWAVRejectsMalformedOrUnsupportedAudio(t *testing.T) {
}
}
func TestDecodeWAVReportsDeclaredRIFFAndDataLengths(t *testing.T) {
encoded := testPCM16WAV(8000, 1, []int16{1000, -1000})
binary.LittleEndian.PutUint32(encoded[4:8], uint32(len(encoded)+32-8))
binary.LittleEndian.PutUint32(encoded[40:44], uint32(len(encoded)-44+16))
_, _, err := DecodeWAVToPCM16Mono(encoded, 16000, 1<<20)
if err == nil {
t.Fatal("truncated WAV was accepted")
}
const want = "RIFF/WAVE length invalid or truncated: RIFF declared 80 bytes, received 48 bytes; data chunk declared 20 bytes, available 4 bytes"
if err.Error() != want {
t.Fatalf("error = %q, want %q", err, want)
}
}
func TestDecodeWAVPCMIntegerDepthsConvertToPCM16(t *testing.T) {
tests := []struct {
name string