feat: improve voice controls and interruption

This commit is contained in:
2026-08-27 22:05:28 +08:00
parent 525ab41725
commit 50bec4fdb6
10 changed files with 589 additions and 83 deletions
+9 -5
View File
@@ -1,6 +1,6 @@
# voice_test — 实时 ASR 客服机器人测试台
Chrome 麦克风 → 实时 ASR(可选多模型)→ LLM 客服人设回答 → TTS 播报,全链路验证
Chrome 麦克风 → 实时 ASR(可选多模型)→ LLM 客服人设回答 → 下拉选择 TTS 模型、预置/复刻音色并配置语速、语调、音量和表达指令;页面还可录制并下载复刻用 WAV 样本
## 运行
@@ -39,7 +39,9 @@ export VOLC_ASR_RESOURCE_ID=volc.bigasr.sauc.duration # 2.0 用 volc.seedasr.sa
| `BAILIAN_BASE_URL` / `BAILIAN_API_KEY` / `BAILIAN_WSS_BASE_URL` | - | 阿里百炼(zshenv 已有) |
| `BAILIAN_LLM_MODEL` | `qwen3.8-flash` | 客服回答模型(OpenAI 兼容流式) |
| `BAILIAN_TTS_MODEL` | `cosyvoice-v3.5-plus` | 语音合成模型 |
| `BAILIAN_TTS_VOICE` | zshenv 音色 | cosyvoice-v3.5-plus + 复刻音色已验证可用 |
| `BAILIAN_TTS_VOICE` | zshenv 音色 | 默认音色或百炼复刻/设计生成的 `voice_id` |
| `BAILIAN_TTS_INSTRUCTION` | 空 | 默认声音表达指令,网页可覆盖 |
| `BAILIAN_VOICE_API_URL` | DashScope 声音管理接口 | 查询当前账号的复刻/设计音色下拉列表 |
| `PORT` | `:8090` | 监听地址 |
## 已验证的端到端指标(同一句“查询订单发货”,50ms/块推流,句级流式 TTS + 无 thinking
@@ -69,7 +71,8 @@ ASR 尾静音实测:语音结束→final 判定 ≈ 0.5sqwen 0.6s),`max
- `asr_volc.go` — 火山 sauc bigmodel(二进制帧协议)
- `llm.go` — OpenAI 兼容流式对话 + 客服 system prompt
- `tts.go` — CosyVoice 流式合成(PCM16/16k
- `web/index.html` — 测试页(AudioWorklet 采集 PCM16/16k、音量条、双栏转录)
- `voices.go` — TTS 模型、预置音色与百炼复刻音色列表
- `web/index.html` — 测试页(AudioWorklet 采集 PCM16/16k、WAV 样本录制、音量条、双栏转录)
## 已知简化
@@ -95,7 +98,7 @@ source ~/.zshenv
cd ~/Workspace/voice_test && go build -o voicetest . && ./voicetest
```
1. [ ] Chrome 打开 `http://localhost:8090`状态栏显示“已连接”,下拉框列出 4 个模型
1. [ ] Chrome 打开 `http://localhost:8090`ASR/TTS 模型及预置/复刻音色均显示为下拉框
2. [ ] 选 `qwen-audio-3.0-asr-flash-streaming` → 🎤 开始 → 授权麦克风
3. [ ] 说“你好,我想查一下我的订单什么时候发货”:
- [ ] 左栏灰色 partial 实时上屏,停顿后固化为 final
@@ -105,7 +108,8 @@ cd ~/Workspace/voice_test && go build -o voicetest . && ./voicetest
4. [ ] 追问“订单号是 12345”:确认多轮上下文生效(回答应引用订单号)
5. [ ] 停止 → 重新开始,切换 `fun-asr-realtime` / `fun-asr-flash-2026-06-15` 重复 3
6. [ ] 点停止后 asr badge 回 idle;服务端日志出现 `ASR started` 而非报错
7. [ ] 火山:选 `volc-bigmodel` 重复 3、4(已自动化验证通过,人工复测语音即可)
7. [ ] 点击“录制复刻样本”,录制后可试听并下载单声道 PCM WAV
8. [ ] 火山:选 `volc-bigmodel` 重复 3、4(已自动化验证通过,人工复测语音即可)
### 🧹 清理
+30 -10
View File
@@ -5,6 +5,7 @@ import (
"fmt"
"strings"
"sync"
"sync/atomic"
"github.com/gorilla/websocket"
)
@@ -15,11 +16,13 @@ type bailianASR struct {
model string
api agentCfg
conn *websocket.Conn
taskID string
out chan asrEvent
wmu sync.Mutex // 上游连接写锁(sendAudio 与 updateContext 并发)
once sync.Once
conn *websocket.Conn
taskID string
out chan asrEvent
wmu sync.Mutex // 上游连接写锁(sendAudio 与 updateContext 并发)
connOnce sync.Once
outOnce sync.Once
stopping atomic.Bool
}
func newBailianASR(model string, api agentCfg) *bailianASR {
@@ -33,6 +36,12 @@ func (a *bailianASR) start(ctx context.Context) error {
return fmt.Errorf("连接百炼 ASR: %w", err)
}
a.conn = conn
started := false
defer func() {
if !started {
conn.Close()
}
}()
a.taskID = uuid()
err = a.writeJSON(map[string]any{
"header": map[string]string{"action": "run-task", "task_id": a.taskID, "streaming": "duplex"},
@@ -59,6 +68,7 @@ func (a *bailianASR) start(ctx context.Context) error {
if ev.Header.Event != "task-started" {
return fmt.Errorf("启动任务失败: %s %s", ev.Header.ErrorCode, ev.Header.ErrorMessage)
}
started = true
go a.readLoop()
return nil
}
@@ -82,7 +92,9 @@ func (a *bailianASR) readLoop() {
} `json:"payload"`
}
if err := a.conn.ReadJSON(&ev); err != nil {
a.emit(asrEvent{Typ: "error", Error: fmt.Sprintf("连接关闭: %v", err)})
if !a.stopping.Load() {
a.emit(asrEvent{Typ: "error", Error: fmt.Sprintf("连接关闭: %v", err)})
}
return
}
switch ev.Header.Event {
@@ -146,13 +158,21 @@ func (a *bailianASR) finish() error {
}
func (a *bailianASR) events() <-chan asrEvent { return a.out }
func (a *bailianASR) close() { a.closeUpstream() }
func (a *bailianASR) close() {
a.stopping.Store(true)
a.closeConn()
}
func (a *bailianASR) closeUpstream() {
a.once.Do(func() {
func (a *bailianASR) closeConn() {
a.connOnce.Do(func() {
if a.conn != nil {
a.conn.Close()
}
close(a.out)
})
}
// 只有 readLoop 退出后才能关闭事件通道,避免 close() 与 emit() 并发导致 panic。
func (a *bailianASR) closeUpstream() {
a.closeConn()
a.outOnce.Do(func() { close(a.out) })
}
+36 -16
View File
@@ -10,6 +10,7 @@ import (
"io"
"net/http"
"sync"
"sync/atomic"
"github.com/gorilla/websocket"
)
@@ -21,9 +22,9 @@ const (
volcURL = "wss://openspeech.bytedance.com/api/v3/sauc/bigmodel"
msgFullClient = 0x1
msgAudioOnly = 0x2
msgFullServer = 0x9
msgError = 0xF
msgAudioOnly = 0x2
msgFullServer = 0x9
msgError = 0xF
flagNoSeq = 0x0 // full client request 用
flagLast = 0x2 // 客户端最后一包音频(负包,无序号)
@@ -34,12 +35,14 @@ const (
)
type volcASR struct {
cfg agentCfg
conn *websocket.Conn
out chan asrEvent
wmu sync.Mutex
once sync.Once
emitted map[int]bool // 已作为 final 发出的 utterance 下标
cfg agentCfg
conn *websocket.Conn
out chan asrEvent
wmu sync.Mutex
connOnce sync.Once
outOnce sync.Once
stopping atomic.Bool
emitted map[int]bool // 已作为 final 发出的 utterance 下标
}
func newVolcASR(cfg agentCfg) *volcASR {
@@ -70,10 +73,16 @@ func (v *volcASR) start(ctx context.Context) error {
return fmt.Errorf("连接火山 ASR (401/403 多为凭证或资源未开通, logid=%s): %w", logid, err)
}
v.conn = conn
started := false
defer func() {
if !started {
conn.Close()
}
}()
req := map[string]any{
"user": map[string]any{"uid": "voice-test-agent"},
"audio": map[string]any{"format": "pcm", "rate": 16000, "bits": 16, "channel": 1},
"user": map[string]any{"uid": "voice-test-agent"},
"audio": map[string]any{"format": "pcm", "rate": 16000, "bits": 16, "channel": 1},
"request": map[string]any{
"model_name": "bigmodel",
"enable_itn": true,
@@ -86,6 +95,7 @@ func (v *volcASR) start(ctx context.Context) error {
if err := v.conn.WriteMessage(websocket.BinaryMessage, volcFrame(msgFullClient, flagNoSeq, serJSON, compNone, body)); err != nil {
return fmt.Errorf("发送 full client request: %w", err)
}
started = true
go v.readLoop()
return nil
}
@@ -107,7 +117,9 @@ func (v *volcASR) readLoop() {
for {
mt, data, err := v.conn.ReadMessage()
if err != nil {
v.emit(asrEvent{Typ: "error", Error: fmt.Sprintf("连接关闭: %v", err)})
if !v.stopping.Load() {
v.emit(asrEvent{Typ: "error", Error: fmt.Sprintf("连接关闭: %v", err)})
}
return
}
if mt != websocket.BinaryMessage || len(data) < 8 {
@@ -205,13 +217,21 @@ func (v *volcASR) finish() error {
}
func (v *volcASR) events() <-chan asrEvent { return v.out }
func (v *volcASR) close() { v.closeUpstream() }
func (v *volcASR) close() {
v.stopping.Store(true)
v.closeConn()
}
func (v *volcASR) closeUpstream() {
v.once.Do(func() {
func (v *volcASR) closeConn() {
v.connOnce.Do(func() {
if v.conn != nil {
v.conn.Close()
}
close(v.out)
})
}
// 只有 readLoop 退出后才能关闭事件通道,避免 close() 与 emit() 并发导致 panic。
func (v *volcASR) closeUpstream() {
v.closeConn()
v.outOnce.Do(func() { close(v.out) })
}
+4
View File
@@ -12,6 +12,8 @@ type agentCfg struct {
BailianKey string
TTSModel string
TTSVoice string
TTSInstruction string
VoiceAPIURL string
LLMModel string
VolcAppKey string // 新版控制台:单凭证
VolcAppID string // 旧版控制台:APP ID + Access Token 两件套
@@ -26,6 +28,8 @@ func loadCfg() agentCfg {
BailianKey: getenv("BAILIAN_API_KEY", ""),
TTSModel: getenv("BAILIAN_TTS_MODEL", "cosyvoice-v3.5-plus"),
TTSVoice: getenv("BAILIAN_TTS_VOICE", ""),
TTSInstruction: getenv("BAILIAN_TTS_INSTRUCTION", ""),
VoiceAPIURL: getenv("BAILIAN_VOICE_API_URL", "https://dashscope.aliyuncs.com/api/v1/services/audio/tts/customization"),
LLMModel: getenv("BAILIAN_LLM_MODEL", "qwen3.8-flash"),
VolcAppKey: getenv("VOLC_ASR_APP_KEY", ""),
VolcAppID: getenv("VOLC_ASR_APP_ID", ""),
+2
View File
@@ -29,6 +29,7 @@ func llmChat(ctx context.Context, cfg agentCfg, history []chatMsg) (<-chan strin
req, err := http.NewRequestWithContext(ctx, "POST", cfg.BailianBaseURL+"/chat/completions",
bytes.NewReader(body))
if err != nil {
close(textCh)
errCh <- err
return textCh, errCh
}
@@ -37,6 +38,7 @@ func llmChat(ctx context.Context, cfg agentCfg, history []chatMsg) (<-chan strin
resp, err := http.DefaultClient.Do(req)
if err != nil {
close(textCh)
errCh <- err
return textCh, errCh
}
+105 -31
View File
@@ -13,7 +13,7 @@ import (
)
var upgrader = websocket.Upgrader{
CheckOrigin: func(*http.Request) bool { return true }, // 本地测试页,放开来源
CheckOrigin: func(*http.Request) bool { return true }, // 本地测试页,放开来源
ReadBufferSize: 16384,
WriteBufferSize: 16384,
}
@@ -30,9 +30,10 @@ func asrModels(cfg agentCfg) []map[string]any {
}
type clientMsg struct {
Type string `json:"type"`
ASR string `json:"asr"`
Mode string `json:"mode"`
Type string `json:"type"`
ASR string `json:"asr"`
Mode string `json:"mode"`
TTS *ttsCfg `json:"tts"`
}
type chatMsg struct {
@@ -41,13 +42,16 @@ type chatMsg struct {
}
type session struct {
cfg agentCfg
conn *websocket.Conn
wmu sync.Mutex // 客户端连接写锁
mu sync.Mutex // provider / history 状态锁
prov asrProvider
hist []chatMsg
cancel context.CancelFunc
cfg agentCfg
conn *websocket.Conn
wmu sync.Mutex // 客户端连接写锁
mu sync.Mutex // provider / history 状态锁
prov asrProvider
tts ttsCfg
hist []chatMsg
cancel context.CancelFunc
turnCancel context.CancelFunc
turnID uint64
}
func (s *session) sendJSON(v any) {
@@ -73,12 +77,29 @@ func main() {
http.HandleFunc("/api/models", func(w http.ResponseWriter, r *http.Request) {
json.NewEncoder(w).Encode(asrModels(cfg))
})
http.HandleFunc("/api/config", func(w http.ResponseWriter, r *http.Request) {
ctx, cancel := context.WithTimeout(r.Context(), 3*time.Second)
defer cancel()
cloned, voiceErr := listClonedVoices(ctx, cfg)
found := false
for _, v := range cloned {
found = found || v.ID == cfg.TTSVoice
}
if cfg.TTSVoice != "" && !found {
cloned = append([]voiceOption{{ID: cfg.TTSVoice, Label: cfg.TTSVoice + "(当前配置)"}}, cloned...)
}
payload := map[string]any{"tts": defaultTTS(cfg), "ttsModels": ttsModels(), "systemVoices": systemVoices(), "clonedVoices": cloned}
if voiceErr != nil {
payload["voiceError"] = voiceErr.Error()
}
json.NewEncoder(w).Encode(payload)
})
http.HandleFunc("/agent", func(w http.ResponseWriter, r *http.Request) {
conn, err := upgrader.Upgrade(w, r, nil)
if err != nil {
return
}
s := &session{cfg: cfg, conn: conn}
s := &session{cfg: cfg, conn: conn, tts: defaultTTS(cfg)}
defer conn.Close()
s.readLoop()
})
@@ -102,14 +123,9 @@ func (s *session) readLoop() {
}
switch m.Type {
case "start":
s.startASR(m.ASR)
s.startASR(m.ASR, m.TTS)
case "stop":
s.mu.Lock()
p := s.prov
s.mu.Unlock()
if p != nil {
p.finish()
}
s.stopASR()
}
case websocket.BinaryMessage:
s.mu.Lock()
@@ -122,7 +138,12 @@ func (s *session) readLoop() {
}
}
func (s *session) startASR(id string) {
func (s *session) startASR(id string, requestedTTS *ttsCfg) {
tts, err := normalizeTTS(requestedTTS, s.cfg)
if err != nil {
s.sendJSON(map[string]any{"type": "error", "error": err.Error()})
return
}
s.stopASR()
ctx, cancel := context.WithCancel(context.Background())
var p asrProvider
@@ -138,13 +159,14 @@ func (s *session) startASR(id string) {
return
}
s.mu.Lock()
s.prov, s.cancel = p, cancel
s.prov, s.cancel, s.tts = p, cancel, tts
s.mu.Unlock()
go s.eventPump(p)
log.Printf("ASR started: %s", id)
}
func (s *session) stopASR() {
s.interruptTurn()
s.mu.Lock()
p, cancel := s.prov, s.cancel
s.prov, s.cancel = nil, nil
@@ -158,7 +180,38 @@ func (s *session) stopASR() {
}
}
// eventPump: ASR 事件 → 转发给浏览器;final 句触发客服回答(LLM 流式 → TTS 流式)。
func (s *session) interruptTurn() {
s.mu.Lock()
cancel := s.turnCancel
s.turnCancel = nil
s.mu.Unlock()
if cancel != nil {
cancel()
if s.conn != nil {
s.sendJSON(map[string]string{"type": "interrupt"})
}
}
}
func (s *session) startTurn() (context.Context, context.CancelFunc, uint64) {
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
s.mu.Lock()
s.turnID++
id := s.turnID
s.turnCancel = cancel
s.mu.Unlock()
return ctx, cancel, id
}
func (s *session) finishTurn(id uint64) {
s.mu.Lock()
if s.turnID == id {
s.turnCancel = nil
}
s.mu.Unlock()
}
// eventPump: 新语音一出现即打断当前回答;final 句异步触发下一轮,保持 ASR 事件可继续消费。
func (s *session) eventPump(p asrProvider) {
defer s.sendJSON(map[string]string{"type": "asr-stopped"})
for ev := range p.events() {
@@ -166,21 +219,24 @@ func (s *session) eventPump(p asrProvider) {
case "error":
s.sendJSON(map[string]any{"type": "error", "error": ev.Error, "code": ev.Code})
case "partial", "final":
s.interruptTurn()
s.sendJSON(map[string]any{"type": ev.Typ, "text": ev.Text})
if ev.Typ == "final" {
s.agentTurn(p, ev.Text)
ctx, cancel, id := s.startTurn()
go s.agentTurn(ctx, cancel, id, p, ev.Text)
}
}
}
}
// agentTurn: final 句 → LLM 流式出字(逐段回传)→ 边生成边按句喂 TTS → 音频流回传。
func (s *session) agentTurn(p asrProvider, userText string) {
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
func (s *session) agentTurn(ctx context.Context, cancel context.CancelFunc, id uint64, p asrProvider, userText string) {
defer cancel()
defer s.finishTurn(id)
s.mu.Lock()
hist := append([]chatMsg(nil), s.hist...)
tts := s.tts
s.mu.Unlock()
hist = append(hist, chatMsg{Role: "user", Content: userText})
@@ -190,7 +246,7 @@ func (s *session) agentTurn(p asrProvider, userText string) {
go func() {
defer close(ttsDone)
for sen := range sentences {
if err := ttsSpeak(ctx, s.cfg, sen, s.sendAudio); err != nil {
if err := ttsSpeak(ctx, s.cfg, tts, sen, s.sendAudio); err != nil && ctx.Err() == nil {
s.sendJSON(map[string]any{"type": "error", "error": err.Error()})
}
}
@@ -199,27 +255,45 @@ func (s *session) agentTurn(p asrProvider, userText string) {
textCh, errCh := llmChat(ctx, s.cfg, hist)
var full []byte
var pending []rune
flush := func() {
flush := func() bool {
if sen := strings.TrimSpace(string(pending)); sen != "" {
sentences <- sen
select {
case sentences <- sen:
case <-ctx.Done():
return false
}
}
pending = pending[:0]
return true
}
for seg := range textCh {
if ctx.Err() != nil {
continue // drain producer so cancellation cannot leave it blocked on textCh
}
full = append(full, seg...)
s.sendJSON(map[string]any{"type": "reply-delta", "text": seg})
for _, r := range seg {
pending = append(pending, r)
// 句末标点即切句;超长无标点时退到逗号,避免一句卡住整段
if strings.ContainsRune("。!?!?;\n", r) || (len(pending) >= 100 && r == '') {
flush()
if !flush() {
break
}
}
}
}
flush()
if ctx.Err() == nil {
flush()
}
close(sentences)
if err := <-errCh; err != nil {
s.sendJSON(map[string]any{"type": "error", "error": "LLM: " + err.Error()})
if ctx.Err() == nil {
s.sendJSON(map[string]any{"type": "error", "error": "LLM: " + err.Error()})
}
<-ttsDone
return
}
if ctx.Err() != nil {
<-ttsDone
return
}
+57 -10
View File
@@ -4,14 +4,52 @@ import (
"context"
"encoding/json"
"fmt"
"strings"
"sync"
"time"
"github.com/gorilla/websocket"
)
type ttsCfg struct {
Model string `json:"model"`
Voice string `json:"voice"`
Instruction string `json:"instruction"`
Rate float64 `json:"rate"`
Pitch float64 `json:"pitch"`
Volume int `json:"volume"`
}
func defaultTTS(cfg agentCfg) ttsCfg {
return ttsCfg{Model: cfg.TTSModel, Voice: cfg.TTSVoice, Instruction: cfg.TTSInstruction, Rate: 1, Pitch: 1, Volume: 50}
}
func normalizeTTS(in *ttsCfg, cfg agentCfg) (ttsCfg, error) {
if in == nil {
return defaultTTS(cfg), nil
}
t := *in
t.Model, t.Voice, t.Instruction = strings.TrimSpace(t.Model), strings.TrimSpace(t.Voice), strings.TrimSpace(t.Instruction)
if t.Model == "" {
t.Model = cfg.TTSModel
}
if t.Voice == "" {
t.Voice = cfg.TTSVoice
}
if t.Voice == "" {
return t, fmt.Errorf("请填写音色或复刻音色 ID")
}
if len(t.Model) > 100 || len(t.Voice) > 200 || len([]rune(t.Instruction)) > 100 {
return t, fmt.Errorf("TTS 配置文本过长")
}
if t.Rate < 0.5 || t.Rate > 2 || t.Pitch < 0.5 || t.Pitch > 2 || t.Volume < 0 || t.Volume > 100 {
return t, fmt.Errorf("TTS 参数超出范围:语速/语调 0.5~2.0,音量 0~100")
}
return t, nil
}
// ttsSpeak: 百炼 CosyVoice 流式合成,音频块实时回调 onAudio。
func ttsSpeak(ctx context.Context, cfg agentCfg, text string, onAudio func([]byte)) error {
func ttsSpeak(ctx context.Context, cfg agentCfg, tts ttsCfg, text string, onAudio func([]byte)) error {
cctx, cancel := context.WithTimeout(ctx, 60*time.Second)
defer cancel()
conn, _, err := websocket.DefaultDialer.DialContext(cctx, cfg.BailianWssBaseURL,
@@ -22,16 +60,21 @@ func ttsSpeak(ctx context.Context, cfg agentCfg, text string, onAudio func([]byt
defer conn.Close()
taskID := uuid()
parameters := map[string]any{
"text_type": "PlainText", "voice": tts.Voice,
"format": "pcm", "sample_rate": 16000,
"volume": tts.Volume, "rate": tts.Rate, "pitch": tts.Pitch,
}
if tts.Instruction != "" {
parameters["instruction"] = tts.Instruction
}
err = conn.WriteJSON(map[string]any{
"header": map[string]string{"action": "run-task", "task_id": taskID, "streaming": "duplex"},
"payload": map[string]any{
"task_group": "audio", "task": "tts", "function": "SpeechSynthesizer",
"model": cfg.TTSModel,
"parameters": map[string]any{
"text_type": "PlainText", "voice": cfg.TTSVoice,
"format": "pcm", "sample_rate": 16000,
},
"input": map[string]any{},
"model": tts.Model,
"parameters": parameters,
"input": map[string]any{},
},
})
if err != nil {
@@ -40,7 +83,6 @@ func ttsSpeak(ctx context.Context, cfg agentCfg, text string, onAudio func([]byt
var once sync.Once
done := make(chan error, 1)
var buf []json.RawMessage // task-started / task-finished 事件
go func() {
for {
mt, data, err := conn.ReadMessage()
@@ -66,7 +108,6 @@ func ttsSpeak(ctx context.Context, cfg agentCfg, text string, onAudio func([]byt
}
switch ev.Header.Event {
case "task-started":
buf = append(buf, json.RawMessage("{}"))
once.Do(func() {
// task-started 到达后才允许发文本
if err := conn.WriteJSON(map[string]any{
@@ -98,5 +139,11 @@ func ttsSpeak(ctx context.Context, cfg agentCfg, text string, onAudio func([]byt
}
}
}()
return <-done
select {
case err := <-done:
return err
case <-cctx.Done():
conn.Close()
return cctx.Err()
}
}
+80
View File
@@ -0,0 +1,80 @@
package main
import (
"context"
"net/http"
"net/http/httptest"
"testing"
)
func TestNormalizeTTS(t *testing.T) {
cfg := agentCfg{TTSModel: "cosyvoice-v3.5-plus", TTSVoice: "clone-id"}
got, err := normalizeTTS(nil, cfg)
if err != nil || got.Voice != "clone-id" || got.Rate != 1 || got.Pitch != 1 || got.Volume != 50 {
t.Fatalf("defaults: got=%+v err=%v", got, err)
}
bad := ttsCfg{Model: "m", Voice: "v", Rate: 2.1, Pitch: 1, Volume: 50}
if _, err := normalizeTTS(&bad, cfg); err == nil {
t.Fatal("expected out-of-range rate to fail")
}
}
func TestASRCloseLeavesEventsForReader(t *testing.T) {
t.Run("bailian", func(t *testing.T) {
a := newBailianASR("model", agentCfg{})
a.close()
a.emit(asrEvent{Typ: "final", Text: "ok"})
if ev := <-a.events(); ev.Text != "ok" {
t.Fatalf("event=%+v", ev)
}
a.closeUpstream()
})
t.Run("volc", func(t *testing.T) {
v := newVolcASR(agentCfg{})
v.close()
v.emit(asrEvent{Typ: "final", Text: "ok"})
if ev := <-v.events(); ev.Text != "ok" {
t.Fatalf("event=%+v", ev)
}
v.closeUpstream()
})
}
func TestLLMRequestErrorClosesStream(t *testing.T) {
text, errs := llmChat(context.Background(), agentCfg{BailianBaseURL: "://"}, nil)
if _, open := <-text; open {
t.Fatal("text stream left open after request error")
}
if err := <-errs; err == nil {
t.Fatal("expected request error")
}
}
func TestInterruptTurn(t *testing.T) {
s := &session{}
ctx, cancel, _ := s.startTurn()
defer cancel()
s.interruptTurn()
select {
case <-ctx.Done():
default:
t.Fatal("active turn was not canceled")
}
}
func TestListClonedVoices(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("Authorization") != "Bearer key" {
t.Fatal("missing authorization")
}
w.Write([]byte(`{"output":{"voice_list":[{"voice_id":"clone-ok","status":"OK"},{"voice_id":"clone-building","status":"DEPLOYING"}]}}`))
}))
defer server.Close()
voices, err := listClonedVoices(context.Background(), agentCfg{BailianKey: "key", VoiceAPIURL: server.URL})
if err != nil || len(voices) != 1 || voices[0].ID != "clone-ok" {
t.Fatalf("voices=%+v err=%v", voices, err)
}
}
+76
View File
@@ -0,0 +1,76 @@
package main
import (
"bytes"
"context"
"encoding/json"
"fmt"
"net/http"
)
type voiceOption struct {
ID string `json:"id"`
Label string `json:"label"`
}
func ttsModels() []voiceOption {
return []voiceOption{
{ID: "cosyvoice-v3.5-plus", Label: "CosyVoice v3.5 Plus(复刻/设计音色)"},
{ID: "cosyvoice-v3.5-flash", Label: "CosyVoice v3.5 Flash(低延迟)"},
{ID: "cosyvoice-v3-plus", Label: "CosyVoice v3 Plus"},
{ID: "cosyvoice-v3-flash", Label: "CosyVoice v3 Flash"},
{ID: "cosyvoice-v2", Label: "CosyVoice v2"},
}
}
func systemVoices() map[string][]voiceOption {
v3 := []voiceOption{{ID: "longanhuan", Label: "龙安欢|欢脱元气女"}, {ID: "longanyang", Label: "龙安洋|阳光男声"}}
return map[string][]voiceOption{
"cosyvoice-v3-plus": v3,
"cosyvoice-v3-flash": v3,
"cosyvoice-v2": {
{ID: "longxiaochun_v2", Label: "龙小淳|温柔女声"},
{ID: "longwan_v2", Label: "龙婉|普通话女声"},
{ID: "loongbella_v2", Label: "Bella 2.0|新闻女声"},
},
}
}
func listClonedVoices(ctx context.Context, cfg agentCfg) ([]voiceOption, error) {
body, _ := json.Marshal(map[string]any{
"model": "voice-enrollment",
"input": map[string]any{"action": "list_voice", "page_index": 0, "page_size": 100},
})
req, err := http.NewRequestWithContext(ctx, http.MethodPost, cfg.VoiceAPIURL, bytes.NewReader(body))
if err != nil {
return nil, err
}
req.Header.Set("Authorization", "Bearer "+cfg.BailianKey)
req.Header.Set("Content-Type", "application/json")
resp, err := http.DefaultClient.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode/100 != 2 {
return nil, fmt.Errorf("查询复刻音色失败: HTTP %d", resp.StatusCode)
}
var result struct {
Output struct {
Voices []struct {
ID string `json:"voice_id"`
Status string `json:"status"`
} `json:"voice_list"`
} `json:"output"`
}
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
return nil, err
}
voices := make([]voiceOption, 0, len(result.Output.Voices))
for _, v := range result.Output.Voices {
if v.ID != "" && (v.Status == "" || v.Status == "OK" || v.Status == "SUCCESS") {
voices = append(voices, voiceOption{ID: v.ID, Label: v.ID})
}
}
return voices, nil
}
+190 -11
View File
@@ -7,24 +7,40 @@
<style>
:root { --bg:#0f1117; --panel:#1a1d27; --line:#2a2e3d; --fg:#e6e8f0; --dim:#8b90a5; --acc:#4f7cff; --ok:#3fbf6f; --warn:#e0a23c; --err:#e05c5c; }
* { box-sizing:border-box; margin:0; padding:0; }
body { background:var(--bg); color:var(--fg); font:14px/1.6 "PingFang SC","Microsoft YaHei",system-ui,sans-serif; min-height:100vh; display:flex; flex-direction:column; }
html, body { height:100%; overflow:hidden; }
body { background:var(--bg); color:var(--fg); font:14px/1.6 "PingFang SC","Microsoft YaHei",system-ui,sans-serif; display:flex; flex-direction:column; }
header { padding:14px 22px; border-bottom:1px solid var(--line); display:flex; align-items:center; gap:14px; flex-wrap:wrap; }
header h1 { font-size:16px; font-weight:600; }
header .sel { display:flex; align-items:center; gap:8px; margin-left:auto; }
select, button { background:var(--panel); color:var(--fg); border:1px solid var(--line); border-radius:8px; padding:8px 14px; font-size:14px; cursor:pointer; }
select, button, input { background:var(--panel); color:var(--fg); border:1px solid var(--line); border-radius:8px; padding:8px 12px; font-size:14px; }
select, button { cursor:pointer; }
select { min-width:280px; }
.config { padding:10px 22px; border-bottom:1px solid var(--line); background:#141720; max-height:45vh; overflow-y:auto; flex:none; }
.config summary { cursor:pointer; color:var(--dim); user-select:none; }
.config-grid { display:grid; grid-template-columns:repeat(5,minmax(130px,1fr)); gap:10px 14px; margin-top:10px; }
.config-grid label { display:flex; flex-direction:column; gap:4px; color:var(--dim); font-size:12px; }
.config-grid label.wide { grid-column:span 2; }
.config-grid input[type="range"] { padding:0; accent-color:var(--acc); }
.config-grid select { min-width:0; width:100%; }
.range-title { display:flex; justify-content:space-between; }
.hint { color:var(--dim); font-size:11px; margin-top:6px; }
.recorder { display:flex; align-items:center; gap:10px; flex-wrap:wrap; margin-top:10px; padding-top:10px; border-top:1px solid var(--line); }
#record.on { background:var(--err); color:#fff; }
#recording-audio { height:34px; max-width:320px; }
#recording-download { color:#8fb0ff; text-decoration:none; }
@media (max-width:900px){ .config-grid{grid-template-columns:1fr 1fr;} .config-grid label.wide{grid-column:span 2;} }
button:hover:not(:disabled) { border-color:var(--acc); }
button:disabled { opacity:.4; cursor:not-allowed; }
#mic { border-color:transparent; color:#fff; font-weight:600; min-width:96px; }
#mic.on { background:var(--err); animation:pulse 1.2s infinite; }
#mic.off { background:var(--acc); }
@keyframes pulse { 0%,100%{box-shadow:0 0 0 0 rgba(224,92,92,.5)} 50%{box-shadow:0 0 0 10px rgba(224,92,92,0)} }
main { flex:1; display:grid; grid-template-columns:1fr 1fr; gap:14px; padding:14px 22px; max-width:1200px; width:100%; margin:0 auto; }
@media (max-width:860px){ main{grid-template-columns:1fr;} }
.panel { background:var(--panel); border:1px solid var(--line); border-radius:12px; display:flex; flex-direction:column; min-height:320px; }
main { flex:1; min-height:0; overflow:hidden; display:grid; grid-template-columns:1fr 1fr; grid-template-rows:minmax(0,1fr); gap:14px; padding:14px 22px; max-width:1200px; width:100%; margin:0 auto; }
@media (max-width:860px){ main{grid-template-columns:1fr; grid-template-rows:repeat(2,minmax(0,1fr));} }
.panel { background:var(--panel); border:1px solid var(--line); border-radius:12px; display:flex; flex-direction:column; min-height:0; overflow:hidden; }
.panel h2 { font-size:13px; color:var(--dim); padding:10px 14px; border-bottom:1px solid var(--line); font-weight:500; display:flex; justify-content:space-between; }
.panel h2 .badge { font-size:11px; padding:1px 8px; border-radius:6px; background:#262b3b; }
.log { flex:1; overflow-y:auto; padding:12px 14px; display:flex; flex-direction:column; gap:10px; }
.log { flex:1; min-height:0; overflow-y:auto; overscroll-behavior:contain; padding:12px 14px; display:flex; flex-direction:column; gap:10px; }
.msg { padding:8px 12px; border-radius:10px; max-width:88%; white-space:pre-wrap; word-break:break-all; }
.msg .who { font-size:11px; color:var(--dim); margin-bottom:2px; }
.msg.user { align-self:flex-end; background:#25315e; border-top-right-radius:3px; }
@@ -47,6 +63,26 @@
<button id="mic" class="off">🎤 开始</button>
</div>
</header>
<details class="config" open>
<summary>TTS 声音配置</summary>
<div class="config-grid">
<label>TTS 模型<select id="tts-model"></select></label>
<label>预置音色<select id="tts-system-voice"></select></label>
<label class="wide">复刻 / 设计音色<select id="tts-cloned-voice"><option value="">请选择复刻音色</option></select></label>
<label><span class="range-title">语速 <b id="rate-value">1.0×</b></span><input id="tts-rate" type="range" min="0.5" max="2" step="0.1" value="1"></label>
<label><span class="range-title">语调 <b id="pitch-value">1.0×</b></span><input id="tts-pitch" type="range" min="0.5" max="2" step="0.1" value="1"></label>
<label><span class="range-title">音量 <b id="volume-value">50</b></span><input id="tts-volume" type="range" min="0" max="100" step="1" value="50"></label>
<label class="wide">声音表达 / 模仿描述<input id="tts-instruction" maxlength="100" placeholder="如:温柔沉稳、略带微笑、语速自然"></label>
</div>
<div class="hint">v3.5 仅支持复刻/设计音色;复刻音色从当前百炼账号自动读取。<span id="voice-status"></span></div>
<div class="recorder">
<button id="record">⏺ 录制复刻样本</button>
<b id="record-time">00:00</b>
<audio id="recording-audio" controls hidden></audio>
<a id="recording-download" download="voice-sample.wav" hidden>下载 WAV</a>
<span class="hint">建议安静环境录制 5~20 秒连续清晰人声,最长 30 秒。</span>
</div>
</details>
<main>
<div class="panel">
<h2>识别(ASR <span class="badge" id="asr-badge">idle</span></h2>
@@ -62,7 +98,7 @@
<span>延迟: <b id="lat">-</b></span>
<span id="vu"><i></i><i></i><i></i><i></i><i></i></span>
</div>
<div class="footer-tip">浏览器采集 PCM16/16k 单声道,实时上行;每句判定结束后自动由客服 LLM 流式回答并 TTS 播报。Chrome 要求 localhost 或 HTTPS 才能使用麦克风。</div>
<div class="footer-tip">浏览器采集 PCM16/16k 单声道,实时上行;每句判定结束后自动由客服 LLM 流式回答并按当前音色配置播报。Chrome 要求 localhost 或 HTTPS 才能使用麦克风。</div>
<script>
"use strict";
@@ -91,14 +127,129 @@ fetch("/api/models").then(r => r.json()).then(models => {
}
});
let systemVoices = {};
fetch("/api/config").then(r => r.json()).then(({tts, ttsModels, systemVoices: voices, clonedVoices, voiceError}) => {
systemVoices = voices;
fillSelect($("tts-model"), ttsModels);
$("tts-model").value = tts.model;
fillSelect($("tts-cloned-voice"), clonedVoices, "请选择复刻/设计音色");
const isSystem = renderSystemVoices(tts.voice);
if (!isSystem) $("tts-cloned-voice").value = tts.voice;
$("tts-instruction").value = tts.instruction;
$("tts-rate").value = tts.rate;
$("tts-pitch").value = tts.pitch;
$("tts-volume").value = tts.volume;
if (voiceError) $("voice-status").textContent = `(读取失败:${voiceError}`;
updateRangeLabels();
});
function fillSelect(select, options, placeholder = "") {
select.replaceChildren();
if (placeholder) select.add(new Option(placeholder, ""));
for (const o of options || []) select.add(new Option(o.label, o.id));
}
function renderSystemVoices(selected = "") {
const options = systemVoices[$("tts-model").value] || [];
fillSelect($("tts-system-voice"), options, options.length ? "请选择预置音色" : "该模型无预置音色");
if (options.some(v => v.id === selected)) {
$("tts-system-voice").value = selected;
return true;
}
return false;
}
$("tts-model").onchange = () => renderSystemVoices();
$("tts-system-voice").onchange = () => { if ($("tts-system-voice").value) $("tts-cloned-voice").value = ""; };
$("tts-cloned-voice").onchange = () => { if ($("tts-cloned-voice").value) $("tts-system-voice").value = ""; };
function updateRangeLabels() {
$("rate-value").textContent = Number($("tts-rate").value).toFixed(1) + "×";
$("pitch-value").textContent = Number($("tts-pitch").value).toFixed(1) + "×";
$("volume-value").textContent = $("tts-volume").value;
}
["tts-rate", "tts-pitch", "tts-volume"].forEach(id => $(id).oninput = updateRangeLabels);
function ttsConfig() {
return {
model: $("tts-model").value,
voice: $("tts-cloned-voice").value || $("tts-system-voice").value,
instruction: $("tts-instruction").value,
rate: Number($("tts-rate").value),
pitch: Number($("tts-pitch").value),
volume: Number($("tts-volume").value)
};
}
// ---------- 复刻样本录制(单声道 PCM WAV ----------
let recordStream = null, recordCtx = null, recordNode = null, recordChunks = [], recordStarted = 0, recordTimer = null, recordURL = "";
$("record").onclick = async () => {
if (recordStream) { stopRecording(); return; }
if ($("mic").classList.contains("on")) {
log($("chat-log"), "⚠ 请先停止 ASR 会话", "err");
return;
}
try {
recordStream = await navigator.mediaDevices.getUserMedia({audio:{channelCount:1, echoCancellation:false, noiseSuppression:false}});
recordCtx = new AudioContext();
const source = recordCtx.createMediaStreamSource(recordStream);
recordNode = recordCtx.createScriptProcessor(4096, 1, 1);
recordNode.onaudioprocess = e => recordChunks.push(new Float32Array(e.inputBuffer.getChannelData(0)));
const silent = recordCtx.createGain(); silent.gain.value = 0;
source.connect(recordNode); recordNode.connect(silent); silent.connect(recordCtx.destination);
recordStarted = Date.now();
$("record").className = "on"; $("record").textContent = "⏹ 停止录制";
recordTimer = setInterval(() => {
const seconds = Math.floor((Date.now() - recordStarted) / 1000);
$("record-time").textContent = `00:${String(seconds).padStart(2, "0")}`;
if (seconds >= 30) stopRecording();
}, 250);
} catch (e) {
recordStream = null;
log($("chat-log"), `⚠ 录音失败: ${esc(e.message)}`, "err");
}
};
function stopRecording() {
clearInterval(recordTimer);
const sampleRate = recordCtx.sampleRate;
recordNode.disconnect();
recordStream.getTracks().forEach(t => t.stop());
recordCtx.close();
recordStream = recordCtx = recordNode = null;
$("record").className = ""; $("record").textContent = "⏺ 重新录制";
if (recordURL) URL.revokeObjectURL(recordURL);
recordURL = URL.createObjectURL(wavBlob(recordChunks, sampleRate));
recordChunks = [];
$("recording-audio").src = recordURL; $("recording-audio").hidden = false;
$("recording-download").href = recordURL; $("recording-download").hidden = false;
}
function wavBlob(chunks, sampleRate) {
const samples = chunks.reduce((n, c) => n + c.length, 0);
const buffer = new ArrayBuffer(44 + samples * 2), view = new DataView(buffer);
const text = (offset, value) => [...value].forEach((c, i) => view.setUint8(offset + i, c.charCodeAt(0)));
text(0, "RIFF"); view.setUint32(4, 36 + samples * 2, true); text(8, "WAVEfmt ");
view.setUint32(16, 16, true); view.setUint16(20, 1, true); view.setUint16(22, 1, true);
view.setUint32(24, sampleRate, true); view.setUint32(28, sampleRate * 2, true);
view.setUint16(32, 2, true); view.setUint16(34, 16, true); text(36, "data"); view.setUint32(40, samples * 2, true);
let offset = 44;
for (const chunk of chunks) for (const value of chunk) {
const s = Math.max(-1, Math.min(1, value));
view.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7fff, true); offset += 2;
}
return new Blob([buffer], {type:"audio/wav"});
}
// ---------- WebSocket ----------
function connectWS() {
ws = new WebSocket(`ws://${location.host}/agent`);
ws.binaryType = "arraybuffer";
ws.onopen = () => $("st").textContent = "已连接";
ws.onclose = () => { $("st").textContent = "连接断开"; stopMic(); };
ws.onclose = () => { $("st").textContent = "连接断开"; stopMic(); stopPlayback(); };
ws.onmessage = ev => {
if (ev.data instanceof ArrayBuffer) { playPcm(new Int16Array(ev.data)); return; }
if (ev.data instanceof ArrayBuffer) { if (playEnabled) playPcm(new Int16Array(ev.data)); return; }
const m = JSON.parse(ev.data);
switch (m.type) {
case "partial":
@@ -113,6 +264,7 @@ function connectWS() {
break;
}
case "reply-delta":
playEnabled = true;
if (!replyEl) { replyText = ""; replyEl = log($("chat-log"), "", "bot"); }
replyText += m.text;
replyEl.innerHTML = `<div class="who">客服</div>${esc(replyText)}`;
@@ -126,6 +278,11 @@ function connectWS() {
case "tts-end":
$("tts-badge").textContent = "idle";
break;
case "interrupt":
stopPlayback();
replyEl = null; replyText = "";
$("tts-badge").textContent = "interrupted";
break;
case "asr-stopped":
$("asr-badge").textContent = "idle";
break;
@@ -177,13 +334,16 @@ function stopMic() {
}
// ---------- TTS 播放(PCM16/16k ----------
let playCtx = null, playHead = 0;
let playCtx = null, playHead = 0, playEnabled = false;
const playSources = new Set();
function playPcm(pcm) {
if (!playCtx || playCtx.state === "closed") { playCtx = new AudioContext({ sampleRate: 16000 }); playHead = 0; }
playCtx.resume();
const buf = playCtx.createBuffer(1, pcm.length, 16000);
buf.getChannelData(0).set(Float32Array.from(pcm, v => v / 32768));
const src = playCtx.createBufferSource();
playSources.add(src);
src.onended = () => playSources.delete(src);
src.buffer = buf; src.connect(playCtx.destination);
const now = playCtx.currentTime;
if (playHead < now + 0.05) playHead = now + 0.05; // 游标滞后(播完已久/首块)则贴到当前时刻
@@ -191,18 +351,37 @@ function playPcm(pcm) {
playHead += buf.duration;
}
function stopPlayback() {
playEnabled = false;
for (const src of playSources) {
try { src.stop(); } catch (_) {}
}
playSources.clear();
playHead = playCtx ? playCtx.currentTime : 0;
}
// ---------- 按钮 ----------
$("mic").onclick = async () => {
const mic = $("mic");
if (mic.classList.contains("on")) { // 停止
ws && ws.send(JSON.stringify({ type: "stop" }));
stopPlayback();
stopMic();
return;
}
if (recordStream) {
log($("chat-log"), "⚠ 请先停止样本录制", "err");
return;
}
if (!ttsConfig().voice) {
log($("chat-log"), "⚠ 请先选择预置音色或复刻音色", "err");
$("tts-cloned-voice").focus();
return;
}
if (!ws || ws.readyState !== 1) connectWS();
await new Promise(r => { const c = () => { if (ws.readyState === 1) r(); else setTimeout(c, 50); }; c(); });
try { await startMic(); } catch (e) { log($("chat-log"), `⚠ 麦克风失败: ${esc(e.message)}`, "err"); return; }
ws.send(JSON.stringify({ type: "start", asr: $("model").value }));
ws.send(JSON.stringify({ type: "start", asr: $("model").value, tts: ttsConfig() }));
mic.className = "on"; mic.textContent = "⏹ 停止";
$("asr-badge").textContent = "listening";
log($("asr-log"), "", "sys").remove();