feat: improve voice controls and interruption
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
# voice_test — 实时 ASR 客服机器人测试台
|
||||
|
||||
Chrome 麦克风 → 实时 ASR(可选多模型)→ LLM 客服人设回答 → TTS 播报,全链路验证。
|
||||
Chrome 麦克风 → 实时 ASR(可选多模型)→ LLM 客服人设回答 → 下拉选择 TTS 模型、预置/复刻音色并配置语速、语调、音量和表达指令;页面还可录制并下载复刻用 WAV 样本。
|
||||
|
||||
## 运行
|
||||
|
||||
@@ -39,7 +39,9 @@ export VOLC_ASR_RESOURCE_ID=volc.bigasr.sauc.duration # 2.0 用 volc.seedasr.sa
|
||||
| `BAILIAN_BASE_URL` / `BAILIAN_API_KEY` / `BAILIAN_WSS_BASE_URL` | - | 阿里百炼(zshenv 已有) |
|
||||
| `BAILIAN_LLM_MODEL` | `qwen3.8-flash` | 客服回答模型(OpenAI 兼容流式) |
|
||||
| `BAILIAN_TTS_MODEL` | `cosyvoice-v3.5-plus` | 语音合成模型 |
|
||||
| `BAILIAN_TTS_VOICE` | zshenv 音色 | cosyvoice-v3.5-plus + 复刻音色已验证可用 |
|
||||
| `BAILIAN_TTS_VOICE` | zshenv 音色 | 默认音色或百炼复刻/设计生成的 `voice_id` |
|
||||
| `BAILIAN_TTS_INSTRUCTION` | 空 | 默认声音表达指令,网页可覆盖 |
|
||||
| `BAILIAN_VOICE_API_URL` | DashScope 声音管理接口 | 查询当前账号的复刻/设计音色下拉列表 |
|
||||
| `PORT` | `:8090` | 监听地址 |
|
||||
|
||||
## 已验证的端到端指标(同一句“查询订单发货”,50ms/块推流,句级流式 TTS + 无 thinking)
|
||||
@@ -69,7 +71,8 @@ ASR 尾静音实测:语音结束→final 判定 ≈ 0.5s(qwen 0.6s),`max
|
||||
- `asr_volc.go` — 火山 sauc bigmodel(二进制帧协议)
|
||||
- `llm.go` — OpenAI 兼容流式对话 + 客服 system prompt
|
||||
- `tts.go` — CosyVoice 流式合成(PCM16/16k)
|
||||
- `web/index.html` — 测试页(AudioWorklet 采集 PCM16/16k、音量条、双栏转录)
|
||||
- `voices.go` — TTS 模型、预置音色与百炼复刻音色列表
|
||||
- `web/index.html` — 测试页(AudioWorklet 采集 PCM16/16k、WAV 样本录制、音量条、双栏转录)
|
||||
|
||||
## 已知简化
|
||||
|
||||
@@ -95,7 +98,7 @@ source ~/.zshenv
|
||||
cd ~/Workspace/voice_test && go build -o voicetest . && ./voicetest
|
||||
```
|
||||
|
||||
1. [ ] Chrome 打开 `http://localhost:8090`,状态栏显示“已连接”,下拉框列出 4 个模型
|
||||
1. [ ] Chrome 打开 `http://localhost:8090`,ASR/TTS 模型及预置/复刻音色均显示为下拉框
|
||||
2. [ ] 选 `qwen-audio-3.0-asr-flash-streaming` → 🎤 开始 → 授权麦克风
|
||||
3. [ ] 说“你好,我想查一下我的订单什么时候发货”:
|
||||
- [ ] 左栏灰色 partial 实时上屏,停顿后固化为 final
|
||||
@@ -105,7 +108,8 @@ cd ~/Workspace/voice_test && go build -o voicetest . && ./voicetest
|
||||
4. [ ] 追问“订单号是 12345”:确认多轮上下文生效(回答应引用订单号)
|
||||
5. [ ] 停止 → 重新开始,切换 `fun-asr-realtime` / `fun-asr-flash-2026-06-15` 重复 3
|
||||
6. [ ] 点停止后 asr badge 回 idle;服务端日志出现 `ASR started` 而非报错
|
||||
7. [ ] 火山:选 `volc-bigmodel` 重复 3、4(已自动化验证通过,人工复测语音即可)
|
||||
7. [ ] 点击“录制复刻样本”,录制后可试听并下载单声道 PCM WAV
|
||||
8. [ ] 火山:选 `volc-bigmodel` 重复 3、4(已自动化验证通过,人工复测语音即可)
|
||||
|
||||
### 🧹 清理
|
||||
|
||||
|
||||
+30
-10
@@ -5,6 +5,7 @@ import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/gorilla/websocket"
|
||||
)
|
||||
@@ -15,11 +16,13 @@ type bailianASR struct {
|
||||
model string
|
||||
api agentCfg
|
||||
|
||||
conn *websocket.Conn
|
||||
taskID string
|
||||
out chan asrEvent
|
||||
wmu sync.Mutex // 上游连接写锁(sendAudio 与 updateContext 并发)
|
||||
once sync.Once
|
||||
conn *websocket.Conn
|
||||
taskID string
|
||||
out chan asrEvent
|
||||
wmu sync.Mutex // 上游连接写锁(sendAudio 与 updateContext 并发)
|
||||
connOnce sync.Once
|
||||
outOnce sync.Once
|
||||
stopping atomic.Bool
|
||||
}
|
||||
|
||||
func newBailianASR(model string, api agentCfg) *bailianASR {
|
||||
@@ -33,6 +36,12 @@ func (a *bailianASR) start(ctx context.Context) error {
|
||||
return fmt.Errorf("连接百炼 ASR: %w", err)
|
||||
}
|
||||
a.conn = conn
|
||||
started := false
|
||||
defer func() {
|
||||
if !started {
|
||||
conn.Close()
|
||||
}
|
||||
}()
|
||||
a.taskID = uuid()
|
||||
err = a.writeJSON(map[string]any{
|
||||
"header": map[string]string{"action": "run-task", "task_id": a.taskID, "streaming": "duplex"},
|
||||
@@ -59,6 +68,7 @@ func (a *bailianASR) start(ctx context.Context) error {
|
||||
if ev.Header.Event != "task-started" {
|
||||
return fmt.Errorf("启动任务失败: %s %s", ev.Header.ErrorCode, ev.Header.ErrorMessage)
|
||||
}
|
||||
started = true
|
||||
go a.readLoop()
|
||||
return nil
|
||||
}
|
||||
@@ -82,7 +92,9 @@ func (a *bailianASR) readLoop() {
|
||||
} `json:"payload"`
|
||||
}
|
||||
if err := a.conn.ReadJSON(&ev); err != nil {
|
||||
a.emit(asrEvent{Typ: "error", Error: fmt.Sprintf("连接关闭: %v", err)})
|
||||
if !a.stopping.Load() {
|
||||
a.emit(asrEvent{Typ: "error", Error: fmt.Sprintf("连接关闭: %v", err)})
|
||||
}
|
||||
return
|
||||
}
|
||||
switch ev.Header.Event {
|
||||
@@ -146,13 +158,21 @@ func (a *bailianASR) finish() error {
|
||||
}
|
||||
|
||||
func (a *bailianASR) events() <-chan asrEvent { return a.out }
|
||||
func (a *bailianASR) close() { a.closeUpstream() }
|
||||
func (a *bailianASR) close() {
|
||||
a.stopping.Store(true)
|
||||
a.closeConn()
|
||||
}
|
||||
|
||||
func (a *bailianASR) closeUpstream() {
|
||||
a.once.Do(func() {
|
||||
func (a *bailianASR) closeConn() {
|
||||
a.connOnce.Do(func() {
|
||||
if a.conn != nil {
|
||||
a.conn.Close()
|
||||
}
|
||||
close(a.out)
|
||||
})
|
||||
}
|
||||
|
||||
// 只有 readLoop 退出后才能关闭事件通道,避免 close() 与 emit() 并发导致 panic。
|
||||
func (a *bailianASR) closeUpstream() {
|
||||
a.closeConn()
|
||||
a.outOnce.Do(func() { close(a.out) })
|
||||
}
|
||||
|
||||
+36
-16
@@ -10,6 +10,7 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/gorilla/websocket"
|
||||
)
|
||||
@@ -21,9 +22,9 @@ const (
|
||||
volcURL = "wss://openspeech.bytedance.com/api/v3/sauc/bigmodel"
|
||||
|
||||
msgFullClient = 0x1
|
||||
msgAudioOnly = 0x2
|
||||
msgFullServer = 0x9
|
||||
msgError = 0xF
|
||||
msgAudioOnly = 0x2
|
||||
msgFullServer = 0x9
|
||||
msgError = 0xF
|
||||
|
||||
flagNoSeq = 0x0 // full client request 用
|
||||
flagLast = 0x2 // 客户端最后一包音频(负包,无序号)
|
||||
@@ -34,12 +35,14 @@ const (
|
||||
)
|
||||
|
||||
type volcASR struct {
|
||||
cfg agentCfg
|
||||
conn *websocket.Conn
|
||||
out chan asrEvent
|
||||
wmu sync.Mutex
|
||||
once sync.Once
|
||||
emitted map[int]bool // 已作为 final 发出的 utterance 下标
|
||||
cfg agentCfg
|
||||
conn *websocket.Conn
|
||||
out chan asrEvent
|
||||
wmu sync.Mutex
|
||||
connOnce sync.Once
|
||||
outOnce sync.Once
|
||||
stopping atomic.Bool
|
||||
emitted map[int]bool // 已作为 final 发出的 utterance 下标
|
||||
}
|
||||
|
||||
func newVolcASR(cfg agentCfg) *volcASR {
|
||||
@@ -70,10 +73,16 @@ func (v *volcASR) start(ctx context.Context) error {
|
||||
return fmt.Errorf("连接火山 ASR (401/403 多为凭证或资源未开通, logid=%s): %w", logid, err)
|
||||
}
|
||||
v.conn = conn
|
||||
started := false
|
||||
defer func() {
|
||||
if !started {
|
||||
conn.Close()
|
||||
}
|
||||
}()
|
||||
|
||||
req := map[string]any{
|
||||
"user": map[string]any{"uid": "voice-test-agent"},
|
||||
"audio": map[string]any{"format": "pcm", "rate": 16000, "bits": 16, "channel": 1},
|
||||
"user": map[string]any{"uid": "voice-test-agent"},
|
||||
"audio": map[string]any{"format": "pcm", "rate": 16000, "bits": 16, "channel": 1},
|
||||
"request": map[string]any{
|
||||
"model_name": "bigmodel",
|
||||
"enable_itn": true,
|
||||
@@ -86,6 +95,7 @@ func (v *volcASR) start(ctx context.Context) error {
|
||||
if err := v.conn.WriteMessage(websocket.BinaryMessage, volcFrame(msgFullClient, flagNoSeq, serJSON, compNone, body)); err != nil {
|
||||
return fmt.Errorf("发送 full client request: %w", err)
|
||||
}
|
||||
started = true
|
||||
go v.readLoop()
|
||||
return nil
|
||||
}
|
||||
@@ -107,7 +117,9 @@ func (v *volcASR) readLoop() {
|
||||
for {
|
||||
mt, data, err := v.conn.ReadMessage()
|
||||
if err != nil {
|
||||
v.emit(asrEvent{Typ: "error", Error: fmt.Sprintf("连接关闭: %v", err)})
|
||||
if !v.stopping.Load() {
|
||||
v.emit(asrEvent{Typ: "error", Error: fmt.Sprintf("连接关闭: %v", err)})
|
||||
}
|
||||
return
|
||||
}
|
||||
if mt != websocket.BinaryMessage || len(data) < 8 {
|
||||
@@ -205,13 +217,21 @@ func (v *volcASR) finish() error {
|
||||
}
|
||||
|
||||
func (v *volcASR) events() <-chan asrEvent { return v.out }
|
||||
func (v *volcASR) close() { v.closeUpstream() }
|
||||
func (v *volcASR) close() {
|
||||
v.stopping.Store(true)
|
||||
v.closeConn()
|
||||
}
|
||||
|
||||
func (v *volcASR) closeUpstream() {
|
||||
v.once.Do(func() {
|
||||
func (v *volcASR) closeConn() {
|
||||
v.connOnce.Do(func() {
|
||||
if v.conn != nil {
|
||||
v.conn.Close()
|
||||
}
|
||||
close(v.out)
|
||||
})
|
||||
}
|
||||
|
||||
// 只有 readLoop 退出后才能关闭事件通道,避免 close() 与 emit() 并发导致 panic。
|
||||
func (v *volcASR) closeUpstream() {
|
||||
v.closeConn()
|
||||
v.outOnce.Do(func() { close(v.out) })
|
||||
}
|
||||
|
||||
@@ -12,6 +12,8 @@ type agentCfg struct {
|
||||
BailianKey string
|
||||
TTSModel string
|
||||
TTSVoice string
|
||||
TTSInstruction string
|
||||
VoiceAPIURL string
|
||||
LLMModel string
|
||||
VolcAppKey string // 新版控制台:单凭证
|
||||
VolcAppID string // 旧版控制台:APP ID + Access Token 两件套
|
||||
@@ -26,6 +28,8 @@ func loadCfg() agentCfg {
|
||||
BailianKey: getenv("BAILIAN_API_KEY", ""),
|
||||
TTSModel: getenv("BAILIAN_TTS_MODEL", "cosyvoice-v3.5-plus"),
|
||||
TTSVoice: getenv("BAILIAN_TTS_VOICE", ""),
|
||||
TTSInstruction: getenv("BAILIAN_TTS_INSTRUCTION", ""),
|
||||
VoiceAPIURL: getenv("BAILIAN_VOICE_API_URL", "https://dashscope.aliyuncs.com/api/v1/services/audio/tts/customization"),
|
||||
LLMModel: getenv("BAILIAN_LLM_MODEL", "qwen3.8-flash"),
|
||||
VolcAppKey: getenv("VOLC_ASR_APP_KEY", ""),
|
||||
VolcAppID: getenv("VOLC_ASR_APP_ID", ""),
|
||||
|
||||
@@ -29,6 +29,7 @@ func llmChat(ctx context.Context, cfg agentCfg, history []chatMsg) (<-chan strin
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", cfg.BailianBaseURL+"/chat/completions",
|
||||
bytes.NewReader(body))
|
||||
if err != nil {
|
||||
close(textCh)
|
||||
errCh <- err
|
||||
return textCh, errCh
|
||||
}
|
||||
@@ -37,6 +38,7 @@ func llmChat(ctx context.Context, cfg agentCfg, history []chatMsg) (<-chan strin
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
close(textCh)
|
||||
errCh <- err
|
||||
return textCh, errCh
|
||||
}
|
||||
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
)
|
||||
|
||||
var upgrader = websocket.Upgrader{
|
||||
CheckOrigin: func(*http.Request) bool { return true }, // 本地测试页,放开来源
|
||||
CheckOrigin: func(*http.Request) bool { return true }, // 本地测试页,放开来源
|
||||
ReadBufferSize: 16384,
|
||||
WriteBufferSize: 16384,
|
||||
}
|
||||
@@ -30,9 +30,10 @@ func asrModels(cfg agentCfg) []map[string]any {
|
||||
}
|
||||
|
||||
type clientMsg struct {
|
||||
Type string `json:"type"`
|
||||
ASR string `json:"asr"`
|
||||
Mode string `json:"mode"`
|
||||
Type string `json:"type"`
|
||||
ASR string `json:"asr"`
|
||||
Mode string `json:"mode"`
|
||||
TTS *ttsCfg `json:"tts"`
|
||||
}
|
||||
|
||||
type chatMsg struct {
|
||||
@@ -41,13 +42,16 @@ type chatMsg struct {
|
||||
}
|
||||
|
||||
type session struct {
|
||||
cfg agentCfg
|
||||
conn *websocket.Conn
|
||||
wmu sync.Mutex // 客户端连接写锁
|
||||
mu sync.Mutex // provider / history 状态锁
|
||||
prov asrProvider
|
||||
hist []chatMsg
|
||||
cancel context.CancelFunc
|
||||
cfg agentCfg
|
||||
conn *websocket.Conn
|
||||
wmu sync.Mutex // 客户端连接写锁
|
||||
mu sync.Mutex // provider / history 状态锁
|
||||
prov asrProvider
|
||||
tts ttsCfg
|
||||
hist []chatMsg
|
||||
cancel context.CancelFunc
|
||||
turnCancel context.CancelFunc
|
||||
turnID uint64
|
||||
}
|
||||
|
||||
func (s *session) sendJSON(v any) {
|
||||
@@ -73,12 +77,29 @@ func main() {
|
||||
http.HandleFunc("/api/models", func(w http.ResponseWriter, r *http.Request) {
|
||||
json.NewEncoder(w).Encode(asrModels(cfg))
|
||||
})
|
||||
http.HandleFunc("/api/config", func(w http.ResponseWriter, r *http.Request) {
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 3*time.Second)
|
||||
defer cancel()
|
||||
cloned, voiceErr := listClonedVoices(ctx, cfg)
|
||||
found := false
|
||||
for _, v := range cloned {
|
||||
found = found || v.ID == cfg.TTSVoice
|
||||
}
|
||||
if cfg.TTSVoice != "" && !found {
|
||||
cloned = append([]voiceOption{{ID: cfg.TTSVoice, Label: cfg.TTSVoice + "(当前配置)"}}, cloned...)
|
||||
}
|
||||
payload := map[string]any{"tts": defaultTTS(cfg), "ttsModels": ttsModels(), "systemVoices": systemVoices(), "clonedVoices": cloned}
|
||||
if voiceErr != nil {
|
||||
payload["voiceError"] = voiceErr.Error()
|
||||
}
|
||||
json.NewEncoder(w).Encode(payload)
|
||||
})
|
||||
http.HandleFunc("/agent", func(w http.ResponseWriter, r *http.Request) {
|
||||
conn, err := upgrader.Upgrade(w, r, nil)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
s := &session{cfg: cfg, conn: conn}
|
||||
s := &session{cfg: cfg, conn: conn, tts: defaultTTS(cfg)}
|
||||
defer conn.Close()
|
||||
s.readLoop()
|
||||
})
|
||||
@@ -102,14 +123,9 @@ func (s *session) readLoop() {
|
||||
}
|
||||
switch m.Type {
|
||||
case "start":
|
||||
s.startASR(m.ASR)
|
||||
s.startASR(m.ASR, m.TTS)
|
||||
case "stop":
|
||||
s.mu.Lock()
|
||||
p := s.prov
|
||||
s.mu.Unlock()
|
||||
if p != nil {
|
||||
p.finish()
|
||||
}
|
||||
s.stopASR()
|
||||
}
|
||||
case websocket.BinaryMessage:
|
||||
s.mu.Lock()
|
||||
@@ -122,7 +138,12 @@ func (s *session) readLoop() {
|
||||
}
|
||||
}
|
||||
|
||||
func (s *session) startASR(id string) {
|
||||
func (s *session) startASR(id string, requestedTTS *ttsCfg) {
|
||||
tts, err := normalizeTTS(requestedTTS, s.cfg)
|
||||
if err != nil {
|
||||
s.sendJSON(map[string]any{"type": "error", "error": err.Error()})
|
||||
return
|
||||
}
|
||||
s.stopASR()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
var p asrProvider
|
||||
@@ -138,13 +159,14 @@ func (s *session) startASR(id string) {
|
||||
return
|
||||
}
|
||||
s.mu.Lock()
|
||||
s.prov, s.cancel = p, cancel
|
||||
s.prov, s.cancel, s.tts = p, cancel, tts
|
||||
s.mu.Unlock()
|
||||
go s.eventPump(p)
|
||||
log.Printf("ASR started: %s", id)
|
||||
}
|
||||
|
||||
func (s *session) stopASR() {
|
||||
s.interruptTurn()
|
||||
s.mu.Lock()
|
||||
p, cancel := s.prov, s.cancel
|
||||
s.prov, s.cancel = nil, nil
|
||||
@@ -158,7 +180,38 @@ func (s *session) stopASR() {
|
||||
}
|
||||
}
|
||||
|
||||
// eventPump: ASR 事件 → 转发给浏览器;final 句触发客服回答(LLM 流式 → TTS 流式)。
|
||||
func (s *session) interruptTurn() {
|
||||
s.mu.Lock()
|
||||
cancel := s.turnCancel
|
||||
s.turnCancel = nil
|
||||
s.mu.Unlock()
|
||||
if cancel != nil {
|
||||
cancel()
|
||||
if s.conn != nil {
|
||||
s.sendJSON(map[string]string{"type": "interrupt"})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *session) startTurn() (context.Context, context.CancelFunc, uint64) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
|
||||
s.mu.Lock()
|
||||
s.turnID++
|
||||
id := s.turnID
|
||||
s.turnCancel = cancel
|
||||
s.mu.Unlock()
|
||||
return ctx, cancel, id
|
||||
}
|
||||
|
||||
func (s *session) finishTurn(id uint64) {
|
||||
s.mu.Lock()
|
||||
if s.turnID == id {
|
||||
s.turnCancel = nil
|
||||
}
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
// eventPump: 新语音一出现即打断当前回答;final 句异步触发下一轮,保持 ASR 事件可继续消费。
|
||||
func (s *session) eventPump(p asrProvider) {
|
||||
defer s.sendJSON(map[string]string{"type": "asr-stopped"})
|
||||
for ev := range p.events() {
|
||||
@@ -166,21 +219,24 @@ func (s *session) eventPump(p asrProvider) {
|
||||
case "error":
|
||||
s.sendJSON(map[string]any{"type": "error", "error": ev.Error, "code": ev.Code})
|
||||
case "partial", "final":
|
||||
s.interruptTurn()
|
||||
s.sendJSON(map[string]any{"type": ev.Typ, "text": ev.Text})
|
||||
if ev.Typ == "final" {
|
||||
s.agentTurn(p, ev.Text)
|
||||
ctx, cancel, id := s.startTurn()
|
||||
go s.agentTurn(ctx, cancel, id, p, ev.Text)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// agentTurn: final 句 → LLM 流式出字(逐段回传)→ 边生成边按句喂 TTS → 音频流回传。
|
||||
func (s *session) agentTurn(p asrProvider, userText string) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
|
||||
func (s *session) agentTurn(ctx context.Context, cancel context.CancelFunc, id uint64, p asrProvider, userText string) {
|
||||
defer cancel()
|
||||
defer s.finishTurn(id)
|
||||
|
||||
s.mu.Lock()
|
||||
hist := append([]chatMsg(nil), s.hist...)
|
||||
tts := s.tts
|
||||
s.mu.Unlock()
|
||||
hist = append(hist, chatMsg{Role: "user", Content: userText})
|
||||
|
||||
@@ -190,7 +246,7 @@ func (s *session) agentTurn(p asrProvider, userText string) {
|
||||
go func() {
|
||||
defer close(ttsDone)
|
||||
for sen := range sentences {
|
||||
if err := ttsSpeak(ctx, s.cfg, sen, s.sendAudio); err != nil {
|
||||
if err := ttsSpeak(ctx, s.cfg, tts, sen, s.sendAudio); err != nil && ctx.Err() == nil {
|
||||
s.sendJSON(map[string]any{"type": "error", "error": err.Error()})
|
||||
}
|
||||
}
|
||||
@@ -199,27 +255,45 @@ func (s *session) agentTurn(p asrProvider, userText string) {
|
||||
textCh, errCh := llmChat(ctx, s.cfg, hist)
|
||||
var full []byte
|
||||
var pending []rune
|
||||
flush := func() {
|
||||
flush := func() bool {
|
||||
if sen := strings.TrimSpace(string(pending)); sen != "" {
|
||||
sentences <- sen
|
||||
select {
|
||||
case sentences <- sen:
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
}
|
||||
}
|
||||
pending = pending[:0]
|
||||
return true
|
||||
}
|
||||
for seg := range textCh {
|
||||
if ctx.Err() != nil {
|
||||
continue // drain producer so cancellation cannot leave it blocked on textCh
|
||||
}
|
||||
full = append(full, seg...)
|
||||
s.sendJSON(map[string]any{"type": "reply-delta", "text": seg})
|
||||
for _, r := range seg {
|
||||
pending = append(pending, r)
|
||||
// 句末标点即切句;超长无标点时退到逗号,避免一句卡住整段
|
||||
if strings.ContainsRune("。!?!?;;\n", r) || (len(pending) >= 100 && r == ',') {
|
||||
flush()
|
||||
if !flush() {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
flush()
|
||||
if ctx.Err() == nil {
|
||||
flush()
|
||||
}
|
||||
close(sentences)
|
||||
if err := <-errCh; err != nil {
|
||||
s.sendJSON(map[string]any{"type": "error", "error": "LLM: " + err.Error()})
|
||||
if ctx.Err() == nil {
|
||||
s.sendJSON(map[string]any{"type": "error", "error": "LLM: " + err.Error()})
|
||||
}
|
||||
<-ttsDone
|
||||
return
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
<-ttsDone
|
||||
return
|
||||
}
|
||||
|
||||
@@ -4,14 +4,52 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/websocket"
|
||||
)
|
||||
|
||||
type ttsCfg struct {
|
||||
Model string `json:"model"`
|
||||
Voice string `json:"voice"`
|
||||
Instruction string `json:"instruction"`
|
||||
Rate float64 `json:"rate"`
|
||||
Pitch float64 `json:"pitch"`
|
||||
Volume int `json:"volume"`
|
||||
}
|
||||
|
||||
func defaultTTS(cfg agentCfg) ttsCfg {
|
||||
return ttsCfg{Model: cfg.TTSModel, Voice: cfg.TTSVoice, Instruction: cfg.TTSInstruction, Rate: 1, Pitch: 1, Volume: 50}
|
||||
}
|
||||
|
||||
func normalizeTTS(in *ttsCfg, cfg agentCfg) (ttsCfg, error) {
|
||||
if in == nil {
|
||||
return defaultTTS(cfg), nil
|
||||
}
|
||||
t := *in
|
||||
t.Model, t.Voice, t.Instruction = strings.TrimSpace(t.Model), strings.TrimSpace(t.Voice), strings.TrimSpace(t.Instruction)
|
||||
if t.Model == "" {
|
||||
t.Model = cfg.TTSModel
|
||||
}
|
||||
if t.Voice == "" {
|
||||
t.Voice = cfg.TTSVoice
|
||||
}
|
||||
if t.Voice == "" {
|
||||
return t, fmt.Errorf("请填写音色或复刻音色 ID")
|
||||
}
|
||||
if len(t.Model) > 100 || len(t.Voice) > 200 || len([]rune(t.Instruction)) > 100 {
|
||||
return t, fmt.Errorf("TTS 配置文本过长")
|
||||
}
|
||||
if t.Rate < 0.5 || t.Rate > 2 || t.Pitch < 0.5 || t.Pitch > 2 || t.Volume < 0 || t.Volume > 100 {
|
||||
return t, fmt.Errorf("TTS 参数超出范围:语速/语调 0.5~2.0,音量 0~100")
|
||||
}
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// ttsSpeak: 百炼 CosyVoice 流式合成,音频块实时回调 onAudio。
|
||||
func ttsSpeak(ctx context.Context, cfg agentCfg, text string, onAudio func([]byte)) error {
|
||||
func ttsSpeak(ctx context.Context, cfg agentCfg, tts ttsCfg, text string, onAudio func([]byte)) error {
|
||||
cctx, cancel := context.WithTimeout(ctx, 60*time.Second)
|
||||
defer cancel()
|
||||
conn, _, err := websocket.DefaultDialer.DialContext(cctx, cfg.BailianWssBaseURL,
|
||||
@@ -22,16 +60,21 @@ func ttsSpeak(ctx context.Context, cfg agentCfg, text string, onAudio func([]byt
|
||||
defer conn.Close()
|
||||
|
||||
taskID := uuid()
|
||||
parameters := map[string]any{
|
||||
"text_type": "PlainText", "voice": tts.Voice,
|
||||
"format": "pcm", "sample_rate": 16000,
|
||||
"volume": tts.Volume, "rate": tts.Rate, "pitch": tts.Pitch,
|
||||
}
|
||||
if tts.Instruction != "" {
|
||||
parameters["instruction"] = tts.Instruction
|
||||
}
|
||||
err = conn.WriteJSON(map[string]any{
|
||||
"header": map[string]string{"action": "run-task", "task_id": taskID, "streaming": "duplex"},
|
||||
"payload": map[string]any{
|
||||
"task_group": "audio", "task": "tts", "function": "SpeechSynthesizer",
|
||||
"model": cfg.TTSModel,
|
||||
"parameters": map[string]any{
|
||||
"text_type": "PlainText", "voice": cfg.TTSVoice,
|
||||
"format": "pcm", "sample_rate": 16000,
|
||||
},
|
||||
"input": map[string]any{},
|
||||
"model": tts.Model,
|
||||
"parameters": parameters,
|
||||
"input": map[string]any{},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
@@ -40,7 +83,6 @@ func ttsSpeak(ctx context.Context, cfg agentCfg, text string, onAudio func([]byt
|
||||
|
||||
var once sync.Once
|
||||
done := make(chan error, 1)
|
||||
var buf []json.RawMessage // task-started / task-finished 事件
|
||||
go func() {
|
||||
for {
|
||||
mt, data, err := conn.ReadMessage()
|
||||
@@ -66,7 +108,6 @@ func ttsSpeak(ctx context.Context, cfg agentCfg, text string, onAudio func([]byt
|
||||
}
|
||||
switch ev.Header.Event {
|
||||
case "task-started":
|
||||
buf = append(buf, json.RawMessage("{}"))
|
||||
once.Do(func() {
|
||||
// task-started 到达后才允许发文本
|
||||
if err := conn.WriteJSON(map[string]any{
|
||||
@@ -98,5 +139,11 @@ func ttsSpeak(ctx context.Context, cfg agentCfg, text string, onAudio func([]byt
|
||||
}
|
||||
}
|
||||
}()
|
||||
return <-done
|
||||
select {
|
||||
case err := <-done:
|
||||
return err
|
||||
case <-cctx.Done():
|
||||
conn.Close()
|
||||
return cctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
+80
@@ -0,0 +1,80 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestNormalizeTTS(t *testing.T) {
|
||||
cfg := agentCfg{TTSModel: "cosyvoice-v3.5-plus", TTSVoice: "clone-id"}
|
||||
|
||||
got, err := normalizeTTS(nil, cfg)
|
||||
if err != nil || got.Voice != "clone-id" || got.Rate != 1 || got.Pitch != 1 || got.Volume != 50 {
|
||||
t.Fatalf("defaults: got=%+v err=%v", got, err)
|
||||
}
|
||||
|
||||
bad := ttsCfg{Model: "m", Voice: "v", Rate: 2.1, Pitch: 1, Volume: 50}
|
||||
if _, err := normalizeTTS(&bad, cfg); err == nil {
|
||||
t.Fatal("expected out-of-range rate to fail")
|
||||
}
|
||||
}
|
||||
|
||||
func TestASRCloseLeavesEventsForReader(t *testing.T) {
|
||||
t.Run("bailian", func(t *testing.T) {
|
||||
a := newBailianASR("model", agentCfg{})
|
||||
a.close()
|
||||
a.emit(asrEvent{Typ: "final", Text: "ok"})
|
||||
if ev := <-a.events(); ev.Text != "ok" {
|
||||
t.Fatalf("event=%+v", ev)
|
||||
}
|
||||
a.closeUpstream()
|
||||
})
|
||||
t.Run("volc", func(t *testing.T) {
|
||||
v := newVolcASR(agentCfg{})
|
||||
v.close()
|
||||
v.emit(asrEvent{Typ: "final", Text: "ok"})
|
||||
if ev := <-v.events(); ev.Text != "ok" {
|
||||
t.Fatalf("event=%+v", ev)
|
||||
}
|
||||
v.closeUpstream()
|
||||
})
|
||||
}
|
||||
|
||||
func TestLLMRequestErrorClosesStream(t *testing.T) {
|
||||
text, errs := llmChat(context.Background(), agentCfg{BailianBaseURL: "://"}, nil)
|
||||
if _, open := <-text; open {
|
||||
t.Fatal("text stream left open after request error")
|
||||
}
|
||||
if err := <-errs; err == nil {
|
||||
t.Fatal("expected request error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestInterruptTurn(t *testing.T) {
|
||||
s := &session{}
|
||||
ctx, cancel, _ := s.startTurn()
|
||||
defer cancel()
|
||||
s.interruptTurn()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
default:
|
||||
t.Fatal("active turn was not canceled")
|
||||
}
|
||||
}
|
||||
|
||||
func TestListClonedVoices(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("Authorization") != "Bearer key" {
|
||||
t.Fatal("missing authorization")
|
||||
}
|
||||
w.Write([]byte(`{"output":{"voice_list":[{"voice_id":"clone-ok","status":"OK"},{"voice_id":"clone-building","status":"DEPLOYING"}]}}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
voices, err := listClonedVoices(context.Background(), agentCfg{BailianKey: "key", VoiceAPIURL: server.URL})
|
||||
if err != nil || len(voices) != 1 || voices[0].ID != "clone-ok" {
|
||||
t.Fatalf("voices=%+v err=%v", voices, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
type voiceOption struct {
|
||||
ID string `json:"id"`
|
||||
Label string `json:"label"`
|
||||
}
|
||||
|
||||
func ttsModels() []voiceOption {
|
||||
return []voiceOption{
|
||||
{ID: "cosyvoice-v3.5-plus", Label: "CosyVoice v3.5 Plus(复刻/设计音色)"},
|
||||
{ID: "cosyvoice-v3.5-flash", Label: "CosyVoice v3.5 Flash(低延迟)"},
|
||||
{ID: "cosyvoice-v3-plus", Label: "CosyVoice v3 Plus"},
|
||||
{ID: "cosyvoice-v3-flash", Label: "CosyVoice v3 Flash"},
|
||||
{ID: "cosyvoice-v2", Label: "CosyVoice v2"},
|
||||
}
|
||||
}
|
||||
|
||||
func systemVoices() map[string][]voiceOption {
|
||||
v3 := []voiceOption{{ID: "longanhuan", Label: "龙安欢|欢脱元气女"}, {ID: "longanyang", Label: "龙安洋|阳光男声"}}
|
||||
return map[string][]voiceOption{
|
||||
"cosyvoice-v3-plus": v3,
|
||||
"cosyvoice-v3-flash": v3,
|
||||
"cosyvoice-v2": {
|
||||
{ID: "longxiaochun_v2", Label: "龙小淳|温柔女声"},
|
||||
{ID: "longwan_v2", Label: "龙婉|普通话女声"},
|
||||
{ID: "loongbella_v2", Label: "Bella 2.0|新闻女声"},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func listClonedVoices(ctx context.Context, cfg agentCfg) ([]voiceOption, error) {
|
||||
body, _ := json.Marshal(map[string]any{
|
||||
"model": "voice-enrollment",
|
||||
"input": map[string]any{"action": "list_voice", "page_index": 0, "page_size": 100},
|
||||
})
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, cfg.VoiceAPIURL, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+cfg.BailianKey)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode/100 != 2 {
|
||||
return nil, fmt.Errorf("查询复刻音色失败: HTTP %d", resp.StatusCode)
|
||||
}
|
||||
var result struct {
|
||||
Output struct {
|
||||
Voices []struct {
|
||||
ID string `json:"voice_id"`
|
||||
Status string `json:"status"`
|
||||
} `json:"voice_list"`
|
||||
} `json:"output"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
voices := make([]voiceOption, 0, len(result.Output.Voices))
|
||||
for _, v := range result.Output.Voices {
|
||||
if v.ID != "" && (v.Status == "" || v.Status == "OK" || v.Status == "SUCCESS") {
|
||||
voices = append(voices, voiceOption{ID: v.ID, Label: v.ID})
|
||||
}
|
||||
}
|
||||
return voices, nil
|
||||
}
|
||||
+190
-11
@@ -7,24 +7,40 @@
|
||||
<style>
|
||||
:root { --bg:#0f1117; --panel:#1a1d27; --line:#2a2e3d; --fg:#e6e8f0; --dim:#8b90a5; --acc:#4f7cff; --ok:#3fbf6f; --warn:#e0a23c; --err:#e05c5c; }
|
||||
* { box-sizing:border-box; margin:0; padding:0; }
|
||||
body { background:var(--bg); color:var(--fg); font:14px/1.6 "PingFang SC","Microsoft YaHei",system-ui,sans-serif; min-height:100vh; display:flex; flex-direction:column; }
|
||||
html, body { height:100%; overflow:hidden; }
|
||||
body { background:var(--bg); color:var(--fg); font:14px/1.6 "PingFang SC","Microsoft YaHei",system-ui,sans-serif; display:flex; flex-direction:column; }
|
||||
header { padding:14px 22px; border-bottom:1px solid var(--line); display:flex; align-items:center; gap:14px; flex-wrap:wrap; }
|
||||
header h1 { font-size:16px; font-weight:600; }
|
||||
header .sel { display:flex; align-items:center; gap:8px; margin-left:auto; }
|
||||
select, button { background:var(--panel); color:var(--fg); border:1px solid var(--line); border-radius:8px; padding:8px 14px; font-size:14px; cursor:pointer; }
|
||||
select, button, input { background:var(--panel); color:var(--fg); border:1px solid var(--line); border-radius:8px; padding:8px 12px; font-size:14px; }
|
||||
select, button { cursor:pointer; }
|
||||
select { min-width:280px; }
|
||||
.config { padding:10px 22px; border-bottom:1px solid var(--line); background:#141720; max-height:45vh; overflow-y:auto; flex:none; }
|
||||
.config summary { cursor:pointer; color:var(--dim); user-select:none; }
|
||||
.config-grid { display:grid; grid-template-columns:repeat(5,minmax(130px,1fr)); gap:10px 14px; margin-top:10px; }
|
||||
.config-grid label { display:flex; flex-direction:column; gap:4px; color:var(--dim); font-size:12px; }
|
||||
.config-grid label.wide { grid-column:span 2; }
|
||||
.config-grid input[type="range"] { padding:0; accent-color:var(--acc); }
|
||||
.config-grid select { min-width:0; width:100%; }
|
||||
.range-title { display:flex; justify-content:space-between; }
|
||||
.hint { color:var(--dim); font-size:11px; margin-top:6px; }
|
||||
.recorder { display:flex; align-items:center; gap:10px; flex-wrap:wrap; margin-top:10px; padding-top:10px; border-top:1px solid var(--line); }
|
||||
#record.on { background:var(--err); color:#fff; }
|
||||
#recording-audio { height:34px; max-width:320px; }
|
||||
#recording-download { color:#8fb0ff; text-decoration:none; }
|
||||
@media (max-width:900px){ .config-grid{grid-template-columns:1fr 1fr;} .config-grid label.wide{grid-column:span 2;} }
|
||||
button:hover:not(:disabled) { border-color:var(--acc); }
|
||||
button:disabled { opacity:.4; cursor:not-allowed; }
|
||||
#mic { border-color:transparent; color:#fff; font-weight:600; min-width:96px; }
|
||||
#mic.on { background:var(--err); animation:pulse 1.2s infinite; }
|
||||
#mic.off { background:var(--acc); }
|
||||
@keyframes pulse { 0%,100%{box-shadow:0 0 0 0 rgba(224,92,92,.5)} 50%{box-shadow:0 0 0 10px rgba(224,92,92,0)} }
|
||||
main { flex:1; display:grid; grid-template-columns:1fr 1fr; gap:14px; padding:14px 22px; max-width:1200px; width:100%; margin:0 auto; }
|
||||
@media (max-width:860px){ main{grid-template-columns:1fr;} }
|
||||
.panel { background:var(--panel); border:1px solid var(--line); border-radius:12px; display:flex; flex-direction:column; min-height:320px; }
|
||||
main { flex:1; min-height:0; overflow:hidden; display:grid; grid-template-columns:1fr 1fr; grid-template-rows:minmax(0,1fr); gap:14px; padding:14px 22px; max-width:1200px; width:100%; margin:0 auto; }
|
||||
@media (max-width:860px){ main{grid-template-columns:1fr; grid-template-rows:repeat(2,minmax(0,1fr));} }
|
||||
.panel { background:var(--panel); border:1px solid var(--line); border-radius:12px; display:flex; flex-direction:column; min-height:0; overflow:hidden; }
|
||||
.panel h2 { font-size:13px; color:var(--dim); padding:10px 14px; border-bottom:1px solid var(--line); font-weight:500; display:flex; justify-content:space-between; }
|
||||
.panel h2 .badge { font-size:11px; padding:1px 8px; border-radius:6px; background:#262b3b; }
|
||||
.log { flex:1; overflow-y:auto; padding:12px 14px; display:flex; flex-direction:column; gap:10px; }
|
||||
.log { flex:1; min-height:0; overflow-y:auto; overscroll-behavior:contain; padding:12px 14px; display:flex; flex-direction:column; gap:10px; }
|
||||
.msg { padding:8px 12px; border-radius:10px; max-width:88%; white-space:pre-wrap; word-break:break-all; }
|
||||
.msg .who { font-size:11px; color:var(--dim); margin-bottom:2px; }
|
||||
.msg.user { align-self:flex-end; background:#25315e; border-top-right-radius:3px; }
|
||||
@@ -47,6 +63,26 @@
|
||||
<button id="mic" class="off">🎤 开始</button>
|
||||
</div>
|
||||
</header>
|
||||
<details class="config" open>
|
||||
<summary>TTS 声音配置</summary>
|
||||
<div class="config-grid">
|
||||
<label>TTS 模型<select id="tts-model"></select></label>
|
||||
<label>预置音色<select id="tts-system-voice"></select></label>
|
||||
<label class="wide">复刻 / 设计音色<select id="tts-cloned-voice"><option value="">请选择复刻音色</option></select></label>
|
||||
<label><span class="range-title">语速 <b id="rate-value">1.0×</b></span><input id="tts-rate" type="range" min="0.5" max="2" step="0.1" value="1"></label>
|
||||
<label><span class="range-title">语调 <b id="pitch-value">1.0×</b></span><input id="tts-pitch" type="range" min="0.5" max="2" step="0.1" value="1"></label>
|
||||
<label><span class="range-title">音量 <b id="volume-value">50</b></span><input id="tts-volume" type="range" min="0" max="100" step="1" value="50"></label>
|
||||
<label class="wide">声音表达 / 模仿描述<input id="tts-instruction" maxlength="100" placeholder="如:温柔沉稳、略带微笑、语速自然"></label>
|
||||
</div>
|
||||
<div class="hint">v3.5 仅支持复刻/设计音色;复刻音色从当前百炼账号自动读取。<span id="voice-status"></span></div>
|
||||
<div class="recorder">
|
||||
<button id="record">⏺ 录制复刻样本</button>
|
||||
<b id="record-time">00:00</b>
|
||||
<audio id="recording-audio" controls hidden></audio>
|
||||
<a id="recording-download" download="voice-sample.wav" hidden>下载 WAV</a>
|
||||
<span class="hint">建议安静环境录制 5~20 秒连续清晰人声,最长 30 秒。</span>
|
||||
</div>
|
||||
</details>
|
||||
<main>
|
||||
<div class="panel">
|
||||
<h2>识别(ASR) <span class="badge" id="asr-badge">idle</span></h2>
|
||||
@@ -62,7 +98,7 @@
|
||||
<span>延迟: <b id="lat">-</b></span>
|
||||
<span id="vu"><i></i><i></i><i></i><i></i><i></i></span>
|
||||
</div>
|
||||
<div class="footer-tip">浏览器采集 PCM16/16k 单声道,实时上行;每句判定结束后自动由客服 LLM 流式回答并 TTS 播报。Chrome 要求 localhost 或 HTTPS 才能使用麦克风。</div>
|
||||
<div class="footer-tip">浏览器采集 PCM16/16k 单声道,实时上行;每句判定结束后自动由客服 LLM 流式回答并按当前音色配置播报。Chrome 要求 localhost 或 HTTPS 才能使用麦克风。</div>
|
||||
|
||||
<script>
|
||||
"use strict";
|
||||
@@ -91,14 +127,129 @@ fetch("/api/models").then(r => r.json()).then(models => {
|
||||
}
|
||||
});
|
||||
|
||||
let systemVoices = {};
|
||||
fetch("/api/config").then(r => r.json()).then(({tts, ttsModels, systemVoices: voices, clonedVoices, voiceError}) => {
|
||||
systemVoices = voices;
|
||||
fillSelect($("tts-model"), ttsModels);
|
||||
$("tts-model").value = tts.model;
|
||||
fillSelect($("tts-cloned-voice"), clonedVoices, "请选择复刻/设计音色");
|
||||
const isSystem = renderSystemVoices(tts.voice);
|
||||
if (!isSystem) $("tts-cloned-voice").value = tts.voice;
|
||||
$("tts-instruction").value = tts.instruction;
|
||||
$("tts-rate").value = tts.rate;
|
||||
$("tts-pitch").value = tts.pitch;
|
||||
$("tts-volume").value = tts.volume;
|
||||
if (voiceError) $("voice-status").textContent = `(读取失败:${voiceError})`;
|
||||
updateRangeLabels();
|
||||
});
|
||||
|
||||
function fillSelect(select, options, placeholder = "") {
|
||||
select.replaceChildren();
|
||||
if (placeholder) select.add(new Option(placeholder, ""));
|
||||
for (const o of options || []) select.add(new Option(o.label, o.id));
|
||||
}
|
||||
|
||||
function renderSystemVoices(selected = "") {
|
||||
const options = systemVoices[$("tts-model").value] || [];
|
||||
fillSelect($("tts-system-voice"), options, options.length ? "请选择预置音色" : "该模型无预置音色");
|
||||
if (options.some(v => v.id === selected)) {
|
||||
$("tts-system-voice").value = selected;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
$("tts-model").onchange = () => renderSystemVoices();
|
||||
$("tts-system-voice").onchange = () => { if ($("tts-system-voice").value) $("tts-cloned-voice").value = ""; };
|
||||
$("tts-cloned-voice").onchange = () => { if ($("tts-cloned-voice").value) $("tts-system-voice").value = ""; };
|
||||
|
||||
function updateRangeLabels() {
|
||||
$("rate-value").textContent = Number($("tts-rate").value).toFixed(1) + "×";
|
||||
$("pitch-value").textContent = Number($("tts-pitch").value).toFixed(1) + "×";
|
||||
$("volume-value").textContent = $("tts-volume").value;
|
||||
}
|
||||
["tts-rate", "tts-pitch", "tts-volume"].forEach(id => $(id).oninput = updateRangeLabels);
|
||||
|
||||
function ttsConfig() {
|
||||
return {
|
||||
model: $("tts-model").value,
|
||||
voice: $("tts-cloned-voice").value || $("tts-system-voice").value,
|
||||
instruction: $("tts-instruction").value,
|
||||
rate: Number($("tts-rate").value),
|
||||
pitch: Number($("tts-pitch").value),
|
||||
volume: Number($("tts-volume").value)
|
||||
};
|
||||
}
|
||||
|
||||
// ---------- 复刻样本录制(单声道 PCM WAV) ----------
|
||||
let recordStream = null, recordCtx = null, recordNode = null, recordChunks = [], recordStarted = 0, recordTimer = null, recordURL = "";
|
||||
|
||||
$("record").onclick = async () => {
|
||||
if (recordStream) { stopRecording(); return; }
|
||||
if ($("mic").classList.contains("on")) {
|
||||
log($("chat-log"), "⚠ 请先停止 ASR 会话", "err");
|
||||
return;
|
||||
}
|
||||
try {
|
||||
recordStream = await navigator.mediaDevices.getUserMedia({audio:{channelCount:1, echoCancellation:false, noiseSuppression:false}});
|
||||
recordCtx = new AudioContext();
|
||||
const source = recordCtx.createMediaStreamSource(recordStream);
|
||||
recordNode = recordCtx.createScriptProcessor(4096, 1, 1);
|
||||
recordNode.onaudioprocess = e => recordChunks.push(new Float32Array(e.inputBuffer.getChannelData(0)));
|
||||
const silent = recordCtx.createGain(); silent.gain.value = 0;
|
||||
source.connect(recordNode); recordNode.connect(silent); silent.connect(recordCtx.destination);
|
||||
recordStarted = Date.now();
|
||||
$("record").className = "on"; $("record").textContent = "⏹ 停止录制";
|
||||
recordTimer = setInterval(() => {
|
||||
const seconds = Math.floor((Date.now() - recordStarted) / 1000);
|
||||
$("record-time").textContent = `00:${String(seconds).padStart(2, "0")}`;
|
||||
if (seconds >= 30) stopRecording();
|
||||
}, 250);
|
||||
} catch (e) {
|
||||
recordStream = null;
|
||||
log($("chat-log"), `⚠ 录音失败: ${esc(e.message)}`, "err");
|
||||
}
|
||||
};
|
||||
|
||||
function stopRecording() {
|
||||
clearInterval(recordTimer);
|
||||
const sampleRate = recordCtx.sampleRate;
|
||||
recordNode.disconnect();
|
||||
recordStream.getTracks().forEach(t => t.stop());
|
||||
recordCtx.close();
|
||||
recordStream = recordCtx = recordNode = null;
|
||||
$("record").className = ""; $("record").textContent = "⏺ 重新录制";
|
||||
if (recordURL) URL.revokeObjectURL(recordURL);
|
||||
recordURL = URL.createObjectURL(wavBlob(recordChunks, sampleRate));
|
||||
recordChunks = [];
|
||||
$("recording-audio").src = recordURL; $("recording-audio").hidden = false;
|
||||
$("recording-download").href = recordURL; $("recording-download").hidden = false;
|
||||
}
|
||||
|
||||
function wavBlob(chunks, sampleRate) {
|
||||
const samples = chunks.reduce((n, c) => n + c.length, 0);
|
||||
const buffer = new ArrayBuffer(44 + samples * 2), view = new DataView(buffer);
|
||||
const text = (offset, value) => [...value].forEach((c, i) => view.setUint8(offset + i, c.charCodeAt(0)));
|
||||
text(0, "RIFF"); view.setUint32(4, 36 + samples * 2, true); text(8, "WAVEfmt ");
|
||||
view.setUint32(16, 16, true); view.setUint16(20, 1, true); view.setUint16(22, 1, true);
|
||||
view.setUint32(24, sampleRate, true); view.setUint32(28, sampleRate * 2, true);
|
||||
view.setUint16(32, 2, true); view.setUint16(34, 16, true); text(36, "data"); view.setUint32(40, samples * 2, true);
|
||||
let offset = 44;
|
||||
for (const chunk of chunks) for (const value of chunk) {
|
||||
const s = Math.max(-1, Math.min(1, value));
|
||||
view.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7fff, true); offset += 2;
|
||||
}
|
||||
return new Blob([buffer], {type:"audio/wav"});
|
||||
}
|
||||
|
||||
// ---------- WebSocket ----------
|
||||
function connectWS() {
|
||||
ws = new WebSocket(`ws://${location.host}/agent`);
|
||||
ws.binaryType = "arraybuffer";
|
||||
ws.onopen = () => $("st").textContent = "已连接";
|
||||
ws.onclose = () => { $("st").textContent = "连接断开"; stopMic(); };
|
||||
ws.onclose = () => { $("st").textContent = "连接断开"; stopMic(); stopPlayback(); };
|
||||
ws.onmessage = ev => {
|
||||
if (ev.data instanceof ArrayBuffer) { playPcm(new Int16Array(ev.data)); return; }
|
||||
if (ev.data instanceof ArrayBuffer) { if (playEnabled) playPcm(new Int16Array(ev.data)); return; }
|
||||
const m = JSON.parse(ev.data);
|
||||
switch (m.type) {
|
||||
case "partial":
|
||||
@@ -113,6 +264,7 @@ function connectWS() {
|
||||
break;
|
||||
}
|
||||
case "reply-delta":
|
||||
playEnabled = true;
|
||||
if (!replyEl) { replyText = ""; replyEl = log($("chat-log"), "", "bot"); }
|
||||
replyText += m.text;
|
||||
replyEl.innerHTML = `<div class="who">客服</div>${esc(replyText)}`;
|
||||
@@ -126,6 +278,11 @@ function connectWS() {
|
||||
case "tts-end":
|
||||
$("tts-badge").textContent = "idle";
|
||||
break;
|
||||
case "interrupt":
|
||||
stopPlayback();
|
||||
replyEl = null; replyText = "";
|
||||
$("tts-badge").textContent = "interrupted";
|
||||
break;
|
||||
case "asr-stopped":
|
||||
$("asr-badge").textContent = "idle";
|
||||
break;
|
||||
@@ -177,13 +334,16 @@ function stopMic() {
|
||||
}
|
||||
|
||||
// ---------- TTS 播放(PCM16/16k) ----------
|
||||
let playCtx = null, playHead = 0;
|
||||
let playCtx = null, playHead = 0, playEnabled = false;
|
||||
const playSources = new Set();
|
||||
function playPcm(pcm) {
|
||||
if (!playCtx || playCtx.state === "closed") { playCtx = new AudioContext({ sampleRate: 16000 }); playHead = 0; }
|
||||
playCtx.resume();
|
||||
const buf = playCtx.createBuffer(1, pcm.length, 16000);
|
||||
buf.getChannelData(0).set(Float32Array.from(pcm, v => v / 32768));
|
||||
const src = playCtx.createBufferSource();
|
||||
playSources.add(src);
|
||||
src.onended = () => playSources.delete(src);
|
||||
src.buffer = buf; src.connect(playCtx.destination);
|
||||
const now = playCtx.currentTime;
|
||||
if (playHead < now + 0.05) playHead = now + 0.05; // 游标滞后(播完已久/首块)则贴到当前时刻
|
||||
@@ -191,18 +351,37 @@ function playPcm(pcm) {
|
||||
playHead += buf.duration;
|
||||
}
|
||||
|
||||
function stopPlayback() {
|
||||
playEnabled = false;
|
||||
for (const src of playSources) {
|
||||
try { src.stop(); } catch (_) {}
|
||||
}
|
||||
playSources.clear();
|
||||
playHead = playCtx ? playCtx.currentTime : 0;
|
||||
}
|
||||
|
||||
// ---------- 按钮 ----------
|
||||
$("mic").onclick = async () => {
|
||||
const mic = $("mic");
|
||||
if (mic.classList.contains("on")) { // 停止
|
||||
ws && ws.send(JSON.stringify({ type: "stop" }));
|
||||
stopPlayback();
|
||||
stopMic();
|
||||
return;
|
||||
}
|
||||
if (recordStream) {
|
||||
log($("chat-log"), "⚠ 请先停止样本录制", "err");
|
||||
return;
|
||||
}
|
||||
if (!ttsConfig().voice) {
|
||||
log($("chat-log"), "⚠ 请先选择预置音色或复刻音色", "err");
|
||||
$("tts-cloned-voice").focus();
|
||||
return;
|
||||
}
|
||||
if (!ws || ws.readyState !== 1) connectWS();
|
||||
await new Promise(r => { const c = () => { if (ws.readyState === 1) r(); else setTimeout(c, 50); }; c(); });
|
||||
try { await startMic(); } catch (e) { log($("chat-log"), `⚠ 麦克风失败: ${esc(e.message)}`, "err"); return; }
|
||||
ws.send(JSON.stringify({ type: "start", asr: $("model").value }));
|
||||
ws.send(JSON.stringify({ type: "start", asr: $("model").value, tts: ttsConfig() }));
|
||||
mic.className = "on"; mic.textContent = "⏹ 停止";
|
||||
$("asr-badge").textContent = "listening";
|
||||
log($("asr-log"), "", "sys").remove();
|
||||
|
||||
Reference in New Issue
Block a user