feat: improve voice controls and interruption
This commit is contained in:
+190
-11
@@ -7,24 +7,40 @@
|
||||
<style>
|
||||
:root { --bg:#0f1117; --panel:#1a1d27; --line:#2a2e3d; --fg:#e6e8f0; --dim:#8b90a5; --acc:#4f7cff; --ok:#3fbf6f; --warn:#e0a23c; --err:#e05c5c; }
|
||||
* { box-sizing:border-box; margin:0; padding:0; }
|
||||
body { background:var(--bg); color:var(--fg); font:14px/1.6 "PingFang SC","Microsoft YaHei",system-ui,sans-serif; min-height:100vh; display:flex; flex-direction:column; }
|
||||
html, body { height:100%; overflow:hidden; }
|
||||
body { background:var(--bg); color:var(--fg); font:14px/1.6 "PingFang SC","Microsoft YaHei",system-ui,sans-serif; display:flex; flex-direction:column; }
|
||||
header { padding:14px 22px; border-bottom:1px solid var(--line); display:flex; align-items:center; gap:14px; flex-wrap:wrap; }
|
||||
header h1 { font-size:16px; font-weight:600; }
|
||||
header .sel { display:flex; align-items:center; gap:8px; margin-left:auto; }
|
||||
select, button { background:var(--panel); color:var(--fg); border:1px solid var(--line); border-radius:8px; padding:8px 14px; font-size:14px; cursor:pointer; }
|
||||
select, button, input { background:var(--panel); color:var(--fg); border:1px solid var(--line); border-radius:8px; padding:8px 12px; font-size:14px; }
|
||||
select, button { cursor:pointer; }
|
||||
select { min-width:280px; }
|
||||
.config { padding:10px 22px; border-bottom:1px solid var(--line); background:#141720; max-height:45vh; overflow-y:auto; flex:none; }
|
||||
.config summary { cursor:pointer; color:var(--dim); user-select:none; }
|
||||
.config-grid { display:grid; grid-template-columns:repeat(5,minmax(130px,1fr)); gap:10px 14px; margin-top:10px; }
|
||||
.config-grid label { display:flex; flex-direction:column; gap:4px; color:var(--dim); font-size:12px; }
|
||||
.config-grid label.wide { grid-column:span 2; }
|
||||
.config-grid input[type="range"] { padding:0; accent-color:var(--acc); }
|
||||
.config-grid select { min-width:0; width:100%; }
|
||||
.range-title { display:flex; justify-content:space-between; }
|
||||
.hint { color:var(--dim); font-size:11px; margin-top:6px; }
|
||||
.recorder { display:flex; align-items:center; gap:10px; flex-wrap:wrap; margin-top:10px; padding-top:10px; border-top:1px solid var(--line); }
|
||||
#record.on { background:var(--err); color:#fff; }
|
||||
#recording-audio { height:34px; max-width:320px; }
|
||||
#recording-download { color:#8fb0ff; text-decoration:none; }
|
||||
@media (max-width:900px){ .config-grid{grid-template-columns:1fr 1fr;} .config-grid label.wide{grid-column:span 2;} }
|
||||
button:hover:not(:disabled) { border-color:var(--acc); }
|
||||
button:disabled { opacity:.4; cursor:not-allowed; }
|
||||
#mic { border-color:transparent; color:#fff; font-weight:600; min-width:96px; }
|
||||
#mic.on { background:var(--err); animation:pulse 1.2s infinite; }
|
||||
#mic.off { background:var(--acc); }
|
||||
@keyframes pulse { 0%,100%{box-shadow:0 0 0 0 rgba(224,92,92,.5)} 50%{box-shadow:0 0 0 10px rgba(224,92,92,0)} }
|
||||
main { flex:1; display:grid; grid-template-columns:1fr 1fr; gap:14px; padding:14px 22px; max-width:1200px; width:100%; margin:0 auto; }
|
||||
@media (max-width:860px){ main{grid-template-columns:1fr;} }
|
||||
.panel { background:var(--panel); border:1px solid var(--line); border-radius:12px; display:flex; flex-direction:column; min-height:320px; }
|
||||
main { flex:1; min-height:0; overflow:hidden; display:grid; grid-template-columns:1fr 1fr; grid-template-rows:minmax(0,1fr); gap:14px; padding:14px 22px; max-width:1200px; width:100%; margin:0 auto; }
|
||||
@media (max-width:860px){ main{grid-template-columns:1fr; grid-template-rows:repeat(2,minmax(0,1fr));} }
|
||||
.panel { background:var(--panel); border:1px solid var(--line); border-radius:12px; display:flex; flex-direction:column; min-height:0; overflow:hidden; }
|
||||
.panel h2 { font-size:13px; color:var(--dim); padding:10px 14px; border-bottom:1px solid var(--line); font-weight:500; display:flex; justify-content:space-between; }
|
||||
.panel h2 .badge { font-size:11px; padding:1px 8px; border-radius:6px; background:#262b3b; }
|
||||
.log { flex:1; overflow-y:auto; padding:12px 14px; display:flex; flex-direction:column; gap:10px; }
|
||||
.log { flex:1; min-height:0; overflow-y:auto; overscroll-behavior:contain; padding:12px 14px; display:flex; flex-direction:column; gap:10px; }
|
||||
.msg { padding:8px 12px; border-radius:10px; max-width:88%; white-space:pre-wrap; word-break:break-all; }
|
||||
.msg .who { font-size:11px; color:var(--dim); margin-bottom:2px; }
|
||||
.msg.user { align-self:flex-end; background:#25315e; border-top-right-radius:3px; }
|
||||
@@ -47,6 +63,26 @@
|
||||
<button id="mic" class="off">🎤 开始</button>
|
||||
</div>
|
||||
</header>
|
||||
<details class="config" open>
|
||||
<summary>TTS 声音配置</summary>
|
||||
<div class="config-grid">
|
||||
<label>TTS 模型<select id="tts-model"></select></label>
|
||||
<label>预置音色<select id="tts-system-voice"></select></label>
|
||||
<label class="wide">复刻 / 设计音色<select id="tts-cloned-voice"><option value="">请选择复刻音色</option></select></label>
|
||||
<label><span class="range-title">语速 <b id="rate-value">1.0×</b></span><input id="tts-rate" type="range" min="0.5" max="2" step="0.1" value="1"></label>
|
||||
<label><span class="range-title">语调 <b id="pitch-value">1.0×</b></span><input id="tts-pitch" type="range" min="0.5" max="2" step="0.1" value="1"></label>
|
||||
<label><span class="range-title">音量 <b id="volume-value">50</b></span><input id="tts-volume" type="range" min="0" max="100" step="1" value="50"></label>
|
||||
<label class="wide">声音表达 / 模仿描述<input id="tts-instruction" maxlength="100" placeholder="如：温柔沉稳、略带微笑、语速自然"></label>
|
||||
</div>
|
||||
<div class="hint">v3.5 仅支持复刻/设计音色；复刻音色从当前百炼账号自动读取。<span id="voice-status"></span></div>
|
||||
<div class="recorder">
|
||||
<button id="record">⏺ 录制复刻样本</button>
|
||||
<b id="record-time">00:00</b>
|
||||
<audio id="recording-audio" controls hidden></audio>
|
||||
<a id="recording-download" download="voice-sample.wav" hidden>下载 WAV</a>
|
||||
<span class="hint">建议安静环境录制 5~20 秒连续清晰人声，最长 30 秒。</span>
|
||||
</div>
|
||||
</details>
|
||||
<main>
|
||||
<div class="panel">
|
||||
<h2>识别（ASR） <span class="badge" id="asr-badge">idle</span></h2>
|
||||
@@ -62,7 +98,7 @@
|
||||
<span>延迟: <b id="lat">-</b></span>
|
||||
<span id="vu"><i></i><i></i><i></i><i></i><i></i></span>
|
||||
</div>
|
||||
<div class="footer-tip">浏览器采集 PCM16/16k 单声道，实时上行；每句判定结束后自动由客服 LLM 流式回答并 TTS 播报。Chrome 要求 localhost 或 HTTPS 才能使用麦克风。</div>
|
||||
<div class="footer-tip">浏览器采集 PCM16/16k 单声道，实时上行；每句判定结束后自动由客服 LLM 流式回答并按当前音色配置播报。Chrome 要求 localhost 或 HTTPS 才能使用麦克风。</div>
|
||||
|
||||
<script>
|
||||
"use strict";
|
||||
@@ -91,14 +127,129 @@ fetch("/api/models").then(r => r.json()).then(models => {
|
||||
}
|
||||
});
|
||||
|
||||
let systemVoices = {};
|
||||
fetch("/api/config").then(r => r.json()).then(({tts, ttsModels, systemVoices: voices, clonedVoices, voiceError}) => {
|
||||
systemVoices = voices;
|
||||
fillSelect($("tts-model"), ttsModels);
|
||||
$("tts-model").value = tts.model;
|
||||
fillSelect($("tts-cloned-voice"), clonedVoices, "请选择复刻/设计音色");
|
||||
const isSystem = renderSystemVoices(tts.voice);
|
||||
if (!isSystem) $("tts-cloned-voice").value = tts.voice;
|
||||
$("tts-instruction").value = tts.instruction;
|
||||
$("tts-rate").value = tts.rate;
|
||||
$("tts-pitch").value = tts.pitch;
|
||||
$("tts-volume").value = tts.volume;
|
||||
if (voiceError) $("voice-status").textContent = `（读取失败：${voiceError}）`;
|
||||
updateRangeLabels();
|
||||
});
|
||||
|
||||
function fillSelect(select, options, placeholder = "") {
|
||||
select.replaceChildren();
|
||||
if (placeholder) select.add(new Option(placeholder, ""));
|
||||
for (const o of options || []) select.add(new Option(o.label, o.id));
|
||||
}
|
||||
|
||||
function renderSystemVoices(selected = "") {
|
||||
const options = systemVoices[$("tts-model").value] || [];
|
||||
fillSelect($("tts-system-voice"), options, options.length ? "请选择预置音色" : "该模型无预置音色");
|
||||
if (options.some(v => v.id === selected)) {
|
||||
$("tts-system-voice").value = selected;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
$("tts-model").onchange = () => renderSystemVoices();
|
||||
$("tts-system-voice").onchange = () => { if ($("tts-system-voice").value) $("tts-cloned-voice").value = ""; };
|
||||
$("tts-cloned-voice").onchange = () => { if ($("tts-cloned-voice").value) $("tts-system-voice").value = ""; };
|
||||
|
||||
function updateRangeLabels() {
|
||||
$("rate-value").textContent = Number($("tts-rate").value).toFixed(1) + "×";
|
||||
$("pitch-value").textContent = Number($("tts-pitch").value).toFixed(1) + "×";
|
||||
$("volume-value").textContent = $("tts-volume").value;
|
||||
}
|
||||
["tts-rate", "tts-pitch", "tts-volume"].forEach(id => $(id).oninput = updateRangeLabels);
|
||||
|
||||
function ttsConfig() {
|
||||
return {
|
||||
model: $("tts-model").value,
|
||||
voice: $("tts-cloned-voice").value || $("tts-system-voice").value,
|
||||
instruction: $("tts-instruction").value,
|
||||
rate: Number($("tts-rate").value),
|
||||
pitch: Number($("tts-pitch").value),
|
||||
volume: Number($("tts-volume").value)
|
||||
};
|
||||
}
|
||||
|
||||
// ---------- 复刻样本录制（单声道 PCM WAV） ----------
|
||||
let recordStream = null, recordCtx = null, recordNode = null, recordChunks = [], recordStarted = 0, recordTimer = null, recordURL = "";
|
||||
|
||||
$("record").onclick = async () => {
|
||||
if (recordStream) { stopRecording(); return; }
|
||||
if ($("mic").classList.contains("on")) {
|
||||
log($("chat-log"), "⚠ 请先停止 ASR 会话", "err");
|
||||
return;
|
||||
}
|
||||
try {
|
||||
recordStream = await navigator.mediaDevices.getUserMedia({audio:{channelCount:1, echoCancellation:false, noiseSuppression:false}});
|
||||
recordCtx = new AudioContext();
|
||||
const source = recordCtx.createMediaStreamSource(recordStream);
|
||||
recordNode = recordCtx.createScriptProcessor(4096, 1, 1);
|
||||
recordNode.onaudioprocess = e => recordChunks.push(new Float32Array(e.inputBuffer.getChannelData(0)));
|
||||
const silent = recordCtx.createGain(); silent.gain.value = 0;
|
||||
source.connect(recordNode); recordNode.connect(silent); silent.connect(recordCtx.destination);
|
||||
recordStarted = Date.now();
|
||||
$("record").className = "on"; $("record").textContent = "⏹ 停止录制";
|
||||
recordTimer = setInterval(() => {
|
||||
const seconds = Math.floor((Date.now() - recordStarted) / 1000);
|
||||
$("record-time").textContent = `00:${String(seconds).padStart(2, "0")}`;
|
||||
if (seconds >= 30) stopRecording();
|
||||
}, 250);
|
||||
} catch (e) {
|
||||
recordStream = null;
|
||||
log($("chat-log"), `⚠ 录音失败: ${esc(e.message)}`, "err");
|
||||
}
|
||||
};
|
||||
|
||||
function stopRecording() {
|
||||
clearInterval(recordTimer);
|
||||
const sampleRate = recordCtx.sampleRate;
|
||||
recordNode.disconnect();
|
||||
recordStream.getTracks().forEach(t => t.stop());
|
||||
recordCtx.close();
|
||||
recordStream = recordCtx = recordNode = null;
|
||||
$("record").className = ""; $("record").textContent = "⏺ 重新录制";
|
||||
if (recordURL) URL.revokeObjectURL(recordURL);
|
||||
recordURL = URL.createObjectURL(wavBlob(recordChunks, sampleRate));
|
||||
recordChunks = [];
|
||||
$("recording-audio").src = recordURL; $("recording-audio").hidden = false;
|
||||
$("recording-download").href = recordURL; $("recording-download").hidden = false;
|
||||
}
|
||||
|
||||
function wavBlob(chunks, sampleRate) {
|
||||
const samples = chunks.reduce((n, c) => n + c.length, 0);
|
||||
const buffer = new ArrayBuffer(44 + samples * 2), view = new DataView(buffer);
|
||||
const text = (offset, value) => [...value].forEach((c, i) => view.setUint8(offset + i, c.charCodeAt(0)));
|
||||
text(0, "RIFF"); view.setUint32(4, 36 + samples * 2, true); text(8, "WAVEfmt ");
|
||||
view.setUint32(16, 16, true); view.setUint16(20, 1, true); view.setUint16(22, 1, true);
|
||||
view.setUint32(24, sampleRate, true); view.setUint32(28, sampleRate * 2, true);
|
||||
view.setUint16(32, 2, true); view.setUint16(34, 16, true); text(36, "data"); view.setUint32(40, samples * 2, true);
|
||||
let offset = 44;
|
||||
for (const chunk of chunks) for (const value of chunk) {
|
||||
const s = Math.max(-1, Math.min(1, value));
|
||||
view.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7fff, true); offset += 2;
|
||||
}
|
||||
return new Blob([buffer], {type:"audio/wav"});
|
||||
}
|
||||
|
||||
// ---------- WebSocket ----------
|
||||
function connectWS() {
|
||||
ws = new WebSocket(`ws://${location.host}/agent`);
|
||||
ws.binaryType = "arraybuffer";
|
||||
ws.onopen = () => $("st").textContent = "已连接";
|
||||
ws.onclose = () => { $("st").textContent = "连接断开"; stopMic(); };
|
||||
ws.onclose = () => { $("st").textContent = "连接断开"; stopMic(); stopPlayback(); };
|
||||
ws.onmessage = ev => {
|
||||
if (ev.data instanceof ArrayBuffer) { playPcm(new Int16Array(ev.data)); return; }
|
||||
if (ev.data instanceof ArrayBuffer) { if (playEnabled) playPcm(new Int16Array(ev.data)); return; }
|
||||
const m = JSON.parse(ev.data);
|
||||
switch (m.type) {
|
||||
case "partial":
|
||||
@@ -113,6 +264,7 @@ function connectWS() {
|
||||
break;
|
||||
}
|
||||
case "reply-delta":
|
||||
playEnabled = true;
|
||||
if (!replyEl) { replyText = ""; replyEl = log($("chat-log"), "", "bot"); }
|
||||
replyText += m.text;
|
||||
replyEl.innerHTML = `<div class="who">客服</div>${esc(replyText)}`;
|
||||
@@ -126,6 +278,11 @@ function connectWS() {
|
||||
case "tts-end":
|
||||
$("tts-badge").textContent = "idle";
|
||||
break;
|
||||
case "interrupt":
|
||||
stopPlayback();
|
||||
replyEl = null; replyText = "";
|
||||
$("tts-badge").textContent = "interrupted";
|
||||
break;
|
||||
case "asr-stopped":
|
||||
$("asr-badge").textContent = "idle";
|
||||
break;
|
||||
@@ -177,13 +334,16 @@ function stopMic() {
|
||||
}
|
||||
|
||||
// ---------- TTS 播放（PCM16/16k） ----------
|
||||
let playCtx = null, playHead = 0;
|
||||
let playCtx = null, playHead = 0, playEnabled = false;
|
||||
const playSources = new Set();
|
||||
function playPcm(pcm) {
|
||||
if (!playCtx || playCtx.state === "closed") { playCtx = new AudioContext({ sampleRate: 16000 }); playHead = 0; }
|
||||
playCtx.resume();
|
||||
const buf = playCtx.createBuffer(1, pcm.length, 16000);
|
||||
buf.getChannelData(0).set(Float32Array.from(pcm, v => v / 32768));
|
||||
const src = playCtx.createBufferSource();
|
||||
playSources.add(src);
|
||||
src.onended = () => playSources.delete(src);
|
||||
src.buffer = buf; src.connect(playCtx.destination);
|
||||
const now = playCtx.currentTime;
|
||||
if (playHead < now + 0.05) playHead = now + 0.05; // 游标滞后（播完已久/首块）则贴到当前时刻
|
||||
@@ -191,18 +351,37 @@ function playPcm(pcm) {
|
||||
playHead += buf.duration;
|
||||
}
|
||||
|
||||
function stopPlayback() {
|
||||
playEnabled = false;
|
||||
for (const src of playSources) {
|
||||
try { src.stop(); } catch (_) {}
|
||||
}
|
||||
playSources.clear();
|
||||
playHead = playCtx ? playCtx.currentTime : 0;
|
||||
}
|
||||
|
||||
// ---------- 按钮 ----------
|
||||
$("mic").onclick = async () => {
|
||||
const mic = $("mic");
|
||||
if (mic.classList.contains("on")) { // 停止
|
||||
ws && ws.send(JSON.stringify({ type: "stop" }));
|
||||
stopPlayback();
|
||||
stopMic();
|
||||
return;
|
||||
}
|
||||
if (recordStream) {
|
||||
log($("chat-log"), "⚠ 请先停止样本录制", "err");
|
||||
return;
|
||||
}
|
||||
if (!ttsConfig().voice) {
|
||||
log($("chat-log"), "⚠ 请先选择预置音色或复刻音色", "err");
|
||||
$("tts-cloned-voice").focus();
|
||||
return;
|
||||
}
|
||||
if (!ws || ws.readyState !== 1) connectWS();
|
||||
await new Promise(r => { const c = () => { if (ws.readyState === 1) r(); else setTimeout(c, 50); }; c(); });
|
||||
try { await startMic(); } catch (e) { log($("chat-log"), `⚠ 麦克风失败: ${esc(e.message)}`, "err"); return; }
|
||||
ws.send(JSON.stringify({ type: "start", asr: $("model").value }));
|
||||
ws.send(JSON.stringify({ type: "start", asr: $("model").value, tts: ttsConfig() }));
|
||||
mic.className = "on"; mic.textContent = "⏹ 停止";
|
||||
$("asr-badge").textContent = "listening";
|
||||
log($("asr-log"), "", "sys").remove();
|
||||
|
||||
Reference in New Issue
Block a user