235 lines
8.7 KiB
Go
235 lines
8.7 KiB
Go
package ai
|
|
|
|
// DashScope's task-based Fun-ASR/CosyVoice protocol is not the Qwen realtime
|
|
// session protocol. The inspected Go SDKs do not support this protocol with
|
|
// per-call approved endpoints and strict started/finished/error boundaries.
|
|
// Keep this adapter small; WebSocket framing is handled by the existing library.
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"log/slog"
|
|
"net/http"
|
|
"net/url"
|
|
"strconv"
|
|
"strings"
|
|
|
|
"github.com/coder/websocket"
|
|
"github.com/google/uuid"
|
|
)
|
|
|
|
type bailianEvent struct {
|
|
Header struct {
|
|
Event string `json:"event"`
|
|
TaskID string `json:"task_id"`
|
|
ErrorCode string `json:"error_code"`
|
|
} `json:"header"`
|
|
Payload struct {
|
|
Output struct {
|
|
Sentence struct {
|
|
Text string `json:"text"`
|
|
Final bool `json:"sentence_end"`
|
|
Begin *int64 `json:"begin_time"`
|
|
End *int64 `json:"end_time"`
|
|
} `json:"sentence"`
|
|
} `json:"output"`
|
|
} `json:"payload"`
|
|
}
|
|
|
|
type bailianTask struct {
|
|
conn *websocket.Conn
|
|
id string
|
|
}
|
|
|
|
func (t bailianTask) command(ctx context.Context, action string, payload any) error {
|
|
body, err := json.Marshal(map[string]any{"header": map[string]any{"action": action, "task_id": t.id, "streaming": "duplex"}, "payload": payload})
|
|
if err != nil {
|
|
return errors.New("encode Bailian task command")
|
|
}
|
|
if err = t.conn.Write(ctx, websocket.MessageText, body); err != nil {
|
|
return fmt.Errorf("Bailian task command delivery failed (cause=%T, context=%v)", err, ctx.Err())
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func runBailianTask(ctx context.Context, endpoint, key string, payload any, send func(bailianTask) error, receive func(websocket.MessageType, []byte, bailianEvent) error) error {
|
|
u, err := url.Parse(endpoint)
|
|
if err != nil || u.Host == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" || (u.Scheme != "wss" && u.Scheme != "ws") || key == "" || strings.ContainsAny(key, "\r\n") {
|
|
return errors.New("approved Bailian WebSocket connection is invalid")
|
|
}
|
|
client := &http.Client{CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}
|
|
conn, response, err := websocket.Dial(ctx, endpoint, &websocket.DialOptions{HTTPClient: client, HTTPHeader: http.Header{"Authorization": []string{"Bearer " + key}}})
|
|
if err != nil {
|
|
if response != nil {
|
|
return fmt.Errorf("Bailian WebSocket handshake rejected: HTTP %d", response.StatusCode)
|
|
}
|
|
return fmt.Errorf("Bailian WebSocket handshake failed (cause=%T, context=%v)", err, ctx.Err())
|
|
}
|
|
defer conn.CloseNow()
|
|
conn.SetReadLimit(maxBailianPCMBytes)
|
|
task := bailianTask{conn: conn, id: uuid.NewString()}
|
|
if err = task.command(ctx, "run-task", payload); err != nil {
|
|
return err
|
|
}
|
|
read := func() (websocket.MessageType, []byte, bailianEvent, error) {
|
|
kind, body, e := conn.Read(ctx)
|
|
if e != nil {
|
|
return kind, nil, bailianEvent{}, fmt.Errorf("Bailian task %s ended without confirmed completion (cause=%T, close_status=%d)", task.id, e, websocket.CloseStatus(e))
|
|
}
|
|
var event bailianEvent
|
|
if kind == websocket.MessageText {
|
|
if json.Unmarshal(body, &event) != nil || event.Header.TaskID != task.id {
|
|
return kind, nil, event, errors.New("Bailian event is malformed or belongs to another task")
|
|
}
|
|
if event.Header.Event == "task-failed" {
|
|
code := event.Header.ErrorCode
|
|
if len(code) == 0 || len(code) > 64 || strings.Contains(code, key) || strings.IndexFunc(code, func(r rune) bool {
|
|
return !(r >= 'A' && r <= 'Z' || r >= 'a' && r <= 'z' || r >= '0' && r <= '9' || r == '_' || r == '-')
|
|
}) >= 0 {
|
|
code = "unavailable"
|
|
}
|
|
return kind, nil, event, fmt.Errorf("Bailian task %s explicitly failed (code=%s)", task.id, code)
|
|
}
|
|
} else if kind != websocket.MessageBinary {
|
|
return kind, nil, event, errors.New("Bailian task returned an unsupported frame")
|
|
}
|
|
return kind, body, event, nil
|
|
}
|
|
kind, body, event, err := read()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if kind != websocket.MessageText || event.Header.Event != "task-started" {
|
|
return errors.New("Bailian did not confirm task start before input")
|
|
}
|
|
slog.Debug("Bailian task start confirmed", "task_id", task.id, "stage", "provider_started")
|
|
// Read concurrently with input writes so provider result frames cannot block
|
|
// a large PCM send. Always join the writer, including cancellation/failure.
|
|
sent := make(chan error, 1)
|
|
joined := false
|
|
go func() {
|
|
e := send(bailianTask{conn: conn, id: task.id})
|
|
sent <- e
|
|
if e != nil {
|
|
conn.CloseNow()
|
|
}
|
|
}()
|
|
defer func() {
|
|
conn.CloseNow()
|
|
if !joined {
|
|
<-sent
|
|
}
|
|
}()
|
|
for {
|
|
kind, body, event, err = read()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if kind == websocket.MessageText {
|
|
switch event.Header.Event {
|
|
case "result-generated":
|
|
case "task-finished":
|
|
// Server completion is required; no partial audio/transcript is returned.
|
|
inputErr := <-sent
|
|
joined = true
|
|
if inputErr == nil {
|
|
slog.Debug("Bailian task completion confirmed", "task_id", task.id, "stage", "provider_finished")
|
|
}
|
|
return inputErr
|
|
default:
|
|
return errors.New("Bailian task returned an unexpected lifecycle event")
|
|
}
|
|
}
|
|
if err = receive(kind, body, event); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
|
|
func recognizeBailian(ctx context.Context, approved ASRConfig, pcm16 []byte) (string, error) {
|
|
if strings.TrimSpace(approved.Model) == "" {
|
|
return "", errors.New("Bailian task ASR needs an explicit model")
|
|
}
|
|
if len(pcm16) == 0 || len(pcm16)%2 != 0 || len(pcm16) > maxBailianPCMBytes {
|
|
return "", errors.New("Bailian task ASR input is empty, incomplete or oversized PCM16")
|
|
}
|
|
input := pcm16
|
|
payload := map[string]any{"task_group": "audio", "task": "asr", "function": "recognition", "model": approved.Model, "parameters": approved.Params, "input": map[string]any{}}
|
|
var finals []string
|
|
finalBytes := 0
|
|
seen := map[string]string{}
|
|
err := runBailianTask(ctx, approved.Provider.Endpoint, approved.Provider.Credential, payload, func(t bailianTask) error {
|
|
for off := 0; off < len(input); off += 3200 {
|
|
if err := t.conn.Write(ctx, websocket.MessageBinary, input[off:min(off+3200, len(input))]); err != nil {
|
|
return errors.New("Bailian task ASR audio delivery failed")
|
|
}
|
|
}
|
|
return t.command(ctx, "finish-task", map[string]any{"input": map[string]any{}})
|
|
}, func(kind websocket.MessageType, _ []byte, event bailianEvent) error {
|
|
if kind != websocket.MessageText {
|
|
return errors.New("Bailian task ASR returned binary output")
|
|
}
|
|
s := event.Payload.Output.Sentence
|
|
if !s.Final {
|
|
return nil
|
|
}
|
|
if s.Begin == nil || s.End == nil || *s.End < *s.Begin || strings.TrimSpace(s.Text) == "" {
|
|
return errors.New("Bailian task ASR returned an incomplete final sentence")
|
|
}
|
|
id := strconv.FormatInt(*s.Begin, 10) + ":" + strconv.FormatInt(*s.End, 10)
|
|
if old, ok := seen[id]; ok {
|
|
if old != s.Text {
|
|
return errors.New("Bailian task ASR changed an already final sentence")
|
|
}
|
|
return nil
|
|
}
|
|
if len(finals) >= 1024 || len(s.Text) > maxBailianPCMBytes-finalBytes {
|
|
return errors.New("Bailian task ASR final text exceeds per-turn protocol bound")
|
|
}
|
|
finalBytes += len(s.Text)
|
|
seen[id] = s.Text
|
|
finals = append(finals, s.Text)
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if len(finals) == 0 {
|
|
return "", errors.New("Bailian task ASR finished without final user text")
|
|
}
|
|
return strings.Join(finals, ""), nil
|
|
}
|
|
|
|
func synthesizeBailianTaskTTS(ctx context.Context, approved TTSConfig, text string) ([]byte, error) {
|
|
if approved.Protocol != TTSProtocolDashScopeTask || strings.TrimSpace(approved.Model) == "" || strings.TrimSpace(approved.Voice) == "" || strings.TrimSpace(text) == "" {
|
|
return nil, errors.New("unapproved Bailian task TTS settings or empty text")
|
|
}
|
|
parameters := cloneParams(approved.Params)
|
|
parameters["voice"], _ = json.Marshal(approved.Voice)
|
|
payload := map[string]any{"task_group": "audio", "task": "tts", "function": "SpeechSynthesizer", "model": approved.Model, "parameters": parameters, "input": map[string]any{}}
|
|
var audio []byte
|
|
err := runBailianTask(ctx, approved.Provider.WSEndpoint, approved.Provider.Credential, payload, func(t bailianTask) error {
|
|
if err := t.command(ctx, "continue-task", map[string]any{"input": map[string]any{"text": text}}); err != nil {
|
|
return err
|
|
}
|
|
return t.command(ctx, "finish-task", map[string]any{"input": map[string]any{}})
|
|
}, func(kind websocket.MessageType, body []byte, _ bailianEvent) error {
|
|
if kind == websocket.MessageBinary {
|
|
if len(body) > maxBailianPCMBytes-len(audio) {
|
|
return errors.New("Bailian task TTS audio exceeds approved PCM bound")
|
|
}
|
|
audio = append(audio, body...)
|
|
}
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if len(audio) == 0 || len(audio)%2 != 0 {
|
|
return nil, errors.New("Bailian task TTS finished without complete PCM16 audio")
|
|
}
|
|
slog.Debug("Bailian task TTS complete PCM16 received", "bytes", len(audio))
|
|
return audio, nil
|
|
}
|