feat: Phase 2 — Audio STT/TTS
- TTS: edge-tts 7.2.8 (fr-FR-DeniseNeural) — génération MP3 asynchrone - STT: Web Speech API navigateur (SpeechRecognition fr-FR) - Audio playback: base64 MP3 décodé et joué via Audio API - Bouton micro avec animation pulse pendant l'écoute - Chemin absolu edge-tts (systemd PATH) - Audio envoyé dans le message 'response' existant (champ audio) Coût: latence TTS ~7s, zéro dépendance serveur pour le STT.
This commit is contained in:
+71
-9
@@ -6,10 +6,12 @@ PoC minimal : WebSocket texte → API Hermes → réponse texte.
|
||||
Pas de Pipecat — intégration directe pour valider le concept.
|
||||
"""
|
||||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import httpx
|
||||
from websockets.asyncio.server import serve
|
||||
from websockets.http11 import Response, Headers
|
||||
@@ -42,6 +44,33 @@ SYSTEM_PROMPT = (
|
||||
"Pas de markdown. Réponds comme si tu parlais à l'oral."
|
||||
)
|
||||
|
||||
# ─── TTS (Text-to-Speech) ─────────────────────────────────────
|
||||
EDGE_TTS_VOICE = "fr-FR-DeniseNeural" # Voix féminine française naturelle
|
||||
EDGE_TTS_BIN = "/home/openclaw/workspace/jarvis-pipecat/venv/bin/edge-tts"
|
||||
|
||||
async def generate_tts(text: str) -> bytes:
|
||||
"""Génère un MP3 à partir du texte avec edge-tts. Retourne les bytes audio."""
|
||||
with tempfile.NamedTemporaryFile(suffix=".mp3", delete=False) as f:
|
||||
mp3_path = f.name
|
||||
try:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
EDGE_TTS_BIN, "--voice", EDGE_TTS_VOICE,
|
||||
"--text", text, "--write-media", mp3_path,
|
||||
stdout=asyncio.subprocess.DEVNULL,
|
||||
stderr=asyncio.subprocess.DEVNULL,
|
||||
)
|
||||
await asyncio.wait_for(proc.wait(), timeout=15)
|
||||
with open(mp3_path, "rb") as f:
|
||||
return f.read()
|
||||
except Exception as e:
|
||||
print(f"⚠️ TTS error: {e}")
|
||||
return b""
|
||||
finally:
|
||||
try:
|
||||
os.unlink(mp3_path)
|
||||
except:
|
||||
pass
|
||||
|
||||
# ─── Page HTML ───────────────────────────────────────────────
|
||||
HTML_PAGE = """<!DOCTYPE html>
|
||||
<html lang="fr">
|
||||
@@ -71,6 +100,9 @@ button{padding:10px 20px;border-radius:8px;border:none;background:var(--cyan);
|
||||
color:var(--bg);font-weight:bold;cursor:pointer;font-size:.95em}
|
||||
button:hover{opacity:.9}
|
||||
button:disabled{opacity:.5;cursor:default}
|
||||
#mic{width:40px;height:40px;font-size:1.2em;flex-shrink:0;padding:0;display:flex;align-items:center;justify-content:center;border-radius:50%}
|
||||
#mic.listening{background:var(--red);animation:pulse 1s infinite}
|
||||
@keyframes pulse{0%,100%{box-shadow:0 0 0 0 var(--red)}50%{box-shadow:0 0 0 8px transparent}}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
@@ -78,14 +110,17 @@ button:disabled{opacity:.5;cursor:default}
|
||||
<div id="status" class="off">🔴 Déconnecté</div>
|
||||
<div id="chat"></div>
|
||||
<div class="row">
|
||||
<input id="inp" placeholder="Pose ta question..." autofocus>
|
||||
<input id="inp" placeholder="Pose ta question ou parle...">
|
||||
<button id="mic" title="Parler">🎤</button>
|
||||
<button id="btn">Envoyer</button>
|
||||
</div>
|
||||
<script>
|
||||
const s=document.getElementById('status'),c=document.getElementById('chat'),
|
||||
i=document.getElementById('inp'),b=document.getElementById('btn');
|
||||
let ws=null,recon=0,reconMax=20,reconDelay=2000;
|
||||
function m(t,r){const d=document.createElement('div');d.className='msg '+r;d.textContent=t;c.appendChild(d);c.scrollTop=c.scrollHeight}
|
||||
i=document.getElementById('inp'),b=document.getElementById('btn'),
|
||||
mic=document.getElementById('mic');
|
||||
let ws=null,recon=0,reconMax=20,reconDelay=2000,audioCtx=null;
|
||||
|
||||
function addMsg(t,r){const d=document.createElement('div');d.className='msg '+r;d.textContent=t;c.appendChild(d);c.scrollTop=c.scrollHeight}
|
||||
function connect(){ws=new WebSocket((location.protocol==='https:'?'wss:':'ws:')+'//'+location.host+'/ws');
|
||||
ws.onopen=()=>{recon=0;reconDelay=2000;s.textContent='🟢 Connecté — JARVIS v3 prêt';s.className='on';b.disabled=false;i.focus()};
|
||||
ws.onclose=()=>{b.disabled=true;if(recon<reconMax){recon++;let d=Math.min(reconDelay*Math.pow(1.5,recon-1),30000);
|
||||
@@ -93,12 +128,34 @@ s.textContent='🟡 Reconnexion '+recon+'/'+reconMax+' dans '+Math.round(d/1000)
|
||||
else{s.textContent='🔴 Connexion perdue — recharge la page';s.className='off'}};
|
||||
ws.onerror=()=>ws.close();
|
||||
ws.onmessage=e=>{try{let d=JSON.parse(e.data);if(ws._tid){clearTimeout(ws._tid);ws._tid=null}
|
||||
if(d.type==='welcome')m(d.content,'sys');
|
||||
else if(d.type==='response')m(d.content,'jarvis');
|
||||
if(d.type==='welcome')addMsg(d.content,'sys');
|
||||
else if(d.type==='response'){addMsg(d.content,'jarvis');if(d.audio)playAudio(d.audio)}
|
||||
else if(d.type==='thinking')s.textContent='🤔 JARVIS réfléchit...';
|
||||
else if(d.type==='error')m('❌ '+d.content,'sys')}catch(x){m(e.data,'jarvis')};
|
||||
s.textContent='🟢 Connecté — JARVIS v3 prêt';b.disabled=false}}
|
||||
function send(){let t=i.value.trim();if(!t||!ws||ws.readyState!==WebSocket.OPEN)return;m(t,'user');ws.send(JSON.stringify({text:t}));i.value='';b.disabled=true;s.textContent='🤔 JARVIS réfléchit...';let tid=setTimeout(()=>{if(b.disabled){b.disabled=false;s.textContent='⚠️ JARVIS prend du temps... tu peux réessayer'}},25000);ws._tid=tid}
|
||||
else if(d.type==='error')addMsg('❌ '+d.content,'sys')}catch(x){addMsg(e.data,'jarvis')};
|
||||
s.textContent='🟢 Connecté — JARVIS v3 prêt';b.disabled=false;mic.disabled=false}}
|
||||
|
||||
function send(){let t=i.value.trim();if(!t||!ws||ws.readyState!==WebSocket.OPEN)return;addMsg(t,'user');ws.send(JSON.stringify({text:t}));i.value='';b.disabled=true;mic.disabled=true;s.textContent='🤔 JARVIS réfléchit...';let tid=setTimeout(()=>{if(b.disabled){b.disabled=false;mic.disabled=false;s.textContent='⚠️ JARVIS prend du temps... tu peux réessayer'}},25000);ws._tid=tid}
|
||||
|
||||
// ── Audio playback ──
|
||||
function playAudio(b64){try{const bin=atob(b64);const bytes=new Uint8Array(bin.length);for(let i=0;i<bin.length;i++)bytes[i]=bin.charCodeAt(i);const blob=new Blob([bytes],{type:'audio/mp3'});const url=URL.createObjectURL(blob);const a=new Audio(url);a.onended=()=>URL.revokeObjectURL(url);a.play().catch(e=>console.log('Audio play:',e))}catch(e){console.log('Audio error:',e)}}
|
||||
|
||||
// ── Speech Recognition ──
|
||||
const SpeechRecognition=window.SpeechRecognition||window.webkitSpeechRecognition;
|
||||
let rec=null,listening=false;
|
||||
if(SpeechRecognition){
|
||||
rec=new SpeechRecognition();rec.lang='fr-FR';rec.interimResults=false;rec.continuous=false;
|
||||
rec.onresult=e=>{const t=e.results[0][0].transcript.trim();if(t){i.value=t;send()}};
|
||||
rec.onerror=e=>{console.log('STT error:',e.error);stopListening()};
|
||||
rec.onend=()=>stopListening();
|
||||
}
|
||||
function toggleMic(){
|
||||
if(!rec){alert('Reconnaissance vocale non supportée sur ce navigateur');return}
|
||||
if(listening){rec.stop();return}
|
||||
listening=true;mic.classList.add('listening');mic.textContent='🔴';s.textContent='🎤 Je t\'écoute...';
|
||||
try{rec.start()}catch(e){stopListening()}
|
||||
}
|
||||
function stopListening(){listening=false;mic.classList.remove('listening');mic.textContent='🎤';if(s.textContent.includes('écoute'))s.textContent='🟢 Connecté — JARVIS v3 prêt'}
|
||||
mic.onclick=toggleMic;
|
||||
b.onclick=send;i.onkeydown=e=>{if(e.key==='Enter')send()};connect()
|
||||
</script>
|
||||
</body>
|
||||
@@ -177,9 +234,14 @@ async def ws_handler(websocket):
|
||||
messages.append({"role": "assistant", "content": reply})
|
||||
print(f"🤖 {reply[:80]}")
|
||||
|
||||
# Générer TTS en parallèle de l'envoi texte
|
||||
audio_bytes = await generate_tts(reply)
|
||||
audio_b64 = base64.b64encode(audio_bytes).decode() if audio_bytes else ""
|
||||
|
||||
await websocket.send(json.dumps({
|
||||
"type": "response",
|
||||
"content": reply,
|
||||
"audio": audio_b64,
|
||||
}))
|
||||
|
||||
except Exception as e:
|
||||
|
||||
Reference in New Issue
Block a user