feat: Phase 2 — Audio STT/TTS

- TTS: edge-tts 7.2.8 (fr-FR-DeniseNeural) — génération MP3 asynchrone
- STT: Web Speech API navigateur (SpeechRecognition fr-FR)
- Audio playback: base64 MP3 décodé et joué via Audio API
- Bouton micro avec animation pulse pendant l'écoute
- Chemin absolu edge-tts (systemd PATH)
- Audio envoyé dans le message 'response' existant (champ audio)

Coût: latence TTS ~7s, zéro dépendance serveur pour le STT.
This commit is contained in:
2026-06-19 23:39:04 -04:00
parent c8d5d2048c
commit 29855bcd1f
+71 -9
View File
@@ -6,10 +6,12 @@ PoC minimal : WebSocket texte → API Hermes → réponse texte.
Pas de Pipecat — intégration directe pour valider le concept.
"""
import asyncio
import base64
import json
import os
import subprocess
import sys
import tempfile
import httpx
from websockets.asyncio.server import serve
from websockets.http11 import Response, Headers
@@ -42,6 +44,33 @@ SYSTEM_PROMPT = (
"Pas de markdown. Réponds comme si tu parlais à l'oral."
)
# ─── TTS (Text-to-Speech) ─────────────────────────────────────
EDGE_TTS_VOICE = "fr-FR-DeniseNeural" # Voix féminine française naturelle
EDGE_TTS_BIN = "/home/openclaw/workspace/jarvis-pipecat/venv/bin/edge-tts"
async def generate_tts(text: str) -> bytes:
"""Génère un MP3 à partir du texte avec edge-tts. Retourne les bytes audio."""
with tempfile.NamedTemporaryFile(suffix=".mp3", delete=False) as f:
mp3_path = f.name
try:
proc = await asyncio.create_subprocess_exec(
EDGE_TTS_BIN, "--voice", EDGE_TTS_VOICE,
"--text", text, "--write-media", mp3_path,
stdout=asyncio.subprocess.DEVNULL,
stderr=asyncio.subprocess.DEVNULL,
)
await asyncio.wait_for(proc.wait(), timeout=15)
with open(mp3_path, "rb") as f:
return f.read()
except Exception as e:
print(f"⚠️ TTS error: {e}")
return b""
finally:
try:
os.unlink(mp3_path)
except:
pass
# ─── Page HTML ───────────────────────────────────────────────
HTML_PAGE = """<!DOCTYPE html>
<html lang="fr">
@@ -71,6 +100,9 @@ button{padding:10px 20px;border-radius:8px;border:none;background:var(--cyan);
color:var(--bg);font-weight:bold;cursor:pointer;font-size:.95em}
button:hover{opacity:.9}
button:disabled{opacity:.5;cursor:default}
#mic{width:40px;height:40px;font-size:1.2em;flex-shrink:0;padding:0;display:flex;align-items:center;justify-content:center;border-radius:50%}
#mic.listening{background:var(--red);animation:pulse 1s infinite}
@keyframes pulse{0%,100%{box-shadow:0 0 0 0 var(--red)}50%{box-shadow:0 0 0 8px transparent}}
</style>
</head>
<body>
@@ -78,14 +110,17 @@ button:disabled{opacity:.5;cursor:default}
<div id="status" class="off">🔴 Déconnecté</div>
<div id="chat"></div>
<div class="row">
<input id="inp" placeholder="Pose ta question..." autofocus>
<input id="inp" placeholder="Pose ta question ou parle...">
<button id="mic" title="Parler">🎤</button>
<button id="btn">Envoyer</button>
</div>
<script>
const s=document.getElementById('status'),c=document.getElementById('chat'),
i=document.getElementById('inp'),b=document.getElementById('btn');
let ws=null,recon=0,reconMax=20,reconDelay=2000;
function m(t,r){const d=document.createElement('div');d.className='msg '+r;d.textContent=t;c.appendChild(d);c.scrollTop=c.scrollHeight}
i=document.getElementById('inp'),b=document.getElementById('btn'),
mic=document.getElementById('mic');
let ws=null,recon=0,reconMax=20,reconDelay=2000,audioCtx=null;
function addMsg(t,r){const d=document.createElement('div');d.className='msg '+r;d.textContent=t;c.appendChild(d);c.scrollTop=c.scrollHeight}
function connect(){ws=new WebSocket((location.protocol==='https:'?'wss:':'ws:')+'//'+location.host+'/ws');
ws.onopen=()=>{recon=0;reconDelay=2000;s.textContent='🟢 Connecté — JARVIS v3 prêt';s.className='on';b.disabled=false;i.focus()};
ws.onclose=()=>{b.disabled=true;if(recon<reconMax){recon++;let d=Math.min(reconDelay*Math.pow(1.5,recon-1),30000);
@@ -93,12 +128,34 @@ s.textContent='🟡 Reconnexion '+recon+'/'+reconMax+' dans '+Math.round(d/1000)
else{s.textContent='🔴 Connexion perdue — recharge la page';s.className='off'}};
ws.onerror=()=>ws.close();
ws.onmessage=e=>{try{let d=JSON.parse(e.data);if(ws._tid){clearTimeout(ws._tid);ws._tid=null}
if(d.type==='welcome')m(d.content,'sys');
else if(d.type==='response')m(d.content,'jarvis');
if(d.type==='welcome')addMsg(d.content,'sys');
else if(d.type==='response'){addMsg(d.content,'jarvis');if(d.audio)playAudio(d.audio)}
else if(d.type==='thinking')s.textContent='🤔 JARVIS réfléchit...';
else if(d.type==='error')m('❌ '+d.content,'sys')}catch(x){m(e.data,'jarvis')};
s.textContent='🟢 Connecté — JARVIS v3 prêt';b.disabled=false}}
function send(){let t=i.value.trim();if(!t||!ws||ws.readyState!==WebSocket.OPEN)return;m(t,'user');ws.send(JSON.stringify({text:t}));i.value='';b.disabled=true;s.textContent='🤔 JARVIS réfléchit...';let tid=setTimeout(()=>{if(b.disabled){b.disabled=false;s.textContent='⚠️ JARVIS prend du temps... tu peux réessayer'}},25000);ws._tid=tid}
else if(d.type==='error')addMsg('❌ '+d.content,'sys')}catch(x){addMsg(e.data,'jarvis')};
s.textContent='🟢 Connecté — JARVIS v3 prêt';b.disabled=false;mic.disabled=false}}
function send(){let t=i.value.trim();if(!t||!ws||ws.readyState!==WebSocket.OPEN)return;addMsg(t,'user');ws.send(JSON.stringify({text:t}));i.value='';b.disabled=true;mic.disabled=true;s.textContent='🤔 JARVIS réfléchit...';let tid=setTimeout(()=>{if(b.disabled){b.disabled=false;mic.disabled=false;s.textContent='⚠️ JARVIS prend du temps... tu peux réessayer'}},25000);ws._tid=tid}
// ── Audio playback ──
function playAudio(b64){try{const bin=atob(b64);const bytes=new Uint8Array(bin.length);for(let i=0;i<bin.length;i++)bytes[i]=bin.charCodeAt(i);const blob=new Blob([bytes],{type:'audio/mp3'});const url=URL.createObjectURL(blob);const a=new Audio(url);a.onended=()=>URL.revokeObjectURL(url);a.play().catch(e=>console.log('Audio play:',e))}catch(e){console.log('Audio error:',e)}}
// ── Speech Recognition ──
const SpeechRecognition=window.SpeechRecognition||window.webkitSpeechRecognition;
let rec=null,listening=false;
if(SpeechRecognition){
rec=new SpeechRecognition();rec.lang='fr-FR';rec.interimResults=false;rec.continuous=false;
rec.onresult=e=>{const t=e.results[0][0].transcript.trim();if(t){i.value=t;send()}};
rec.onerror=e=>{console.log('STT error:',e.error);stopListening()};
rec.onend=()=>stopListening();
}
function toggleMic(){
if(!rec){alert('Reconnaissance vocale non supportée sur ce navigateur');return}
if(listening){rec.stop();return}
listening=true;mic.classList.add('listening');mic.textContent='🔴';s.textContent='🎤 Je t\'écoute...';
try{rec.start()}catch(e){stopListening()}
}
function stopListening(){listening=false;mic.classList.remove('listening');mic.textContent='🎤';if(s.textContent.includes('écoute'))s.textContent='🟢 Connecté — JARVIS v3 prêt'}
mic.onclick=toggleMic;
b.onclick=send;i.onkeydown=e=>{if(e.key==='Enter')send()};connect()
</script>
</body>
@@ -177,9 +234,14 @@ async def ws_handler(websocket):
messages.append({"role": "assistant", "content": reply})
print(f"🤖 {reply[:80]}")
# Générer TTS en parallèle de l'envoi texte
audio_bytes = await generate_tts(reply)
audio_b64 = base64.b64encode(audio_bytes).decode() if audio_bytes else ""
await websocket.send(json.dumps({
"type": "response",
"content": reply,
"audio": audio_b64,
}))
except Exception as e: