Jkdsjksj

backend/src/config.ts

@@ -1,5 +1,6 @@
 import dotenv from 'dotenv';
 import { readFileSync } from 'fs';
+import { join } from 'path';
 
 dotenv.config();
 
@@ -7,6 +8,7 @@ let _raspberryHost = process.env.RASPBERRY_PI_HOST ?? 'http://raspberrypi.local'
 let _raspberryPort = Number(process.env.RASPBERRY_PI_PORT) || 8000;
 let _token = process.env.QUIBOT_TOKEN ?? 'MY_SECRET_TOKEN';
 const APP_PORT = Number(process.env.PORT) || 5000;
+const piperUrl = process.env.PIPER_URL ?? '';
 const llamacppUrl = process.env.LLAMA_CPP_URL ?? '';
 const llamacppApiKey = process.env.LLAMA_API_KEY ?? '';
 const llamaPreambleRaw = process.env.LLAMA_PREAMBLE ?? '';
@@ -42,4 +44,9 @@ export const getLlamacppUrl = () => llamacppUrl;
 export const getLlamacppApiKey = () => llamacppApiKey;
 export const getLlamacppPreamble = () => llamacppPreamble;
 
+export const getPiperUrl = () => piperUrl;
+export const getPiperModel = () =>
+  process.env.PIPER_MODEL ||
+  join('/tmp', 'quibot-piper-models', 'ca_ES-upc_ona-medium.onnx');
+
 export const getAppPort = () => APP_PORT;

backend/src/controllers/tts.controller.ts

@@ -0,0 +1,52 @@
+import { Router } from 'express';
+import { randomUUID } from 'crypto';
+import { join } from 'path';
+import { mkdirSync, writeFileSync } from 'fs';
+import { piperService } from '../services/piper.service.js';
+import { getToken } from '../config.js';
+
+const router = Router();
+
+const TTS_AUDIO_DIR = join('/tmp', 'quibot-audio', 'tts');
+try { mkdirSync(TTS_AUDIO_DIR, { recursive: true }); } catch { /* ignore */ }
+
+router.post('/', async (req, res) => {
+  try {
+    const text = (req.query.text as string) || (req.body?.text as string);
+    // Accept 'lang' or 'language' — APK sends 'language', old tests use 'lang'
+    const lang = (req.query.lang as string) || (req.query.language as string) || 'ca';
+    const token = (req.query.token as string) || '';
+
+    if (!text?.trim()) {
+      return res.status(400).json({ error: 'Missing query parameter: text' });
+    }
+
+    const expectedToken = getToken();
+    if (token && token !== expectedToken) {
+      return res.status(401).json({ error: 'Unauthorized: invalid token' });
+    }
+
+    console.log(`[tts] Generating audio for text (${lang}): "${text.substring(0, 60)}..."`);
+
+    // Ensure Piper subprocess is initialized before synthesis
+    await piperService.initWav();
+
+    const wavBuffer = await piperService.synthWav(text.trim());
+
+    const filename = `${randomUUID()}.wav`;
+    writeFileSync(join(TTS_AUDIO_DIR, filename), wavBuffer);
+
+    console.log(`[tts] Audio saved: ${filename}`);
+
+    return res.json({
+      audioUrl: `/tts-audio/${filename}`,
+      filename,
+    });
+  } catch (err: unknown) {
+    const message = err instanceof Error ? err.message : 'Unknown error';
+    console.error(`[tts] Error: ${message}`);
+    return res.status(500).json({ error: `TTS synthesis failed: ${message}` });
+  }
+});
+
+export default router;

backend/src/index.ts

@@ -3,6 +3,7 @@ import cors from 'cors';
 import router from './routes/router.js';
 import { getAppPort, getConfig } from './config.js';
 import { whisperService } from './services/whisper.service.js';
+import { piperService as piperWorker } from './services/piper.service.js';
 
 const app = express();
 
@@ -14,6 +15,9 @@ app.use('/audio', express.json());
 app.use('/motor', express.json());
 app.use('/commands', express.json());
 
+// Serve generated TTS audio files to the APK
+app.use('/tts-audio', express.static('/tmp/quibot-audio/tts'));
+
 app.use(router);
 
 app.get('/health', (_req, res) => {
@@ -24,12 +28,13 @@ app.get('/health', (_req, res) => {
 const server = app.listen(getAppPort(), () => {
   console.log(`QuiBot backend listening on port ${getAppPort()}`);
   whisperService.spawn();
+  piperWorker.initWav().catch(() => { /* model may not exist yet → lazy init on first TTS call */ });
 });
 
 async function shutdown(signal: string) {
   console.log(`[server] ${signal} received, shutting down...`);
   server.close(async () => {
-    await whisperService.shutdown();
+    await Promise.all([whisperService.shutdown(), piperWorker.shutdown()]);
     process.exit(0);
   });
 }

backend/src/piper-worker.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+"""Persistent Piper TTS worker – single subprocess, model loaded once."""
+
+import sys
+import json
+import wave
+import io
+
+
+class PiperStdoutSink:
+    """Wraps stdout so we write exactly N bytes."""
+
+    def __init__(self):
+        self._remaining = 0
+
+    def set_length(self, length: int):
+        self._remaining = length
+
+    def write(self, data: bytes):
+        to_write = min(len(data), self._remaining)
+        if to_write > 0:
+            sys.stdout.buffer.write(data[:to_write])
+            sys.stdout.buffer.flush()
+            self._remaining -= to_write
+
+
+def main():
+    from piper import PiperVoice
+
+    model_path = ""
+    config_path = None
+    voice = None
+
+    # Defaults (will be overridden by init message)
+    DEFAULT_MODEL = "/tmp/quibot-piper-models/ca_ES-upc_ona-medium.onnx"
+    DEFAULT_CONFIG = "/tmp/quibot-piper-models/ca_ES-upc_ona-medium.onnx.json"
+
+    # Signal node that the process is alive and listening
+    print(json.dumps({"type": "ready"}), flush=True)
+
+    for line in sys.stdin:
+        line = line.strip()
+        if not line:
+            continue
+
+        try:
+            msg = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+
+        if msg.get("type") == "init":
+            model_path = msg.get("model", DEFAULT_MODEL) or DEFAULT_MODEL
+            config_path = msg.get("config", None)
+            print(f"[piper-worker] Loading model='{model_path}'", file=sys.stderr, flush=True)
+            try:
+                voice = PiperVoice.load(model_path, config_path=config_path)
+                print(json.dumps({"type": "init_ok"}), flush=True)
+            except Exception as exc:
+                err_msg = str(exc).replace('"', '\\"')
+                print(json.dumps({"type": "init_error", "error": err_msg}), flush=True)
+
+        elif msg.get("type") == "synthesize":
+            text = msg.get("text", "")
+            msg_id = msg.get("msgId", "")
+            if not voice:
+                print(json.dumps({
+                    "type": "error",
+                    "text": "Model not loaded, send init first",
+                    "msgId": msg_id,
+                }), flush=True)
+                continue
+
+            try:
+                buf = io.BytesIO()
+                wf = wave.open(buf, 'wb')
+                # set_wav_config is called inside synthesize_wav via set_wav_format=True (default)
+                voice.synthesize_wav(text, wf)
+                wf.close()
+                wav_bytes = buf.getvalue()
+
+                # Frame: send length prefix as ASCII number + newline, then raw bytes
+                header = f"{len(wav_bytes)}\n".encode('ascii')
+                sys.stdout.buffer.write(header)
+                sys.stdout.buffer.write(wav_bytes)
+                sys.stdout.buffer.flush()
+
+                print(json.dumps({
+                    "type": "synthesized",
+                    "bytes": len(wav_bytes),
+                    "msgId": msg_id,
+                }), flush=True)
+            except Exception as exc:
+                err_msg = str(exc).replace('"', '\\"')
+                print(json.dumps({
+                    "type": "error",
+                    "text": err_msg,
+                    "msgId": msg_id,
+                }), flush=True)
+
+
+if __name__ == "__main__":
+    main()

backend/src/routes/router.ts

@@ -3,6 +3,7 @@ import motorController from '../controllers/motor.controller.js';
 import audioController from '../controllers/audio.controller.js';
 import commandController from '../controllers/command.controller.js';
 import settingsController from '../controllers/settings.controller.js';
+import ttsController from '../controllers/tts.controller.js';
 
 const router = Router();
 
@@ -10,5 +11,6 @@ router.use('/motor', motorController);
 router.use('/audio', audioController);
 router.use('/commands', commandController);
 router.use('/settings', settingsController);
+router.use('/tts', ttsController);
 
 export default router;

backend/src/services/piper.http.service.ts

@@ -0,0 +1,26 @@
+import axios from 'axios';
+import { getPiperUrl } from '../config.js';
+
+export interface PiperSynthesisParams {
+  text: string;
+  lang?: string;
+}
+
+class PiperHttpService {
+  async synthesize(params: PiperSynthesisParams): Promise<Buffer> {
+    const piperUrl = getPiperUrl();
+    if (!piperUrl) throw new Error('PIPER_URL not configured');
+
+    const speakerId = params.lang === 'en' ? 1 : 0;
+
+    const response = await axios.post(
+      `${piperUrl}/api/tts`,
+      { text: params.text, speaker_id: speakerId },
+      { responseType: 'arraybuffer', timeout: 30_000 },
+    );
+
+    return Buffer.from(response.data, 'binary');
+  }
+}
+
+export const piperService = new PiperHttpService();

backend/src/services/piper.service.ts

@@ -0,0 +1,210 @@
+import { spawn, ChildProcess } from 'child_process';
+import { join } from 'path';
+import { fileURLToPath } from 'url';
+import { randomUUID } from 'crypto';
+import { readFileSync, rmSync, readdirSync, statSync, unlinkSync, mkdirSync } from 'fs';
+import { piperService as httpPiperService } from './piper.http.service.js';
+import { getPiperUrl, getPiperModel } from '../config.js';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = join(__filename, '..');
+const SCRIPT_DIR = join(__dirname, '..');
+
+const TTS_DIR = join('/tmp', 'quibot-audio', 'tts-piper');
+mkdirSync(TTS_DIR, { recursive: true });
+
+// ─── type-guard for JSON messages from piper-worker ───
+type PiperMsg =
+  | { type: 'ready' }
+  | { type: 'init_ok' }
+  | { type: 'init_error'; error: string }
+  | { type: 'synthesized'; wavPath: string; bytes?: number; msgId: string }
+  | { type: 'error'; text: string; msgId: string };
+
+class PiperLocalService {
+  private proc: ChildProcess | null = null;
+  private pendingInit: Promise<void> | null = null;
+  private cleanupTimer: ReturnType<typeof setInterval> | null = null;
+
+  // ── spawn + stdout parser (simple: pure JSON on stdout, WAV on disk) ──
+
+  private setupStdout(): void {
+    if (!this.proc?.stdout) return;
+    let buf = '';
+    this.proc.stdout.on('data', (chunk: Buffer) => {
+      buf += chunk.toString();
+      while (true) {
+        const nl = buf.indexOf('\n');
+        if (nl === -1) break;
+        const line = buf.slice(0, nl).trim();
+        buf = buf.slice(nl + 1);
+        if (!line) continue;
+        this.handleLine(line);
+      }
+    });
+  }
+
+  private handleLine(line: string): void {
+    try { const msg = JSON.parse(line) as PiperMsg; /* handled below */ } catch { return; }
+    const msg = JSON.parse(line) as PiperMsg;
+    if (msg.type === 'ready') return;
+    if (msg.type === 'init_ok') { this.resolveInit(); }
+    if (msg.type === 'init_error') {
+      if (this.pendingInit) { this.pendingInit = null; this.initResolve = null; }
+      this.initReject(new Error(msg.error));
+    }
+    if (msg.type === 'synthesized') {
+      this.resolveResponse(msg.msgId, msg.wavPath);
+    }
+  }
+
+  private async writeStdin(line: string): Promise<void> {
+    if (!this.proc?.stdin) throw new Error('piper-svc: stdin unavailable');
+    this.proc.stdin.write(line + '\n');
+  }
+
+  // ── pending-init promises (simplest possible design) ──
+  private initResolve: (() => void) | null = null;
+  private initReject: (e: Error) => void = () => { throw new Error('no reject handler'); };
+
+  private resolveInit(): void {
+    if (!this.pendingInit) return;
+    this.initResolve?.(); this.pendingInit = null;
+    this.initResolve = null;
+  }
+
+  // ── pending synth responses ──
+  private respMap = new Map<string, (wavPath: string) => void>();
+
+  private resolveResponse(msgId: string, wavPath: string): void {
+    const fn = this.respMap.get(msgId);
+    this.respMap.delete(msgId);
+    if (fn) fn(wavPath);
+  }
+
+  // ── public spawn / initWav / synthWav ──
+
+  private async _spawn(): Promise<void> {
+    if (this.proc) return;
+
+    const workerPath = join(SCRIPT_DIR, 'src', 'piper-worker.py');
+    const venv = process.env.VIRTUAL_ENV || join(SCRIPT_DIR, '.venv', 'bin', 'python3');
+    this.proc = spawn(venv, [workerPath], { stdio: ['pipe', 'pipe', 'pipe'] });
+
+    this.setupStdout();
+
+    this.proc.on('exit', () => {
+      // reject all pending
+      for (const [, fn] of this.respMap) fn('');
+      this.respMap.clear();
+      if (this.pendingInit) { this.initReject(new Error('piper process exited')); this.pendingInit = null; }
+    });
+
+    // ── cleanup old WAV files every 5 min ──
+    const timer = setInterval(() => {
+      try {
+        const now = Date.now();
+        for (const entry of readdirSync(TTS_DIR)) {
+          const fp = join(TTS_DIR, entry);
+          try {
+            const s = statSync(fp);
+            if (s.isFile() && now - s.mtimeMs > 300_000) unlinkSync(fp);
+          } catch { /* ignore */ }
+        }
+      } catch { /* ignore */ }
+    }, 5 * 60 * 1000);
+  }
+
+  async initWav(): Promise<void> {
+    if (!this.proc) await this._spawn();
+    if (this.pendingInit) return this.pendingInit;
+
+    this.pendingInit = new Promise<void>((resolve, reject) => {
+      // override the global reject
+      const origReject = this.initReject;
+      this.initResolve = resolve;
+      this.initReject = (e: Error) => {
+        // cleanup
+        for (const [, fn] of this.respMap) fn('');
+        this.respMap.clear();
+        reject(e);
+      };
+    });
+
+    const modelPath = getPiperModel() || join('/tmp', 'quibot-piper-models', 'ca_ES-upc_ona-medium.onnx');
+    const cfgPath = modelPath.replace(/\.onnx$/, '.onnx.json');
+    await this.writeStdin(
+      JSON.stringify({ type: 'init', model: modelPath, config: cfgPath }),
+    );
+    return this.pendingInit;
+  }
+
+  /** Synthesize with local Piper subprocess (primary) and HTTP Piper fallback */
+  async synthesize(params: { text: string }): Promise<Buffer> {
+    try {
+      await this.initWav();
+      return await this.synthWav(params.text);
+    } catch (localErr) {
+      const url = getPiperUrl();
+      if (url) {
+        console.log(`[tts] Local Piper failed: ${localErr instanceof Error ? localErr.message : localErr}. Falling back to remote.`);
+        return await httpPiperService.synthesize({ ...params, lang: 'ca' });
+      }
+      throw localErr;
+    }
+  }
+
+  async synthWav(text: string): Promise<Buffer> {
+    if (!this.proc) await this._spawn(); // auto-spawn; init runs concurrently
+    const msgId = randomUUID() + '-' + Date.now();
+
+    return new Promise((resolve, reject) => {
+      let cleared = false;
+      const timer = setTimeout(() => {
+        if (cleared) return;
+        cleared = true;
+        this.respMap.delete(msgId);
+        reject(new Error('piper-svc: synthesis timed out'));
+      }, 30_000);
+
+      this.respMap.set(msgId, (wavPath: string) => {
+        if (cleared) return;
+        cleared = true;
+        clearTimeout(timer);
+        try {
+          const buf = readFileSync(wavPath);
+          rmSync(wavPath, { force: true });
+          resolve(buf);
+        } catch (err: unknown) {
+          reject(err instanceof Error ? err : new Error('read WAV failed'));
+        }
+      });
+
+      this.writeStdin(
+        JSON.stringify({ type: 'synthesize', text, msgId }),
+      ).catch((e) => {
+        if (cleared) return;
+        cleared = true;
+        this.respMap.delete(msgId);
+        reject(e instanceof Error ? e : new Error(String(e)));
+      });
+    });
+  }
+
+  // ── shutdown ──
+
+  async shutdown(): Promise<void> {
+    this.cleanupTimer?.refresh(); this.cleanupTimer = null;
+    if (!this.proc) return;
+    const p = this.proc; this.proc = null;
+    await new Promise<void>((res) => {
+      p.on('exit', res);
+      setTimeout(res, 3000);
+      if (!p.killed) { try { p.stdin?.end(); } catch {} p.kill('SIGTERM'); }
+    });
+  }
+}
+
+// ─── singleton ────────────────────────────────────────────────
+
+export const piperService = new PiperLocalService();