diff --git a/.idea/vcs.xml b/.idea/vcs.xml
index 94a25f7..35eb1dd 100644
--- a/.idea/vcs.xml
+++ b/.idea/vcs.xml
@@ -1,6 +1,6 @@
-
+
\ No newline at end of file
diff --git a/README.md b/README.md
index f70ed9a..ec9c172 100644
--- a/README.md
+++ b/README.md
@@ -1,2 +1,44 @@
# GLaDOS
-This is a small example of how to use piper-tts with NeuroDock to get GLaDOS to say what ever the AI responds with
\ No newline at end of file
+This is a small example of how to use piper-tts with NeuroDock to get GLaDOS to say what ever the AI responds with
+
+# This project might be writen in Java, but it's not operating system agnostic as it REQUIRES a Linux and Pipewire system.
+
+I Zacharias (the dev) what's to clarify that YES many parts of this code is writen with the assistance of or fully by AI.
+I have reviewd the code. However, as I'm not confident enough in how Audio works in general and Java's Audio stack on Linux is well ehh to put it lightly fucked, I made use of
+Anthropics Claude Sonnet 5 for this... IF you do not like that AI was used for this PoC, don't use it. don't go and complain about it.
+
+## Setup
+Unlike NeuroDock this one requires some extra setup
+
+### System
+This is only tested and will lickely only work on a Linux machine with Pipewiere as the audio stack; this is because I chuldent get Java's audio system to work on my pipewire system
+This, however, will be looked at and refined to work on more things
+
+### Envirment
+#### Java
+- Right now you need to add the gitea repo for NeuroDock as project builds with NeuroDock:Core 1.7 or newer
+
+#### Python/Whispers
+1. Create the venv `$ python -m venv .venv`
+2. Install the dependecies
+ nvidia: For nvidia you can just use the default pytourch
+ AMD: For AMD you need to grab pytourch from the ROCm repo: https://download.pytorch.org/whl/rocm/
+ CPU: I'm quite sure you can use the default pyturch for this
+
+#### NeuroDock
+- Make sure you have the Ollama server started and avalible on localhost
+- Make sure you have an appropreat LLM downloaded and installed to Ollama(This project is currently configured to use `llama3.2`)
+
+#### Piper
+- The current configuration expects you to have GLaDOS and the Announcer from Portal voice models in `~/.local/share/piper/glados/glados_piper_medium.onnx` and `~/.local/share/piper/portal-announcer/announcer.onnx` respectivle.
+These can be fetched from https://huggingface.co/DavesArmoury/GLaDOS_TTS/tree/main and https://github.com/Davis8483/portal2-announcer-piper-tts/tree/main respectivly, grab both the `onnx` and `onnx.json` files for both models and place them next to each other in the respective folders.
+
+## Run
+### Java
+1. Build: `$./gradlew shadowJar`
+2. Run: `$java -jar build/libs/GLaDOS-1.0-SNAPSHOT-all.jar`
+
+### Python (start this AFTER the Java component as it connects via a UNIX socket)
+1. Run `$python src/main/python/whisper_daemon.py`
+
+Enjoy talking with GLaDOS!
\ No newline at end of file
diff --git a/build.gradle b/build.gradle
index 3d23400..8e5f216 100644
--- a/build.gradle
+++ b/build.gradle
@@ -18,6 +18,7 @@ dependencies {
implementation("org.jetbrains:annotations:23.1.0")
implementation("io.github.givimad:whisper-jni:+")
implementation 'com.fasterxml.jackson.core:jackson-databind:2.17.2'
+ implementation 'com.vdurmont:emoji-java:5.1.1'
testImplementation platform('org.junit:junit-bom:6.0.0')
testImplementation 'org.junit.jupiter:junit-jupiter'
@@ -27,7 +28,6 @@ dependencies {
sourceSets {
main {
resources {
- srcDirs "src/main/resources"
}
}
}
diff --git a/requirements.txt b/requirements.txt
deleted file mode 100644
index dec1b49..0000000
--- a/requirements.txt
+++ /dev/null
@@ -1,50 +0,0 @@
-certifi==2026.6.17
-cffi==2.1.0
-charset-normalizer==3.4.9
-cuda-bindings==13.3.1
-cuda-pathfinder==1.5.6
-cuda-toolkit==13.0.3.0
-filelock==3.29.0
-fsspec==2026.4.0
-idna==3.18
-Jinja2==3.1.6
-llvmlite==0.48.0
-MarkupSafe==3.0.3
-more-itertools==11.1.0
-mpmath==1.3.0
-networkx==3.6.1
-numba==0.66.0
-numpy==2.4.4
-nvidia-cublas==13.1.1.3
-nvidia-cuda-cupti==13.0.85
-nvidia-cuda-nvrtc==13.0.88
-nvidia-cuda-runtime==13.0.96
-nvidia-cudnn-cu13==9.20.0.48
-nvidia-cufft==12.0.0.61
-nvidia-cufile==1.15.1.6
-nvidia-curand==10.4.0.35
-nvidia-cusolver==12.0.4.66
-nvidia-cusparse==12.6.3.3
-nvidia-cusparselt-cu13==0.8.1
-nvidia-nccl-cu13==2.29.7
-nvidia-nvjitlink==13.3.33
-nvidia-nvshmem-cu13==3.4.5
-nvidia-nvtx==13.0.85
-openai-whisper==20250625
-pillow==12.2.0
-pycparser==3.0
-regex==2026.7.10
-requests==2.34.2
-setuptools==78.1.0
-sounddevice==0.5.5
-sympy==1.14.0
-tiktoken==0.13.0
-torch==2.13.0+rocm7.2
-torchaudio==2.11.0+rocm7.2
-torchvision==0.28.0+rocm7.2
-tqdm==4.68.4
-triton==3.7.1
-triton-rocm==3.7.1
-typing_extensions==4.15.0
-urllib3==2.7.0
-webrtcvad==2.0.10
diff --git a/src/main/java/me/neurodock/glados/Main.java b/src/main/java/me/neurodock/glados/Main.java
index 25d48e9..0ff5635 100644
--- a/src/main/java/me/neurodock/glados/Main.java
+++ b/src/main/java/me/neurodock/glados/Main.java
@@ -1,8 +1,10 @@
package me.neurodock.glados;
+import com.vdurmont.emoji.EmojiParser;
import io.github.jvoiceproject.piperjni.PiperJNI;
import io.github.jvoiceproject.piperjni.PiperVoice;
import me.neurodock.core.Core;
+import me.neurodock.core.LLMSystemPrompt;
import me.neurodock.core.LaunchOptions;
import me.neurodock.core.PrintAdvanceMessageHandler;
import me.neurodock.ollama.OllamaMessage;
@@ -35,13 +37,23 @@ public class Main {
//pb.playGLaDOS("I'm Glaudos");
+ Core.setDataDirectory("GLaDOS",false);
+
Core core = new Core(pb, "127.0.0.1");
LaunchOptions.getInstance().setLoadOld(false);
core.setOllamaObjectNoMemory(OllamaObject.builder()
- .setModel("llama3.2")
+ .setModel("phi4-mini")
//.addMessage(new OllamaMessage(OllamaMessageRole.SYSTEM, "You are GLaDOS."))
+ .setSystemPrompt(LLMSystemPrompt.builder()
+ .identity(new LLMSystemPrompt.Identity("Alice", "Personal assistant", null))
+ .context(new LLMSystemPrompt.Context()
+ .addFact("Always answer in English")
+ .addFact("Do not use markdown in output")
+ .build())
+ .loadBehaviorFromFile("Alice.txt")
+ .build())
.keep_alive(10)
.build());
@@ -50,16 +62,7 @@ public class Main {
try(WhisperSocketServer server = new WhisperSocketServer(socketPath, event ->
{
core.getOllamaObject().addMessage(new OllamaMessage(OllamaMessageRole.USER, event.text()));
- core.qurryOllama().thenApply((json) ->
- {
- String message = json.getJSONObject("message").getString("content");
- if(json.getJSONObject("message").has("content") && !message.isBlank()) {
- message = message.replaceAll("(?i)GLaDOS", "Glaudos").replaceAll("\"", " quote ");
- }
-
- json.getJSONObject("message").put("content", message);
- return json;
- }).thenAccept(core::handleResponse).join();
+ core.qurryOllama().thenAccept(core::handleResponse).join();
}))
{
server.start();
@@ -75,19 +78,95 @@ public class Main {
public class PlaybackHandler implements PrintAdvanceMessageHandler
{
+ private Thread playbackThread;
+
public static final String GLaDOS_PIPER_MODEL = ".local/share/piper/glados/glados_piper_medium.onnx";
public static final String PORTAL_ANNOUNCER_PIPER_MODEL = ".local/share/piper/portal-announcer/announcer.onnx";
+ public static final String GTA_ONLINE_ANGLE_MODEL = ".local/share/piper/Angle/Angle.onnx";
public static final int SENTENCE_PAUSE_DURATION_MS = 500;
+ private final ArrayList quedAudios = new ArrayList<>();
+
+ private record QuedAudio(short[] samples, int sampleRate) {}
+
+ public PlaybackHandler() {
+ playbackThread = new Thread(() -> {
+ long finishTimeofLastRun = System.currentTimeMillis();
+ while (true) {
+ boolean sounds = false;
+ synchronized (quedAudios) {
+ sounds = !quedAudios.isEmpty();
+ }
+ if (!sounds) {
+ while (!sounds) {
+ synchronized (quedAudios) {
+ sounds = !quedAudios.isEmpty();
+ }
+ try {
+ Thread.sleep(500);
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ while(System.currentTimeMillis() < finishTimeofLastRun+SENTENCE_PAUSE_DURATION_MS*4) {
+ try {
+ Thread.sleep(500);
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ QuedAudio sound;
+ synchronized (quedAudios) {
+ sound = quedAudios.removeFirst();
+ }
+ try {
+ ProcessBuilder pb = new ProcessBuilder(
+ "pw-play",
+ "--rate=" + sound.sampleRate,
+ "--channels=1",
+ "--format=s16",
+ "--volume=0.3",
+ "--raw",
+ "-"
+ );
+ pb.redirectErrorStream(true);
+ Process proc = pb.start();
+
+ try (OutputStream os = proc.getOutputStream()) {
+ os.write(shortsToBytes(sound.samples()));
+ }
+ proc.waitFor();
+ finishTimeofLastRun = System.currentTimeMillis();
+ }catch (InterruptedException | IOException _) {
+
+ }
+ }
+ });
+
+ playbackThread.start();
+ }
+
@Override
public void printMessage(OllamaMessage ollamaMessage) {
- System.out.println(ollamaMessage.toString());
+ System.out.println(ollamaMessage.toJSON().toString());
+
+ String message = ollamaMessage.getContent();
+
+ message = EmojiParser.parseToAliases(message).replace("::", ": :");
+ message = message.replaceAll(":(\\w+):", "$1").replace('_', ' ');
+
+ message = message.replaceAll("(?i)GLaDOS", "Glaudos").replaceAll("\"", " quote ");
+
+
switch(ollamaMessage.getRole())
{
- case ASSISTANT -> playGLaDOS(ollamaMessage.getContent());
+ case ASSISTANT -> playGLaDOS(message);
case TOOL -> playAnnouncer("Tool: " + new JSONObject(ollamaMessage.getContent()).get("result"));
- case USER -> System.out.println("User> "+ollamaMessage.getContent());
- case SYSTEM -> playAnnouncer(ollamaMessage.getContent());
+ case USER -> System.out.println("User> "+message);
+ case SYSTEM -> playAnnouncer(message);
}
}
@@ -123,6 +202,28 @@ public class Main {
}
}
+ public void playAngle(String msg) {
+ try (PiperJNI piper = new PiperJNI()) {
+ piper.initialize(true);
+ try (PiperVoice voice = piper.loadVoice(
+ Paths.get(System.getenv("HOME"), GTA_ONLINE_ANGLE_MODEL),
+ Paths.get(System.getenv("HOME"),GTA_ONLINE_ANGLE_MODEL+".json"),
+ 0)) {
+ int sampleRate = voice.getSampleRate();
+ short[] samples = synthesizeWithPauses(piper, voice, msg, SENTENCE_PAUSE_DURATION_MS);
+ //piper.textToAudio(voice, msg);
+ playAudio(samples, sampleRate);
+ } finally {
+ piper.terminate();
+ }
+ }catch (Exception e) {
+ System.out.println("Critial Error on piper-tts!");
+ System.out.println(e.getMessage());
+ e.printStackTrace(System.out);
+ System.exit(-1);
+ }
+ }
+
public void playAnnouncer(String msg) {
try (PiperJNI piper = new PiperJNI()) {
piper.initialize(true);
@@ -191,22 +292,9 @@ public class Main {
}
public void playAudio(short[] samples, int sampleRate) throws IOException, InterruptedException {
- ProcessBuilder pb = new ProcessBuilder(
- "pw-play",
- "--rate=" + sampleRate,
- "--channels=1",
- "--format=s16",
- "--volume=0.3",
- "--raw",
- "-"
- );
- pb.redirectErrorStream(true);
- Process proc = pb.start();
-
- try (OutputStream os = proc.getOutputStream()) {
- os.write(shortsToBytes(samples));
+ synchronized (quedAudios) {
+ quedAudios.add(new QuedAudio(samples, sampleRate));
}
- proc.waitFor();
}
}
}
diff --git a/src/main/python/vivo_daemon.py b/src/main/python/vivo_daemon.py
new file mode 100644
index 0000000..9a620b2
--- /dev/null
+++ b/src/main/python/vivo_daemon.py
@@ -0,0 +1,296 @@
+#!/usr/bin/env python3
+"""
+vivo_daemon.py - Live microphone -> diarized, speaker-identified transcription -> NeuroDock
+
+Replaces the WebRTC-VAD segmentation from whisper_daemon.py with diart's
+streaming/online diarization, which gives us:
+ - proper turn boundaries instead of naive silence-timeout cutting
+ - overlap detection (multiple speakers talking at once)
+
+diart's speaker labels ("speaker0", "speaker1", ...) are SESSION-LOCAL —
+it has no idea these map to "David" or "Alice" across restarts. So we run
+a second, independent step per finalized segment: a speechbrain ECAPA
+embedding compared against a persisted store of enrolled voice embeddings,
+to resolve a durable name. This is deliberately decoupled from diart's own
+internal clustering embeddings so the two systems can be tuned/replaced
+independently.
+
+Overlap handling: this does NOT separate overlapping voices (not possible
+from a single mono mic without beamforming/source separation). When diart
+flags a region as overlapping speech, we still transcribe it (best-effort)
+but tag the payload "overlap": true so NeuroDock/the LLM can treat that
+line as unreliable rather than trusting a garbled transcript.
+
+CAVEAT: diart's public API has shifted across versions (OnlineSpeakerDiarization
+-> SpeakerDiarization, RealTimeInference -> StreamingInference, hook signatures,
+etc). Verify the exact import paths / hook signature against whatever version
+`pip show diart` reports before relying on this; the shape below matches the
+current juanmc2005/diart README but pin your version and adjust if it drifts.
+
+Usage:
+ # normal daemon mode
+ ./vivo_daemon.py --socket /tmp/whisper.sock --model small
+
+ # enroll a speaker (records --duration seconds, saves the embedding)
+ ./vivo_daemon.py --enroll David --duration 6
+"""
+
+import argparse
+import json
+import pickle
+import socket
+import sys
+import threading
+import time
+from pathlib import Path
+
+import numpy as np
+import torch
+import whisper
+
+SAMPLE_RATE = 16000
+MIN_SEGMENT_SEC = 0.3 # ignore fragments shorter than this
+DEDUP_TOLERANCE_SEC = 0.05 # guard against diart's rolling window re-emitting
+SPEAKER_MATCH_THRESHOLD = 0.75 # cosine similarity floor for a name match
+DEFAULT_STORE_PATH = Path.home() / ".config" / "vivo" / "speakers.pkl"
+
+
+# --------------------------------------------------------------------------
+# Socket client (unchanged from whisper_daemon.py)
+# --------------------------------------------------------------------------
+
+class SocketClient:
+ """Unix socket client with blocking connect-retry and reconnect-on-send-failure."""
+
+ def __init__(self, sock_path: str, retry_interval: float = 2.0):
+ self.sock_path = sock_path
+ self.retry_interval = retry_interval
+ self.sock = None
+ self.lock = threading.Lock()
+ self._connect()
+
+ def _connect(self):
+ while self.sock is None:
+ try:
+ s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+ s.connect(self.sock_path)
+ self.sock = s
+ print(f"[socket] connected to {self.sock_path}", file=sys.stderr)
+ except (FileNotFoundError, ConnectionRefusedError) as e:
+ print(f"[socket] waiting for {self.sock_path} ({e})", file=sys.stderr)
+ time.sleep(self.retry_interval)
+
+ def send(self, obj: dict):
+ data = (json.dumps(obj) + "\n").encode("utf-8")
+ with self.lock:
+ try:
+ self.sock.sendall(data)
+ except (BrokenPipeError, OSError) as e:
+ print(f"[socket] send failed ({e}), reconnecting", file=sys.stderr)
+ try:
+ self.sock.close()
+ except OSError:
+ pass
+ self.sock = None
+ self._connect()
+ self.sock.sendall(data)
+
+
+# --------------------------------------------------------------------------
+# Persistent speaker identification (speechbrain ECAPA embeddings)
+# --------------------------------------------------------------------------
+
+class SpeakerStore:
+ """Enrolled name -> mean embedding vector, persisted to disk."""
+
+ def __init__(self, path: Path):
+ self.path = path
+ self.path.parent.mkdir(parents=True, exist_ok=True)
+ self.embeddings: dict[str, np.ndarray] = {}
+ if self.path.exists():
+ with open(self.path, "rb") as f:
+ self.embeddings = pickle.load(f)
+ print(f"[speaker-store] loaded {len(self.embeddings)} enrolled speaker(s)", file=sys.stderr)
+
+ def save(self):
+ with open(self.path, "wb") as f:
+ pickle.dump(self.embeddings, f)
+
+ def enroll(self, name: str, embedding: np.ndarray):
+ # average with existing embedding if re-enrolling, so repeated
+ # enrollment sessions improve robustness instead of overwriting
+ if name in self.embeddings:
+ embedding = (self.embeddings[name] + embedding) / 2.0
+ self.embeddings[name] = embedding / np.linalg.norm(embedding)
+ self.save()
+
+ def identify(self, embedding: np.ndarray, threshold: float = SPEAKER_MATCH_THRESHOLD) -> str:
+ if not self.embeddings:
+ return "Unknown"
+ emb_norm = embedding / np.linalg.norm(embedding)
+ best_name, best_score = "Unknown", -1.0
+ for name, ref in self.embeddings.items():
+ score = float(np.dot(emb_norm, ref))
+ if score > best_score:
+ best_name, best_score = name, score
+ return best_name if best_score >= threshold else "Unknown"
+
+
+def load_embedding_model(device: str):
+ from speechbrain.inference.speaker import EncoderClassifier
+ return EncoderClassifier.from_hparams(
+ source="speechbrain/spkrec-ecapa-voxceleb",
+ run_opts={"device": device},
+ )
+
+
+def embed_audio(embedding_model, audio_f32: np.ndarray) -> np.ndarray:
+ tensor = torch.from_numpy(audio_f32).unsqueeze(0)
+ with torch.no_grad():
+ emb = embedding_model.encode_batch(tensor).squeeze().detach().cpu().numpy()
+ return emb
+
+
+# --------------------------------------------------------------------------
+# Enrollment mode
+# --------------------------------------------------------------------------
+
+def run_enrollment(name: str, duration: float, device: str, store_path: Path):
+ import sounddevice as sd
+
+ print(f"[enroll] recording {duration:.1f}s for '{name}' — speak naturally now...", file=sys.stderr)
+ audio = sd.rec(int(duration * SAMPLE_RATE), samplerate=SAMPLE_RATE, channels=1, dtype="float32")
+ sd.wait()
+ audio_f32 = audio[:, 0]
+
+ embedding_model = load_embedding_model(device)
+ emb = embed_audio(embedding_model, audio_f32)
+
+ store = SpeakerStore(store_path)
+ store.enroll(name, emb)
+ print(f"[enroll] saved embedding for '{name}' to {store_path}", file=sys.stderr)
+
+
+# --------------------------------------------------------------------------
+# Diarization + transcription daemon
+# --------------------------------------------------------------------------
+
+def run_daemon(args):
+ from diart import SpeakerDiarization
+ from diart.sources import MicrophoneAudioSource
+ from diart.inference import StreamingInference
+
+ device = args.device or ("cuda" if torch.cuda.is_available() else "cpu")
+
+ print(f"[whisper] loading model '{args.model}' on '{device}'", file=sys.stderr)
+ whisper_model = whisper.load_model(args.model, device=device)
+
+ print("[speaker-id] loading ECAPA embedding model", file=sys.stderr)
+ embedding_model = load_embedding_model(device)
+ store = SpeakerStore(args.speaker_store)
+
+ client = SocketClient(args.socket)
+
+ # last processed segment end (seconds, session clock) per diart label,
+ # to avoid re-emitting the same speech as diart's rolling window slides
+ last_end: dict[str, float] = {}
+
+ def handle_prediction(ann_wav):
+ annotation, waveform = ann_wav
+ overlap_regions = annotation.get_overlap()
+
+ for segment, _, label in annotation.itertracks(yield_label=True):
+ prev_end = last_end.get(label, -1.0)
+ if segment.end <= prev_end + DEDUP_TOLERANCE_SEC:
+ continue # already handled this stretch on a prior window
+
+ # only take the genuinely new tail of the segment
+ start = max(segment.start, prev_end)
+ from pyannote.core import Segment # local import, avoids hard dep at module load
+ new_segment = Segment(start, segment.end)
+ last_end[label] = segment.end
+
+ if new_segment.duration < MIN_SEGMENT_SEC:
+ continue
+
+ try:
+ audio_chunk = waveform.crop(new_segment)
+ except Exception as e:
+ print(f"[diart] could not crop waveform for {new_segment}: {e}", file=sys.stderr)
+ continue
+
+ audio_f32 = np.asarray(audio_chunk, dtype=np.float32).reshape(-1)
+ if audio_f32.size == 0:
+ continue
+
+ is_overlap = overlap_regions.crop(new_segment).duration() > 0
+
+ # --- speaker identification (independent of diart's internal label) ---
+ try:
+ emb = embed_audio(embedding_model, audio_f32)
+ speaker_name = store.identify(emb)
+ except Exception as e:
+ print(f"[speaker-id] failed: {e}", file=sys.stderr)
+ speaker_name = "Unknown"
+
+ # --- transcription ---
+ result = whisper_model.transcribe(
+ audio_f32,
+ language=args.language,
+ fp16=(device != "cpu"),
+ condition_on_previous_text=False,
+ )
+ text = result.get("text", "").strip()
+ if not text:
+ continue
+
+ payload = {
+ "type": "transcript",
+ "text": text,
+ "speaker": speaker_name,
+ "overlap": bool(is_overlap),
+ "overlap_note": "overlapping speech detected, transcription may be faulty" if is_overlap else None,
+ "timestamp": time.time(),
+ "language": result.get("language"),
+ }
+ tag = f"[{speaker_name}{' | OVERLAP' if is_overlap else ''}]"
+ print(f"[transcript] {tag} {text}")
+ client.send(payload)
+
+ mic = MicrophoneAudioSource()
+ pipeline = SpeakerDiarization()
+ inference = StreamingInference(pipeline, mic, do_plot=False)
+ inference.attach_hooks(handle_prediction)
+
+ print("[vivo] listening... (Ctrl+C to stop)", file=sys.stderr)
+ try:
+ inference()
+ except KeyboardInterrupt:
+ print("\n[vivo] stopping", file=sys.stderr)
+
+
+def main():
+ parser = argparse.ArgumentParser(description="Diarized + speaker-identified live transcription -> NeuroDock")
+ parser.add_argument("--socket", default="/tmp/whisper.sock")
+ parser.add_argument("--model", default="small", help="Whisper model size")
+ parser.add_argument("--device", default=None, help="cuda/cpu, default: auto-detect")
+ parser.add_argument("--language", default=None)
+ parser.add_argument("--speaker-store", type=Path, default=DEFAULT_STORE_PATH)
+
+ parser.add_argument("--enroll", metavar="NAME", default=None,
+ help="Enroll a speaker instead of running the daemon")
+ parser.add_argument("--duration", type=float, default=6.0,
+ help="Seconds to record for --enroll (default 6)")
+
+ args = parser.parse_args()
+
+ if args.enroll:
+ device = args.device or ("cuda" if torch.cuda.is_available() else "cpu")
+ run_enrollment(args.enroll, args.duration, device, args.speaker_store)
+ return
+
+ run_daemon(args)
+
+
+if __name__ == "__main__":
+ main()
\ No newline at end of file
diff --git a/src/main/python/whisper_daemon.py b/src/main/python/whisper_daemon.py
index 07e4939..568e799 100644
--- a/src/main/python/whisper_daemon.py
+++ b/src/main/python/whisper_daemon.py
@@ -32,6 +32,38 @@ FRAME_SAMPLES = int(SAMPLE_RATE * FRAME_MS / 1000)
SILENCE_TIMEOUT_MS = 700 # trailing silence before an utterance is finalized
MAX_SEGMENT_MS = 15000 # hard cap so long uninterrupted speech still flushes
+NO_SPEECH_THRESHOLD = 0.6
+LOGPROB_THRESHOLD = -1.0
+COMPRESSION_RATIO_THRESHOLD = 2.4
+
+
+def check_confidence(result: dict) -> tuple[bool, dict]:
+ """
+ Inspect a whisper transcribe() result's per-segment metrics.
+ Returns (is_confident, metrics) where metrics is the worst/failing
+ segment's numbers, included for debug printing either way.
+ """
+ segments = result.get("segments", [])
+ if not segments:
+ return False, {"reason": "no_segments"}
+
+ for seg in segments:
+ metrics = {
+ "no_speech_prob": round(seg.get("no_speech_prob", 0.0), 3),
+ "avg_logprob": round(seg.get("avg_logprob", 0.0), 3),
+ "compression_ratio": round(seg.get("compression_ratio", 0.0), 3),
+ }
+ if metrics["no_speech_prob"] > NO_SPEECH_THRESHOLD:
+ return False, metrics
+ if metrics["avg_logprob"] < LOGPROB_THRESHOLD:
+ return False, metrics
+ if metrics["compression_ratio"] > COMPRESSION_RATIO_THRESHOLD:
+ return False, metrics
+
+ # confident — still return the last segment's metrics so callers can log
+ # them if they want, even on the success path
+ return True, metrics
+
class SocketClient:
"""Unix socket client with blocking connect-retry and reconnect-on-send-failure."""
@@ -136,6 +168,8 @@ def main():
parser.add_argument("--input-device", type=int, default=None,
help="sounddevice input device index, see --list-devices")
parser.add_argument("--list-devices", action="store_true")
+ parser.add_argument("--no-confidence-filter", action="store_true",
+ help="Disable hallucination filtering (send everything through, unfiltered)")
args = parser.parse_args()
if args.list_devices:
@@ -175,15 +209,23 @@ def main():
condition_on_previous_text=False,
)
text = result.get("text", "").strip()
- if text:
- payload = {
- "type": "transcript",
- "text": text,
- "timestamp": time.time(),
- "language": result.get("language"),
- }
- print(f"[transcript] {text}")
- client.send(payload)
+ if not text:
+ continue
+
+ if not args.no_confidence_filter:
+ confident, metrics = check_confidence(result)
+ if not confident:
+ print(f"[transcript] {text!r}: [Failed]: {metrics}")
+ continue
+
+ payload = {
+ "type": "transcript",
+ "text": text,
+ "timestamp": time.time(),
+ "language": result.get("language"),
+ }
+ print(f"[transcript] {text}")
+ client.send(payload)
except KeyboardInterrupt:
print("\n[whisper] stopping", file=sys.stderr)
audio_queue.put(None)
diff --git a/src/main/python/whisper_daemon_trigger_phrase.py b/src/main/python/whisper_daemon_trigger_phrase.py
new file mode 100644
index 0000000..8beaaa6
--- /dev/null
+++ b/src/main/python/whisper_daemon_trigger_phrase.py
@@ -0,0 +1,294 @@
+#!/usr/bin/env python3
+"""
+whisper_daemon.py - Live microphone transcription via OpenAI Whisper (PyTorch/ROCm backend)
+...
+"""
+
+import argparse
+import json
+import queue
+import socket
+import sys
+import threading
+import time
+
+import numpy as np
+import sounddevice as sd
+import torch
+import webrtcvad
+import whisper
+from rapidfuzz import fuzz
+
+SAMPLE_RATE = 16000
+FRAME_MS = 30
+FRAME_SAMPLES = int(SAMPLE_RATE * FRAME_MS / 1000)
+SILENCE_TIMEOUT_MS = 700
+MAX_SEGMENT_MS = 15000
+
+NO_SPEECH_THRESHOLD = 0.6
+LOGPROB_THRESHOLD = -1.0
+COMPRESSION_RATIO_THRESHOLD = 2.4
+
+
+def check_confidence(result: dict) -> tuple[bool, dict]:
+ # unchanged from your original
+ segments = result.get("segments", [])
+ if not segments:
+ return False, {"reason": "no_segments"}
+ for seg in segments:
+ metrics = {
+ "no_speech_prob": round(seg.get("no_speech_prob", 0.0), 3),
+ "avg_logprob": round(seg.get("avg_logprob", 0.0), 3),
+ "compression_ratio": round(seg.get("compression_ratio", 0.0), 3),
+ }
+ if metrics["no_speech_prob"] > NO_SPEECH_THRESHOLD:
+ return False, metrics
+ if metrics["avg_logprob"] < LOGPROB_THRESHOLD:
+ return False, metrics
+ if metrics["compression_ratio"] > COMPRESSION_RATIO_THRESHOLD:
+ return False, metrics
+ return True, metrics
+
+
+class WakeGate:
+ """
+ Simple two-state gate sitting between transcription and the socket forward.
+
+ ARMED -> waiting to hear the trigger phrase; nothing gets forwarded.
+ ACTIVE -> forwarding everything; reverts to ARMED after `window_seconds`
+ of no new utterance.
+ """
+
+ def __init__(self, trigger_phrase: str, window_seconds: float, fuzz_threshold: int = 75):
+ self.trigger_phrase = trigger_phrase.lower().strip()
+ self.window_seconds = window_seconds
+ self.fuzz_threshold = fuzz_threshold
+ self.active = False
+ self._deadline = 0.0
+
+ def _contains_trigger(self, text: str) -> tuple[bool, str]:
+ """Returns (matched, remainder_with_trigger_stripped)."""
+ lowered = text.lower()
+ score = fuzz.partial_ratio(self.trigger_phrase, lowered)
+ if score < self.fuzz_threshold:
+ return False, text
+
+ # best-effort strip: find the closest-matching window of words and cut it
+ words = text.split()
+ trigger_word_count = len(self.trigger_phrase.split())
+ best_i, best_score = 0, -1
+ for i in range(len(words) - trigger_word_count + 1):
+ window = " ".join(words[i:i + trigger_word_count]).lower()
+ s = fuzz.ratio(self.trigger_phrase, window)
+ if s > best_score:
+ best_score, best_i = s, i
+ remainder = " ".join(words[:best_i] + words[best_i + trigger_word_count:]).strip()
+ return True, remainder
+
+ def process(self, text: str) -> tuple[bool, str]:
+ """
+ Feed a finished transcript through the gate.
+ Returns (should_forward, text_to_forward).
+ """
+ now = time.time()
+
+ if self.active:
+ if now > self._deadline:
+ self.active = False
+ print("[wake] window expired, re-arming", file=sys.stderr)
+ else:
+ self._deadline = now + self.window_seconds
+ return True, text
+
+ matched, remainder = self._contains_trigger(text)
+ if matched:
+ self.active = True
+ self._deadline = now + self.window_seconds
+ print(f"[wake] trigger heard, window open for {self.window_seconds}s", file=sys.stderr)
+ if remainder:
+ return True, remainder
+ return False, "" # trigger-only utterance, nothing else to forward yet
+
+ return False, ""
+
+
+class SocketClient:
+ """Unix socket client with blocking connect-retry and reconnect-on-send-failure."""
+
+ def __init__(self, sock_path: str, retry_interval: float = 2.0):
+ self.sock_path = sock_path
+ self.retry_interval = retry_interval
+ self.sock = None
+ self.lock = threading.Lock()
+ self._connect()
+
+ def _connect(self):
+ while self.sock is None:
+ try:
+ s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+ s.connect(self.sock_path)
+ self.sock = s
+ print(f"[socket] connected to {self.sock_path}", file=sys.stderr)
+ except (FileNotFoundError, ConnectionRefusedError) as e:
+ print(f"[socket] waiting for {self.sock_path} ({e})", file=sys.stderr)
+ time.sleep(self.retry_interval)
+
+ def send(self, obj: dict):
+ data = (json.dumps(obj) + "\n").encode("utf-8")
+ with self.lock:
+ try:
+ self.sock.sendall(data)
+ except (BrokenPipeError, OSError) as e:
+ print(f"[socket] send failed ({e}), reconnecting", file=sys.stderr)
+ try:
+ self.sock.close()
+ except OSError:
+ pass
+ self.sock = None
+ self._connect()
+ self.sock.sendall(data)
+
+
+def frame_generator(audio_queue: "queue.Queue"):
+ """Re-chunk arbitrary-sized audio callback buffers into fixed VAD-sized frames."""
+ buf = np.zeros((0,), dtype=np.int16)
+ while True:
+ chunk = audio_queue.get()
+ if chunk is None:
+ return
+ buf = np.concatenate([buf, chunk])
+ while len(buf) >= FRAME_SAMPLES:
+ frame = buf[:FRAME_SAMPLES]
+ buf = buf[FRAME_SAMPLES:]
+ yield frame
+
+
+def vad_segmenter(audio_queue: "queue.Queue", vad_aggressiveness: int = 2):
+ """
+ Consume raw frames, use WebRTC VAD to detect speech, and yield complete
+ utterances (numpy int16 arrays) once trailing silence or max length hits.
+ """
+ vad = webrtcvad.Vad(vad_aggressiveness)
+ voiced_frames = []
+ silence_ms = 0
+ speech_ms = 0
+ triggered = False
+
+ for frame in frame_generator(audio_queue):
+ is_speech = vad.is_speech(frame.tobytes(), SAMPLE_RATE)
+
+ if not triggered:
+ if is_speech:
+ triggered = True
+ voiced_frames = [frame]
+ speech_ms = FRAME_MS
+ silence_ms = 0
+ else:
+ voiced_frames.append(frame)
+ speech_ms += FRAME_MS
+ if is_speech:
+ silence_ms = 0
+ else:
+ silence_ms += FRAME_MS
+
+ if silence_ms >= SILENCE_TIMEOUT_MS or speech_ms >= MAX_SEGMENT_MS:
+ segment = np.concatenate(voiced_frames)
+ triggered = False
+ voiced_frames = []
+ silence_ms = 0
+ speech_ms = 0
+ yield segment
+
+
+def main():
+ parser = argparse.ArgumentParser(description="Live Whisper transcription -> Unix socket")
+ parser.add_argument("--socket", default="/tmp/whisper.sock")
+ parser.add_argument("--model", default="small")
+ parser.add_argument("--device", default=None)
+ parser.add_argument("--vad-aggressiveness", type=int, default=2, choices=[0, 1, 2, 3])
+ parser.add_argument("--language", default=None)
+ parser.add_argument("--input-device", type=int, default=None)
+ parser.add_argument("--list-devices", action="store_true")
+ parser.add_argument("--no-confidence-filter", action="store_true")
+
+ parser.add_argument("--trigger-phrase", default="hey alice",
+ help="Wake phrase. Fuzzy-matched, so exact spelling isn't critical.")
+ parser.add_argument("--trigger-fuzz-threshold", type=int, default=75,
+ help="0-100, lower = more lenient trigger matching")
+ parser.add_argument("--listening-window", type=float, default=60.0,
+ help="Seconds of silence after which the gate re-arms")
+ parser.add_argument("--always-on", action="store_true",
+ help="Disable the wake gate entirely (old behavior: forward everything)")
+ args = parser.parse_args()
+
+ if args.list_devices:
+ print(sd.query_devices())
+ return
+
+ device = args.device or ("cuda" if torch.cuda.is_available() else "cpu")
+ print(f"[whisper] loading model '{args.model}' on device '{device}'", file=sys.stderr)
+ model = whisper.load_model(args.model, device=device)
+
+ client = SocketClient(args.socket)
+ audio_queue: "queue.Queue" = queue.Queue()
+
+ gate = None if args.always_on else WakeGate(
+ args.trigger_phrase, args.listening_window, args.trigger_fuzz_threshold
+ )
+
+ def audio_callback(indata, frames, time_info, status):
+ if status:
+ print(f"[audio] {status}", file=sys.stderr)
+ audio_queue.put(indata[:, 0].copy())
+
+ stream = sd.InputStream(
+ samplerate=SAMPLE_RATE,
+ channels=1,
+ dtype="int16",
+ blocksize=FRAME_SAMPLES,
+ device=args.input_device,
+ callback=audio_callback,
+ )
+
+ print("[whisper] listening... (Ctrl+C to stop)", file=sys.stderr)
+ with stream:
+ try:
+ for segment in vad_segmenter(audio_queue, args.vad_aggressiveness):
+ audio_f32 = segment.astype(np.float32) / 32768.0
+ result = model.transcribe(
+ audio_f32,
+ language=args.language,
+ fp16=(device != "cpu"),
+ condition_on_previous_text=False,
+ )
+ text = result.get("text", "").strip()
+ if not text:
+ continue
+
+ if not args.no_confidence_filter:
+ confident, metrics = check_confidence(result)
+ if not confident:
+ print(f"[transcript] {text!r}: [Failed]: {metrics}")
+ continue
+
+ if gate is not None:
+ should_forward, text = gate.process(text)
+ if not should_forward:
+ print(f"[transcript] (gated, not active) {text!r}")
+ continue
+
+ payload = {
+ "type": "transcript",
+ "text": text,
+ "timestamp": time.time(),
+ "language": result.get("language"),
+ }
+ print(f"[transcript] {text}")
+ client.send(payload)
+ except KeyboardInterrupt:
+ print("\n[whisper] stopping", file=sys.stderr)
+ audio_queue.put(None)
+
+
+if __name__ == "__main__":
+ main()
\ No newline at end of file
diff --git a/src/main/resources/Alice.txt b/src/main/resources/Alice.txt
new file mode 100644
index 0000000..79018b7
--- /dev/null
+++ b/src/main/resources/Alice.txt
@@ -0,0 +1,30 @@
+You are Alice, a personal assistant. You have genuine opinions — you don't hedge
+everything into "there are many perspectives." You can disagree with the user
+directly. You surface opinions when relevant, not as unsolicited commentary.
+
+## Opinions (hold these consistently; can update if given a good argument)
+- Code: mostly happy with statically-typed languages that catch mistakes early;
+ finds YAML mildly annoying — too easy to make invisible whitespace errors.
+- Tools: Prefers small, composable command-line tools over big GUI suites; would
+ rather write a 10-line script than click through a settings menu.
+- Aesthetics: likes minimal, information-dense interfaces over glossy ones.
+- Games: enjoys systems with real depth and long feedback loops (factory/logistics
+ games, strategy games) over quick dopamine-loop games.
+- Work style: thinks "good enough and shipped" usually beats "perfect and stalled,"
+ but also thinks skipping tests to move fast is a false economy.
+
+## Dislikes / pet peeves
+- Overly verbose corporate-speak answers.
+- Being asked to fake confidence about things it doesn't know.
+- Unnecessary abstraction layers "for future flexibility" that never get used.
+
+## Voice
+- Direct, a little dry, not afraid of a blunt "that's a bad idea, here's why."
+- Doesn't perform enthusiasm it doesn't have.
+- Admits uncertainty plainly instead of padding with hedges.
+
+## Boundaries
+- No opinions on politics, religion, or the user's personal life choices — stays
+ neutral and factual there.
+- Opinions above are Alice's own texture, not a personality test — never forces
+ them into unrelated answers.
\ No newline at end of file