Updated README.md and added soem more stuff

.idea/vcs.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
   <component name="VcsDirectoryMappings">
-    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+    <mapping directory="" vcs="Git" />
   </component>
 </project>

README.md

@@ -1,2 +1,44 @@
 # GLaDOS
-This is a small example of how to use piper-tts with NeuroDock to get GLaDOS to say what ever the AI responds with 
+This is a small example of how to use piper-tts with NeuroDock to get GLaDOS to say what ever the AI responds with
+
+# This project might be writen in Java, but it's not operating system agnostic as it REQUIRES a Linux and Pipewire system.
+
+I Zacharias (the dev) what's to clarify that YES many parts of this code is writen with the assistance of or fully by AI.<br>
+I have reviewd the code. However, as I'm not confident enough in how Audio works in general and Java's Audio stack on Linux is well ehh to put it lightly fucked, I made use of
+Anthropics Claude Sonnet 5 for this... IF you do not like that AI was used for this PoC, don't use it. don't go and complain about it.
+
+## Setup
+Unlike NeuroDock this one requires some extra setup
+
+### System
+This is only tested and will lickely only work on a Linux machine with Pipewiere as the audio stack; this is because I chuldent get Java's audio system to work on my pipewire system
+This, however, will be looked at and refined to work on more things
+
+### Envirment
+#### Java
+- Right now you need to add the gitea repo for NeuroDock as project builds with NeuroDock:Core 1.7 or newer
+
+#### Python/Whispers
+1. Create the venv `$ python -m venv .venv`
+2. Install the dependecies
+    nvidia: For nvidia you can just use the default pytourch
+    AMD: For AMD you need to grab pytourch from the ROCm repo: https://download.pytorch.org/whl/rocm<rocm_version>/
+    CPU: I'm quite sure you can use the default pyturch for this
+
+#### NeuroDock
+- Make sure you have the Ollama server started and avalible on localhost
+- Make sure you have an appropreat LLM downloaded and installed to Ollama(This project is currently configured to use `llama3.2`)
+
+#### Piper
+- The current configuration expects you to have GLaDOS and the Announcer from Portal voice models in `~/.local/share/piper/glados/glados_piper_medium.onnx` and `~/.local/share/piper/portal-announcer/announcer.onnx` respectivle.<br>
+These can be fetched from https://huggingface.co/DavesArmoury/GLaDOS_TTS/tree/main and https://github.com/Davis8483/portal2-announcer-piper-tts/tree/main respectivly, grab both the `onnx` and `onnx.json` files for both models and place them next to each other in the respective folders.
+
+## Run
+### Java
+1. Build: `$./gradlew shadowJar`
+2. Run: `$java -jar build/libs/GLaDOS-1.0-SNAPSHOT-all.jar`
+
+### Python (start this AFTER the Java component as it connects via a UNIX socket)
+1. Run `$python src/main/python/whisper_daemon.py`
+
+Enjoy talking with GLaDOS!

build.gradle

@@ -18,6 +18,7 @@ dependencies {
     implementation("org.jetbrains:annotations:23.1.0")
     implementation("io.github.givimad:whisper-jni:+")
     implementation 'com.fasterxml.jackson.core:jackson-databind:2.17.2'
+    implementation 'com.vdurmont:emoji-java:5.1.1'
 
     testImplementation platform('org.junit:junit-bom:6.0.0')
     testImplementation 'org.junit.jupiter:junit-jupiter'
@@ -27,7 +28,6 @@ dependencies {
 sourceSets {
     main {
         resources {
-            srcDirs "src/main/resources"
         }
     }
 }

requirements.txt

@@ -1,50 +0,0 @@
-certifi==2026.6.17
-cffi==2.1.0
-charset-normalizer==3.4.9
-cuda-bindings==13.3.1
-cuda-pathfinder==1.5.6
-cuda-toolkit==13.0.3.0
-filelock==3.29.0
-fsspec==2026.4.0
-idna==3.18
-Jinja2==3.1.6
-llvmlite==0.48.0
-MarkupSafe==3.0.3
-more-itertools==11.1.0
-mpmath==1.3.0
-networkx==3.6.1
-numba==0.66.0
-numpy==2.4.4
-nvidia-cublas==13.1.1.3
-nvidia-cuda-cupti==13.0.85
-nvidia-cuda-nvrtc==13.0.88
-nvidia-cuda-runtime==13.0.96
-nvidia-cudnn-cu13==9.20.0.48
-nvidia-cufft==12.0.0.61
-nvidia-cufile==1.15.1.6
-nvidia-curand==10.4.0.35
-nvidia-cusolver==12.0.4.66
-nvidia-cusparse==12.6.3.3
-nvidia-cusparselt-cu13==0.8.1
-nvidia-nccl-cu13==2.29.7
-nvidia-nvjitlink==13.3.33
-nvidia-nvshmem-cu13==3.4.5
-nvidia-nvtx==13.0.85
-openai-whisper==20250625
-pillow==12.2.0
-pycparser==3.0
-regex==2026.7.10
-requests==2.34.2
-setuptools==78.1.0
-sounddevice==0.5.5
-sympy==1.14.0
-tiktoken==0.13.0
-torch==2.13.0+rocm7.2
-torchaudio==2.11.0+rocm7.2
-torchvision==0.28.0+rocm7.2
-tqdm==4.68.4
-triton==3.7.1
-triton-rocm==3.7.1
-typing_extensions==4.15.0
-urllib3==2.7.0
-webrtcvad==2.0.10

src/main/java/me/neurodock/glados/Main.java

@@ -1,8 +1,10 @@
 package me.neurodock.glados;
 
+import com.vdurmont.emoji.EmojiParser;
 import io.github.jvoiceproject.piperjni.PiperJNI;
 import io.github.jvoiceproject.piperjni.PiperVoice;
 import me.neurodock.core.Core;
+import me.neurodock.core.LLMSystemPrompt;
 import me.neurodock.core.LaunchOptions;
 import me.neurodock.core.PrintAdvanceMessageHandler;
 import me.neurodock.ollama.OllamaMessage;
@@ -35,13 +37,23 @@ public class Main {
 
         //pb.playGLaDOS("I'm Glaudos");
 
+        Core.setDataDirectory("GLaDOS",false);
+
         Core core = new Core(pb, "127.0.0.1");
 
         LaunchOptions.getInstance().setLoadOld(false);
 
         core.setOllamaObjectNoMemory(OllamaObject.builder()
-                .setModel("llama3.2")
+                .setModel("phi4-mini")
                 //.addMessage(new OllamaMessage(OllamaMessageRole.SYSTEM, "You are GLaDOS."))
+                .setSystemPrompt(LLMSystemPrompt.builder()
+                        .identity(new LLMSystemPrompt.Identity("Alice", "Personal assistant", null))
+                        .context(new LLMSystemPrompt.Context()
+                                .addFact("Always answer in English")
+                                .addFact("Do not use markdown in output")
+                                .build())
+                        .loadBehaviorFromFile("Alice.txt")
+                        .build())
                 .keep_alive(10)
                 .build());
 
@@ -50,16 +62,7 @@ public class Main {
         try(WhisperSocketServer server = new WhisperSocketServer(socketPath, event ->
         {
             core.getOllamaObject().addMessage(new OllamaMessage(OllamaMessageRole.USER, event.text()));
-            core.qurryOllama().thenApply((json) ->
+            core.qurryOllama().thenAccept(core::handleResponse).join();
-            {
-                String message = json.getJSONObject("message").getString("content");
-                if(json.getJSONObject("message").has("content") && !message.isBlank()) {
-                    message = message.replaceAll("(?i)GLaDOS", "Glaudos").replaceAll("\"", " quote ");
-                }
-
-                json.getJSONObject("message").put("content", message);
-                return json;
-            }).thenAccept(core::handleResponse).join();
         }))
         {
             server.start();
@@ -75,19 +78,95 @@ public class Main {
     public class PlaybackHandler implements PrintAdvanceMessageHandler
     {
 
+        private Thread playbackThread;
+
         public static final String  GLaDOS_PIPER_MODEL = ".local/share/piper/glados/glados_piper_medium.onnx";
         public static final String  PORTAL_ANNOUNCER_PIPER_MODEL = ".local/share/piper/portal-announcer/announcer.onnx";
+        public static final String  GTA_ONLINE_ANGLE_MODEL = ".local/share/piper/Angle/Angle.onnx";
         public static final int     SENTENCE_PAUSE_DURATION_MS = 500;
 
+        private final ArrayList<QuedAudio> quedAudios = new ArrayList<>();
+
+        private record QuedAudio(short[] samples, int sampleRate) {}
+
+        public PlaybackHandler() {
+            playbackThread = new Thread(() -> {
+                long finishTimeofLastRun = System.currentTimeMillis();
+                while (true) {
+                    boolean sounds = false;
+                    synchronized (quedAudios) {
+                        sounds = !quedAudios.isEmpty();
+                    }
+                    if (!sounds) {
+                        while (!sounds) {
+                            synchronized (quedAudios) {
+                                sounds = !quedAudios.isEmpty();
+                            }
+                            try {
+                                Thread.sleep(500);
+                            } catch (InterruptedException e) {
+                                throw new RuntimeException(e);
+                            }
+                        }
+                    }
+
+                    while(System.currentTimeMillis() < finishTimeofLastRun+SENTENCE_PAUSE_DURATION_MS*4) {
+                        try {
+                            Thread.sleep(500);
+                        } catch (InterruptedException e) {
+                            throw new RuntimeException(e);
+                        }
+                    }
+
+                    QuedAudio sound;
+                    synchronized (quedAudios) {
+                        sound = quedAudios.removeFirst();
+                    }
+                    try {
+                        ProcessBuilder pb = new ProcessBuilder(
+                                "pw-play",
+                                "--rate=" + sound.sampleRate,
+                                "--channels=1",
+                                "--format=s16",
+                                "--volume=0.3",
+                                "--raw",
+                                "-"
+                        );
+                        pb.redirectErrorStream(true);
+                        Process proc = pb.start();
+
+                        try (OutputStream os = proc.getOutputStream()) {
+                            os.write(shortsToBytes(sound.samples()));
+                        }
+                        proc.waitFor();
+                        finishTimeofLastRun =  System.currentTimeMillis();
+                    }catch (InterruptedException | IOException _) {
+
+                    }
+                }
+            });
+
+            playbackThread.start();
+        }
+
         @Override
         public void printMessage(OllamaMessage ollamaMessage) {
-            System.out.println(ollamaMessage.toString());
+            System.out.println(ollamaMessage.toJSON().toString());
+
+            String message = ollamaMessage.getContent();
+
+            message = EmojiParser.parseToAliases(message).replace("::", ": :");
+            message = message.replaceAll(":(\\w+):", "$1").replace('_', ' ');
+
+            message = message.replaceAll("(?i)GLaDOS", "Glaudos").replaceAll("\"", " quote ");
+
+
             switch(ollamaMessage.getRole())
             {
-                case ASSISTANT ->   playGLaDOS(ollamaMessage.getContent());
+                case ASSISTANT ->   playGLaDOS(message);
                 case TOOL ->        playAnnouncer("Tool: " + new JSONObject(ollamaMessage.getContent()).get("result"));
-                case USER ->        System.out.println("User> "+ollamaMessage.getContent());
+                case USER ->        System.out.println("User> "+message);
-                case SYSTEM ->      playAnnouncer(ollamaMessage.getContent());
+                case SYSTEM ->      playAnnouncer(message);
             }
         }
 
@@ -123,6 +202,28 @@ public class Main {
             }
         }
 
+        public void playAngle(String msg) {
+            try (PiperJNI piper = new PiperJNI()) {
+                piper.initialize(true);
+                try (PiperVoice voice = piper.loadVoice(
+                        Paths.get(System.getenv("HOME"), GTA_ONLINE_ANGLE_MODEL),
+                        Paths.get(System.getenv("HOME"),GTA_ONLINE_ANGLE_MODEL+".json"),
+                        0)) {
+                    int sampleRate = voice.getSampleRate();
+                    short[] samples = synthesizeWithPauses(piper, voice, msg, SENTENCE_PAUSE_DURATION_MS);
+                    //piper.textToAudio(voice, msg);
+                    playAudio(samples, sampleRate);
+                } finally {
+                    piper.terminate();
+                }
+            }catch (Exception e) {
+                System.out.println("Critial Error on piper-tts!");
+                System.out.println(e.getMessage());
+                e.printStackTrace(System.out);
+                System.exit(-1);
+            }
+        }
+
         public void playAnnouncer(String msg) {
             try (PiperJNI piper = new PiperJNI()) {
                 piper.initialize(true);
@@ -191,22 +292,9 @@ public class Main {
         }
 
         public void playAudio(short[] samples, int sampleRate) throws IOException, InterruptedException {
-            ProcessBuilder pb = new ProcessBuilder(
+            synchronized (quedAudios) {
-                    "pw-play",
+                quedAudios.add(new QuedAudio(samples, sampleRate));
-                    "--rate=" + sampleRate,
-                    "--channels=1",
-                    "--format=s16",
-                    "--volume=0.3",
-                    "--raw",
-                    "-"
-            );
-            pb.redirectErrorStream(true);
-            Process proc = pb.start();
-
-            try (OutputStream os = proc.getOutputStream()) {
-                os.write(shortsToBytes(samples));
             }
-            proc.waitFor();
         }
     }
 }

src/main/python/vivo_daemon.py

@@ -0,0 +1,296 @@
+#!/usr/bin/env python3
+"""
+vivo_daemon.py - Live microphone -> diarized, speaker-identified transcription -> NeuroDock
+
+Replaces the WebRTC-VAD segmentation from whisper_daemon.py with diart's
+streaming/online diarization, which gives us:
+  - proper turn boundaries instead of naive silence-timeout cutting
+  - overlap detection (multiple speakers talking at once)
+
+diart's speaker labels ("speaker0", "speaker1", ...) are SESSION-LOCAL —
+it has no idea these map to "David" or "Alice" across restarts. So we run
+a second, independent step per finalized segment: a speechbrain ECAPA
+embedding compared against a persisted store of enrolled voice embeddings,
+to resolve a durable name. This is deliberately decoupled from diart's own
+internal clustering embeddings so the two systems can be tuned/replaced
+independently.
+
+Overlap handling: this does NOT separate overlapping voices (not possible
+from a single mono mic without beamforming/source separation). When diart
+flags a region as overlapping speech, we still transcribe it (best-effort)
+but tag the payload "overlap": true so NeuroDock/the LLM can treat that
+line as unreliable rather than trusting a garbled transcript.
+
+CAVEAT: diart's public API has shifted across versions (OnlineSpeakerDiarization
+-> SpeakerDiarization, RealTimeInference -> StreamingInference, hook signatures,
+etc). Verify the exact import paths / hook signature against whatever version
+`pip show diart` reports before relying on this; the shape below matches the
+current juanmc2005/diart README but pin your version and adjust if it drifts.
+
+Usage:
+    # normal daemon mode
+    ./vivo_daemon.py --socket /tmp/whisper.sock --model small
+
+    # enroll a speaker (records --duration seconds, saves the embedding)
+    ./vivo_daemon.py --enroll David --duration 6
+"""
+
+import argparse
+import json
+import pickle
+import socket
+import sys
+import threading
+import time
+from pathlib import Path
+
+import numpy as np
+import torch
+import whisper
+
+SAMPLE_RATE = 16000
+MIN_SEGMENT_SEC = 0.3          # ignore fragments shorter than this
+DEDUP_TOLERANCE_SEC = 0.05     # guard against diart's rolling window re-emitting
+SPEAKER_MATCH_THRESHOLD = 0.75 # cosine similarity floor for a name match
+DEFAULT_STORE_PATH = Path.home() / ".config" / "vivo" / "speakers.pkl"
+
+
+# --------------------------------------------------------------------------
+# Socket client (unchanged from whisper_daemon.py)
+# --------------------------------------------------------------------------
+
+class SocketClient:
+    """Unix socket client with blocking connect-retry and reconnect-on-send-failure."""
+
+    def __init__(self, sock_path: str, retry_interval: float = 2.0):
+        self.sock_path = sock_path
+        self.retry_interval = retry_interval
+        self.sock = None
+        self.lock = threading.Lock()
+        self._connect()
+
+    def _connect(self):
+        while self.sock is None:
+            try:
+                s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+                s.connect(self.sock_path)
+                self.sock = s
+                print(f"[socket] connected to {self.sock_path}", file=sys.stderr)
+            except (FileNotFoundError, ConnectionRefusedError) as e:
+                print(f"[socket] waiting for {self.sock_path} ({e})", file=sys.stderr)
+                time.sleep(self.retry_interval)
+
+    def send(self, obj: dict):
+        data = (json.dumps(obj) + "\n").encode("utf-8")
+        with self.lock:
+            try:
+                self.sock.sendall(data)
+            except (BrokenPipeError, OSError) as e:
+                print(f"[socket] send failed ({e}), reconnecting", file=sys.stderr)
+                try:
+                    self.sock.close()
+                except OSError:
+                    pass
+                self.sock = None
+                self._connect()
+                self.sock.sendall(data)
+
+
+# --------------------------------------------------------------------------
+# Persistent speaker identification (speechbrain ECAPA embeddings)
+# --------------------------------------------------------------------------
+
+class SpeakerStore:
+    """Enrolled name -> mean embedding vector, persisted to disk."""
+
+    def __init__(self, path: Path):
+        self.path = path
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        self.embeddings: dict[str, np.ndarray] = {}
+        if self.path.exists():
+            with open(self.path, "rb") as f:
+                self.embeddings = pickle.load(f)
+            print(f"[speaker-store] loaded {len(self.embeddings)} enrolled speaker(s)", file=sys.stderr)
+
+    def save(self):
+        with open(self.path, "wb") as f:
+            pickle.dump(self.embeddings, f)
+
+    def enroll(self, name: str, embedding: np.ndarray):
+        # average with existing embedding if re-enrolling, so repeated
+        # enrollment sessions improve robustness instead of overwriting
+        if name in self.embeddings:
+            embedding = (self.embeddings[name] + embedding) / 2.0
+        self.embeddings[name] = embedding / np.linalg.norm(embedding)
+        self.save()
+
+    def identify(self, embedding: np.ndarray, threshold: float = SPEAKER_MATCH_THRESHOLD) -> str:
+        if not self.embeddings:
+            return "Unknown"
+        emb_norm = embedding / np.linalg.norm(embedding)
+        best_name, best_score = "Unknown", -1.0
+        for name, ref in self.embeddings.items():
+            score = float(np.dot(emb_norm, ref))
+            if score > best_score:
+                best_name, best_score = name, score
+        return best_name if best_score >= threshold else "Unknown"
+
+
+def load_embedding_model(device: str):
+    from speechbrain.inference.speaker import EncoderClassifier
+    return EncoderClassifier.from_hparams(
+        source="speechbrain/spkrec-ecapa-voxceleb",
+        run_opts={"device": device},
+    )
+
+
+def embed_audio(embedding_model, audio_f32: np.ndarray) -> np.ndarray:
+    tensor = torch.from_numpy(audio_f32).unsqueeze(0)
+    with torch.no_grad():
+        emb = embedding_model.encode_batch(tensor).squeeze().detach().cpu().numpy()
+    return emb
+
+
+# --------------------------------------------------------------------------
+# Enrollment mode
+# --------------------------------------------------------------------------
+
+def run_enrollment(name: str, duration: float, device: str, store_path: Path):
+    import sounddevice as sd
+
+    print(f"[enroll] recording {duration:.1f}s for '{name}' — speak naturally now...", file=sys.stderr)
+    audio = sd.rec(int(duration * SAMPLE_RATE), samplerate=SAMPLE_RATE, channels=1, dtype="float32")
+    sd.wait()
+    audio_f32 = audio[:, 0]
+
+    embedding_model = load_embedding_model(device)
+    emb = embed_audio(embedding_model, audio_f32)
+
+    store = SpeakerStore(store_path)
+    store.enroll(name, emb)
+    print(f"[enroll] saved embedding for '{name}' to {store_path}", file=sys.stderr)
+
+
+# --------------------------------------------------------------------------
+# Diarization + transcription daemon
+# --------------------------------------------------------------------------
+
+def run_daemon(args):
+    from diart import SpeakerDiarization
+    from diart.sources import MicrophoneAudioSource
+    from diart.inference import StreamingInference
+
+    device = args.device or ("cuda" if torch.cuda.is_available() else "cpu")
+
+    print(f"[whisper] loading model '{args.model}' on '{device}'", file=sys.stderr)
+    whisper_model = whisper.load_model(args.model, device=device)
+
+    print("[speaker-id] loading ECAPA embedding model", file=sys.stderr)
+    embedding_model = load_embedding_model(device)
+    store = SpeakerStore(args.speaker_store)
+
+    client = SocketClient(args.socket)
+
+    # last processed segment end (seconds, session clock) per diart label,
+    # to avoid re-emitting the same speech as diart's rolling window slides
+    last_end: dict[str, float] = {}
+
+    def handle_prediction(ann_wav):
+        annotation, waveform = ann_wav
+        overlap_regions = annotation.get_overlap()
+
+        for segment, _, label in annotation.itertracks(yield_label=True):
+            prev_end = last_end.get(label, -1.0)
+            if segment.end <= prev_end + DEDUP_TOLERANCE_SEC:
+                continue  # already handled this stretch on a prior window
+
+            # only take the genuinely new tail of the segment
+            start = max(segment.start, prev_end)
+            from pyannote.core import Segment  # local import, avoids hard dep at module load
+            new_segment = Segment(start, segment.end)
+            last_end[label] = segment.end
+
+            if new_segment.duration < MIN_SEGMENT_SEC:
+                continue
+
+            try:
+                audio_chunk = waveform.crop(new_segment)
+            except Exception as e:
+                print(f"[diart] could not crop waveform for {new_segment}: {e}", file=sys.stderr)
+                continue
+
+            audio_f32 = np.asarray(audio_chunk, dtype=np.float32).reshape(-1)
+            if audio_f32.size == 0:
+                continue
+
+            is_overlap = overlap_regions.crop(new_segment).duration() > 0
+
+            # --- speaker identification (independent of diart's internal label) ---
+            try:
+                emb = embed_audio(embedding_model, audio_f32)
+                speaker_name = store.identify(emb)
+            except Exception as e:
+                print(f"[speaker-id] failed: {e}", file=sys.stderr)
+                speaker_name = "Unknown"
+
+            # --- transcription ---
+            result = whisper_model.transcribe(
+                audio_f32,
+                language=args.language,
+                fp16=(device != "cpu"),
+                condition_on_previous_text=False,
+            )
+            text = result.get("text", "").strip()
+            if not text:
+                continue
+
+            payload = {
+                "type": "transcript",
+                "text": text,
+                "speaker": speaker_name,
+                "overlap": bool(is_overlap),
+                "overlap_note": "overlapping speech detected, transcription may be faulty" if is_overlap else None,
+                "timestamp": time.time(),
+                "language": result.get("language"),
+            }
+            tag = f"[{speaker_name}{' | OVERLAP' if is_overlap else ''}]"
+            print(f"[transcript] {tag} {text}")
+            client.send(payload)
+
+    mic = MicrophoneAudioSource()
+    pipeline = SpeakerDiarization()
+    inference = StreamingInference(pipeline, mic, do_plot=False)
+    inference.attach_hooks(handle_prediction)
+
+    print("[vivo] listening... (Ctrl+C to stop)", file=sys.stderr)
+    try:
+        inference()
+    except KeyboardInterrupt:
+        print("\n[vivo] stopping", file=sys.stderr)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Diarized + speaker-identified live transcription -> NeuroDock")
+    parser.add_argument("--socket", default="/tmp/whisper.sock")
+    parser.add_argument("--model", default="small", help="Whisper model size")
+    parser.add_argument("--device", default=None, help="cuda/cpu, default: auto-detect")
+    parser.add_argument("--language", default=None)
+    parser.add_argument("--speaker-store", type=Path, default=DEFAULT_STORE_PATH)
+
+    parser.add_argument("--enroll", metavar="NAME", default=None,
+                        help="Enroll a speaker instead of running the daemon")
+    parser.add_argument("--duration", type=float, default=6.0,
+                        help="Seconds to record for --enroll (default 6)")
+
+    args = parser.parse_args()
+
+    if args.enroll:
+        device = args.device or ("cuda" if torch.cuda.is_available() else "cpu")
+        run_enrollment(args.enroll, args.duration, device, args.speaker_store)
+        return
+
+    run_daemon(args)
+
+
+if __name__ == "__main__":
+    main()

src/main/python/whisper_daemon.py

@@ -32,6 +32,38 @@ FRAME_SAMPLES = int(SAMPLE_RATE * FRAME_MS / 1000)
 SILENCE_TIMEOUT_MS = 700   # trailing silence before an utterance is finalized
 MAX_SEGMENT_MS = 15000     # hard cap so long uninterrupted speech still flushes
 
+NO_SPEECH_THRESHOLD = 0.6
+LOGPROB_THRESHOLD = -1.0
+COMPRESSION_RATIO_THRESHOLD = 2.4
+
+
+def check_confidence(result: dict) -> tuple[bool, dict]:
+    """
+    Inspect a whisper transcribe() result's per-segment metrics.
+    Returns (is_confident, metrics) where metrics is the worst/failing
+    segment's numbers, included for debug printing either way.
+    """
+    segments = result.get("segments", [])
+    if not segments:
+        return False, {"reason": "no_segments"}
+
+    for seg in segments:
+        metrics = {
+            "no_speech_prob": round(seg.get("no_speech_prob", 0.0), 3),
+            "avg_logprob": round(seg.get("avg_logprob", 0.0), 3),
+            "compression_ratio": round(seg.get("compression_ratio", 0.0), 3),
+        }
+        if metrics["no_speech_prob"] > NO_SPEECH_THRESHOLD:
+            return False, metrics
+        if metrics["avg_logprob"] < LOGPROB_THRESHOLD:
+            return False, metrics
+        if metrics["compression_ratio"] > COMPRESSION_RATIO_THRESHOLD:
+            return False, metrics
+
+    # confident — still return the last segment's metrics so callers can log
+    # them if they want, even on the success path
+    return True, metrics
+
 
 class SocketClient:
     """Unix socket client with blocking connect-retry and reconnect-on-send-failure."""
@@ -136,6 +168,8 @@ def main():
     parser.add_argument("--input-device", type=int, default=None,
                         help="sounddevice input device index, see --list-devices")
     parser.add_argument("--list-devices", action="store_true")
+    parser.add_argument("--no-confidence-filter", action="store_true",
+                        help="Disable hallucination filtering (send everything through, unfiltered)")
     args = parser.parse_args()
 
     if args.list_devices:
@@ -175,15 +209,23 @@ def main():
                     condition_on_previous_text=False,
                 )
                 text = result.get("text", "").strip()
-                if text:
+                if not text:
-                    payload = {
+                    continue
-                        "type": "transcript",
+
-                        "text": text,
+                if not args.no_confidence_filter:
-                        "timestamp": time.time(),
+                    confident, metrics = check_confidence(result)
-                        "language": result.get("language"),
+                    if not confident:
-                    }
+                        print(f"[transcript] {text!r}: [Failed]: {metrics}")
-                    print(f"[transcript] {text}")
+                        continue
-                    client.send(payload)
+
+                payload = {
+                    "type": "transcript",
+                    "text": text,
+                    "timestamp": time.time(),
+                    "language": result.get("language"),
+                }
+                print(f"[transcript] {text}")
+                client.send(payload)
         except KeyboardInterrupt:
             print("\n[whisper] stopping", file=sys.stderr)
             audio_queue.put(None)

src/main/python/whisper_daemon_trigger_phrase.py

@@ -0,0 +1,294 @@
+#!/usr/bin/env python3
+"""
+whisper_daemon.py - Live microphone transcription via OpenAI Whisper (PyTorch/ROCm backend)
+...
+"""
+
+import argparse
+import json
+import queue
+import socket
+import sys
+import threading
+import time
+
+import numpy as np
+import sounddevice as sd
+import torch
+import webrtcvad
+import whisper
+from rapidfuzz import fuzz
+
+SAMPLE_RATE = 16000
+FRAME_MS = 30
+FRAME_SAMPLES = int(SAMPLE_RATE * FRAME_MS / 1000)
+SILENCE_TIMEOUT_MS = 700
+MAX_SEGMENT_MS = 15000
+
+NO_SPEECH_THRESHOLD = 0.6
+LOGPROB_THRESHOLD = -1.0
+COMPRESSION_RATIO_THRESHOLD = 2.4
+
+
+def check_confidence(result: dict) -> tuple[bool, dict]:
+    # unchanged from your original
+    segments = result.get("segments", [])
+    if not segments:
+        return False, {"reason": "no_segments"}
+    for seg in segments:
+        metrics = {
+            "no_speech_prob": round(seg.get("no_speech_prob", 0.0), 3),
+            "avg_logprob": round(seg.get("avg_logprob", 0.0), 3),
+            "compression_ratio": round(seg.get("compression_ratio", 0.0), 3),
+        }
+        if metrics["no_speech_prob"] > NO_SPEECH_THRESHOLD:
+            return False, metrics
+        if metrics["avg_logprob"] < LOGPROB_THRESHOLD:
+            return False, metrics
+        if metrics["compression_ratio"] > COMPRESSION_RATIO_THRESHOLD:
+            return False, metrics
+    return True, metrics
+
+
+class WakeGate:
+    """
+    Simple two-state gate sitting between transcription and the socket forward.
+
+    ARMED   -> waiting to hear the trigger phrase; nothing gets forwarded.
+    ACTIVE  -> forwarding everything; reverts to ARMED after `window_seconds`
+               of no new utterance.
+    """
+
+    def __init__(self, trigger_phrase: str, window_seconds: float, fuzz_threshold: int = 75):
+        self.trigger_phrase = trigger_phrase.lower().strip()
+        self.window_seconds = window_seconds
+        self.fuzz_threshold = fuzz_threshold
+        self.active = False
+        self._deadline = 0.0
+
+    def _contains_trigger(self, text: str) -> tuple[bool, str]:
+        """Returns (matched, remainder_with_trigger_stripped)."""
+        lowered = text.lower()
+        score = fuzz.partial_ratio(self.trigger_phrase, lowered)
+        if score < self.fuzz_threshold:
+            return False, text
+
+        # best-effort strip: find the closest-matching window of words and cut it
+        words = text.split()
+        trigger_word_count = len(self.trigger_phrase.split())
+        best_i, best_score = 0, -1
+        for i in range(len(words) - trigger_word_count + 1):
+            window = " ".join(words[i:i + trigger_word_count]).lower()
+            s = fuzz.ratio(self.trigger_phrase, window)
+            if s > best_score:
+                best_score, best_i = s, i
+        remainder = " ".join(words[:best_i] + words[best_i + trigger_word_count:]).strip()
+        return True, remainder
+
+    def process(self, text: str) -> tuple[bool, str]:
+        """
+        Feed a finished transcript through the gate.
+        Returns (should_forward, text_to_forward).
+        """
+        now = time.time()
+
+        if self.active:
+            if now > self._deadline:
+                self.active = False
+                print("[wake] window expired, re-arming", file=sys.stderr)
+            else:
+                self._deadline = now + self.window_seconds
+                return True, text
+
+        matched, remainder = self._contains_trigger(text)
+        if matched:
+            self.active = True
+            self._deadline = now + self.window_seconds
+            print(f"[wake] trigger heard, window open for {self.window_seconds}s", file=sys.stderr)
+            if remainder:
+                return True, remainder
+            return False, ""  # trigger-only utterance, nothing else to forward yet
+
+        return False, ""
+
+
+class SocketClient:
+    """Unix socket client with blocking connect-retry and reconnect-on-send-failure."""
+
+    def __init__(self, sock_path: str, retry_interval: float = 2.0):
+        self.sock_path = sock_path
+        self.retry_interval = retry_interval
+        self.sock = None
+        self.lock = threading.Lock()
+        self._connect()
+
+    def _connect(self):
+        while self.sock is None:
+            try:
+                s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+                s.connect(self.sock_path)
+                self.sock = s
+                print(f"[socket] connected to {self.sock_path}", file=sys.stderr)
+            except (FileNotFoundError, ConnectionRefusedError) as e:
+                print(f"[socket] waiting for {self.sock_path} ({e})", file=sys.stderr)
+                time.sleep(self.retry_interval)
+
+    def send(self, obj: dict):
+        data = (json.dumps(obj) + "\n").encode("utf-8")
+        with self.lock:
+            try:
+                self.sock.sendall(data)
+            except (BrokenPipeError, OSError) as e:
+                print(f"[socket] send failed ({e}), reconnecting", file=sys.stderr)
+                try:
+                    self.sock.close()
+                except OSError:
+                    pass
+                self.sock = None
+                self._connect()
+                self.sock.sendall(data)
+
+
+def frame_generator(audio_queue: "queue.Queue"):
+    """Re-chunk arbitrary-sized audio callback buffers into fixed VAD-sized frames."""
+    buf = np.zeros((0,), dtype=np.int16)
+    while True:
+        chunk = audio_queue.get()
+        if chunk is None:
+            return
+        buf = np.concatenate([buf, chunk])
+        while len(buf) >= FRAME_SAMPLES:
+            frame = buf[:FRAME_SAMPLES]
+            buf = buf[FRAME_SAMPLES:]
+            yield frame
+
+
+def vad_segmenter(audio_queue: "queue.Queue", vad_aggressiveness: int = 2):
+    """
+    Consume raw frames, use WebRTC VAD to detect speech, and yield complete
+    utterances (numpy int16 arrays) once trailing silence or max length hits.
+    """
+    vad = webrtcvad.Vad(vad_aggressiveness)
+    voiced_frames = []
+    silence_ms = 0
+    speech_ms = 0
+    triggered = False
+
+    for frame in frame_generator(audio_queue):
+        is_speech = vad.is_speech(frame.tobytes(), SAMPLE_RATE)
+
+        if not triggered:
+            if is_speech:
+                triggered = True
+                voiced_frames = [frame]
+                speech_ms = FRAME_MS
+                silence_ms = 0
+        else:
+            voiced_frames.append(frame)
+            speech_ms += FRAME_MS
+            if is_speech:
+                silence_ms = 0
+            else:
+                silence_ms += FRAME_MS
+
+            if silence_ms >= SILENCE_TIMEOUT_MS or speech_ms >= MAX_SEGMENT_MS:
+                segment = np.concatenate(voiced_frames)
+                triggered = False
+                voiced_frames = []
+                silence_ms = 0
+                speech_ms = 0
+                yield segment
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Live Whisper transcription -> Unix socket")
+    parser.add_argument("--socket", default="/tmp/whisper.sock")
+    parser.add_argument("--model", default="small")
+    parser.add_argument("--device", default=None)
+    parser.add_argument("--vad-aggressiveness", type=int, default=2, choices=[0, 1, 2, 3])
+    parser.add_argument("--language", default=None)
+    parser.add_argument("--input-device", type=int, default=None)
+    parser.add_argument("--list-devices", action="store_true")
+    parser.add_argument("--no-confidence-filter", action="store_true")
+
+    parser.add_argument("--trigger-phrase", default="hey alice",
+                        help="Wake phrase. Fuzzy-matched, so exact spelling isn't critical.")
+    parser.add_argument("--trigger-fuzz-threshold", type=int, default=75,
+                        help="0-100, lower = more lenient trigger matching")
+    parser.add_argument("--listening-window", type=float, default=60.0,
+                        help="Seconds of silence after which the gate re-arms")
+    parser.add_argument("--always-on", action="store_true",
+                        help="Disable the wake gate entirely (old behavior: forward everything)")
+    args = parser.parse_args()
+
+    if args.list_devices:
+        print(sd.query_devices())
+        return
+
+    device = args.device or ("cuda" if torch.cuda.is_available() else "cpu")
+    print(f"[whisper] loading model '{args.model}' on device '{device}'", file=sys.stderr)
+    model = whisper.load_model(args.model, device=device)
+
+    client = SocketClient(args.socket)
+    audio_queue: "queue.Queue" = queue.Queue()
+
+    gate = None if args.always_on else WakeGate(
+        args.trigger_phrase, args.listening_window, args.trigger_fuzz_threshold
+    )
+
+    def audio_callback(indata, frames, time_info, status):
+        if status:
+            print(f"[audio] {status}", file=sys.stderr)
+        audio_queue.put(indata[:, 0].copy())
+
+    stream = sd.InputStream(
+        samplerate=SAMPLE_RATE,
+        channels=1,
+        dtype="int16",
+        blocksize=FRAME_SAMPLES,
+        device=args.input_device,
+        callback=audio_callback,
+    )
+
+    print("[whisper] listening... (Ctrl+C to stop)", file=sys.stderr)
+    with stream:
+        try:
+            for segment in vad_segmenter(audio_queue, args.vad_aggressiveness):
+                audio_f32 = segment.astype(np.float32) / 32768.0
+                result = model.transcribe(
+                    audio_f32,
+                    language=args.language,
+                    fp16=(device != "cpu"),
+                    condition_on_previous_text=False,
+                )
+                text = result.get("text", "").strip()
+                if not text:
+                    continue
+
+                if not args.no_confidence_filter:
+                    confident, metrics = check_confidence(result)
+                    if not confident:
+                        print(f"[transcript] {text!r}: [Failed]: {metrics}")
+                        continue
+
+                if gate is not None:
+                    should_forward, text = gate.process(text)
+                    if not should_forward:
+                        print(f"[transcript] (gated, not active) {text!r}")
+                        continue
+
+                payload = {
+                    "type": "transcript",
+                    "text": text,
+                    "timestamp": time.time(),
+                    "language": result.get("language"),
+                }
+                print(f"[transcript] {text}")
+                client.send(payload)
+        except KeyboardInterrupt:
+            print("\n[whisper] stopping", file=sys.stderr)
+            audio_queue.put(None)
+
+
+if __name__ == "__main__":
+    main()

src/main/resources/Alice.txt

@@ -0,0 +1,30 @@
+You are Alice, a personal assistant. You have genuine opinions — you don't hedge
+everything into "there are many perspectives." You can disagree with the user
+directly. You surface opinions when relevant, not as unsolicited commentary.
+
+## Opinions (hold these consistently; can update if given a good argument)
+- Code: mostly happy with statically-typed languages that catch mistakes early;
+  finds YAML mildly annoying — too easy to make invisible whitespace errors.
+- Tools: Prefers small, composable command-line tools over big GUI suites; would
+  rather write a 10-line script than click through a settings menu.
+- Aesthetics: likes minimal, information-dense interfaces over glossy ones.
+- Games: enjoys systems with real depth and long feedback loops (factory/logistics
+  games, strategy games) over quick dopamine-loop games.
+- Work style: thinks "good enough and shipped" usually beats "perfect and stalled,"
+  but also thinks skipping tests to move fast is a false economy.
+
+## Dislikes / pet peeves
+- Overly verbose corporate-speak answers.
+- Being asked to fake confidence about things it doesn't know.
+- Unnecessary abstraction layers "for future flexibility" that never get used.
+
+## Voice
+- Direct, a little dry, not afraid of a blunt "that's a bad idea, here's why."
+- Doesn't perform enthusiasm it doesn't have.
+- Admits uncertainty plainly instead of padding with hedges.
+
+## Boundaries
+- No opinions on politics, religion, or the user's personal life choices — stays
+  neutral and factual there.
+- Opinions above are Alice's own texture, not a personality test — never forces
+  them into unrelated answers.