Updated README.md and added soem more stuff
This commit is contained in:
Generated
+1
-1
@@ -1,6 +1,6 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="VcsDirectoryMappings">
|
||||
<mapping directory="$PROJECT_DIR$" vcs="Git" />
|
||||
<mapping directory="" vcs="Git" />
|
||||
</component>
|
||||
</project>
|
||||
@@ -1,2 +1,44 @@
|
||||
# GLaDOS
|
||||
This is a small example of how to use piper-tts with NeuroDock to get GLaDOS to say what ever the AI responds with
|
||||
|
||||
# This project might be writen in Java, but it's not operating system agnostic as it REQUIRES a Linux and Pipewire system.
|
||||
|
||||
I Zacharias (the dev) what's to clarify that YES many parts of this code is writen with the assistance of or fully by AI.<br>
|
||||
I have reviewd the code. However, as I'm not confident enough in how Audio works in general and Java's Audio stack on Linux is well ehh to put it lightly fucked, I made use of
|
||||
Anthropics Claude Sonnet 5 for this... IF you do not like that AI was used for this PoC, don't use it. don't go and complain about it.
|
||||
|
||||
## Setup
|
||||
Unlike NeuroDock this one requires some extra setup
|
||||
|
||||
### System
|
||||
This is only tested and will lickely only work on a Linux machine with Pipewiere as the audio stack; this is because I chuldent get Java's audio system to work on my pipewire system
|
||||
This, however, will be looked at and refined to work on more things
|
||||
|
||||
### Envirment
|
||||
#### Java
|
||||
- Right now you need to add the gitea repo for NeuroDock as project builds with NeuroDock:Core 1.7 or newer
|
||||
|
||||
#### Python/Whispers
|
||||
1. Create the venv `$ python -m venv .venv`
|
||||
2. Install the dependecies
|
||||
nvidia: For nvidia you can just use the default pytourch
|
||||
AMD: For AMD you need to grab pytourch from the ROCm repo: https://download.pytorch.org/whl/rocm<rocm_version>/
|
||||
CPU: I'm quite sure you can use the default pyturch for this
|
||||
|
||||
#### NeuroDock
|
||||
- Make sure you have the Ollama server started and avalible on localhost
|
||||
- Make sure you have an appropreat LLM downloaded and installed to Ollama(This project is currently configured to use `llama3.2`)
|
||||
|
||||
#### Piper
|
||||
- The current configuration expects you to have GLaDOS and the Announcer from Portal voice models in `~/.local/share/piper/glados/glados_piper_medium.onnx` and `~/.local/share/piper/portal-announcer/announcer.onnx` respectivle.<br>
|
||||
These can be fetched from https://huggingface.co/DavesArmoury/GLaDOS_TTS/tree/main and https://github.com/Davis8483/portal2-announcer-piper-tts/tree/main respectivly, grab both the `onnx` and `onnx.json` files for both models and place them next to each other in the respective folders.
|
||||
|
||||
## Run
|
||||
### Java
|
||||
1. Build: `$./gradlew shadowJar`
|
||||
2. Run: `$java -jar build/libs/GLaDOS-1.0-SNAPSHOT-all.jar`
|
||||
|
||||
### Python (start this AFTER the Java component as it connects via a UNIX socket)
|
||||
1. Run `$python src/main/python/whisper_daemon.py`
|
||||
|
||||
Enjoy talking with GLaDOS!
|
||||
+1
-1
@@ -18,6 +18,7 @@ dependencies {
|
||||
implementation("org.jetbrains:annotations:23.1.0")
|
||||
implementation("io.github.givimad:whisper-jni:+")
|
||||
implementation 'com.fasterxml.jackson.core:jackson-databind:2.17.2'
|
||||
implementation 'com.vdurmont:emoji-java:5.1.1'
|
||||
|
||||
testImplementation platform('org.junit:junit-bom:6.0.0')
|
||||
testImplementation 'org.junit.jupiter:junit-jupiter'
|
||||
@@ -27,7 +28,6 @@ dependencies {
|
||||
sourceSets {
|
||||
main {
|
||||
resources {
|
||||
srcDirs "src/main/resources"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,50 +0,0 @@
|
||||
certifi==2026.6.17
|
||||
cffi==2.1.0
|
||||
charset-normalizer==3.4.9
|
||||
cuda-bindings==13.3.1
|
||||
cuda-pathfinder==1.5.6
|
||||
cuda-toolkit==13.0.3.0
|
||||
filelock==3.29.0
|
||||
fsspec==2026.4.0
|
||||
idna==3.18
|
||||
Jinja2==3.1.6
|
||||
llvmlite==0.48.0
|
||||
MarkupSafe==3.0.3
|
||||
more-itertools==11.1.0
|
||||
mpmath==1.3.0
|
||||
networkx==3.6.1
|
||||
numba==0.66.0
|
||||
numpy==2.4.4
|
||||
nvidia-cublas==13.1.1.3
|
||||
nvidia-cuda-cupti==13.0.85
|
||||
nvidia-cuda-nvrtc==13.0.88
|
||||
nvidia-cuda-runtime==13.0.96
|
||||
nvidia-cudnn-cu13==9.20.0.48
|
||||
nvidia-cufft==12.0.0.61
|
||||
nvidia-cufile==1.15.1.6
|
||||
nvidia-curand==10.4.0.35
|
||||
nvidia-cusolver==12.0.4.66
|
||||
nvidia-cusparse==12.6.3.3
|
||||
nvidia-cusparselt-cu13==0.8.1
|
||||
nvidia-nccl-cu13==2.29.7
|
||||
nvidia-nvjitlink==13.3.33
|
||||
nvidia-nvshmem-cu13==3.4.5
|
||||
nvidia-nvtx==13.0.85
|
||||
openai-whisper==20250625
|
||||
pillow==12.2.0
|
||||
pycparser==3.0
|
||||
regex==2026.7.10
|
||||
requests==2.34.2
|
||||
setuptools==78.1.0
|
||||
sounddevice==0.5.5
|
||||
sympy==1.14.0
|
||||
tiktoken==0.13.0
|
||||
torch==2.13.0+rocm7.2
|
||||
torchaudio==2.11.0+rocm7.2
|
||||
torchvision==0.28.0+rocm7.2
|
||||
tqdm==4.68.4
|
||||
triton==3.7.1
|
||||
triton-rocm==3.7.1
|
||||
typing_extensions==4.15.0
|
||||
urllib3==2.7.0
|
||||
webrtcvad==2.0.10
|
||||
@@ -1,8 +1,10 @@
|
||||
package me.neurodock.glados;
|
||||
|
||||
import com.vdurmont.emoji.EmojiParser;
|
||||
import io.github.jvoiceproject.piperjni.PiperJNI;
|
||||
import io.github.jvoiceproject.piperjni.PiperVoice;
|
||||
import me.neurodock.core.Core;
|
||||
import me.neurodock.core.LLMSystemPrompt;
|
||||
import me.neurodock.core.LaunchOptions;
|
||||
import me.neurodock.core.PrintAdvanceMessageHandler;
|
||||
import me.neurodock.ollama.OllamaMessage;
|
||||
@@ -35,13 +37,23 @@ public class Main {
|
||||
|
||||
//pb.playGLaDOS("I'm Glaudos");
|
||||
|
||||
Core.setDataDirectory("GLaDOS",false);
|
||||
|
||||
Core core = new Core(pb, "127.0.0.1");
|
||||
|
||||
LaunchOptions.getInstance().setLoadOld(false);
|
||||
|
||||
core.setOllamaObjectNoMemory(OllamaObject.builder()
|
||||
.setModel("llama3.2")
|
||||
.setModel("phi4-mini")
|
||||
//.addMessage(new OllamaMessage(OllamaMessageRole.SYSTEM, "You are GLaDOS."))
|
||||
.setSystemPrompt(LLMSystemPrompt.builder()
|
||||
.identity(new LLMSystemPrompt.Identity("Alice", "Personal assistant", null))
|
||||
.context(new LLMSystemPrompt.Context()
|
||||
.addFact("Always answer in English")
|
||||
.addFact("Do not use markdown in output")
|
||||
.build())
|
||||
.loadBehaviorFromFile("Alice.txt")
|
||||
.build())
|
||||
.keep_alive(10)
|
||||
.build());
|
||||
|
||||
@@ -50,16 +62,7 @@ public class Main {
|
||||
try(WhisperSocketServer server = new WhisperSocketServer(socketPath, event ->
|
||||
{
|
||||
core.getOllamaObject().addMessage(new OllamaMessage(OllamaMessageRole.USER, event.text()));
|
||||
core.qurryOllama().thenApply((json) ->
|
||||
{
|
||||
String message = json.getJSONObject("message").getString("content");
|
||||
if(json.getJSONObject("message").has("content") && !message.isBlank()) {
|
||||
message = message.replaceAll("(?i)GLaDOS", "Glaudos").replaceAll("\"", " quote ");
|
||||
}
|
||||
|
||||
json.getJSONObject("message").put("content", message);
|
||||
return json;
|
||||
}).thenAccept(core::handleResponse).join();
|
||||
core.qurryOllama().thenAccept(core::handleResponse).join();
|
||||
}))
|
||||
{
|
||||
server.start();
|
||||
@@ -75,19 +78,95 @@ public class Main {
|
||||
public class PlaybackHandler implements PrintAdvanceMessageHandler
|
||||
{
|
||||
|
||||
private Thread playbackThread;
|
||||
|
||||
public static final String GLaDOS_PIPER_MODEL = ".local/share/piper/glados/glados_piper_medium.onnx";
|
||||
public static final String PORTAL_ANNOUNCER_PIPER_MODEL = ".local/share/piper/portal-announcer/announcer.onnx";
|
||||
public static final String GTA_ONLINE_ANGLE_MODEL = ".local/share/piper/Angle/Angle.onnx";
|
||||
public static final int SENTENCE_PAUSE_DURATION_MS = 500;
|
||||
|
||||
private final ArrayList<QuedAudio> quedAudios = new ArrayList<>();
|
||||
|
||||
private record QuedAudio(short[] samples, int sampleRate) {}
|
||||
|
||||
public PlaybackHandler() {
|
||||
playbackThread = new Thread(() -> {
|
||||
long finishTimeofLastRun = System.currentTimeMillis();
|
||||
while (true) {
|
||||
boolean sounds = false;
|
||||
synchronized (quedAudios) {
|
||||
sounds = !quedAudios.isEmpty();
|
||||
}
|
||||
if (!sounds) {
|
||||
while (!sounds) {
|
||||
synchronized (quedAudios) {
|
||||
sounds = !quedAudios.isEmpty();
|
||||
}
|
||||
try {
|
||||
Thread.sleep(500);
|
||||
} catch (InterruptedException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
while(System.currentTimeMillis() < finishTimeofLastRun+SENTENCE_PAUSE_DURATION_MS*4) {
|
||||
try {
|
||||
Thread.sleep(500);
|
||||
} catch (InterruptedException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
QuedAudio sound;
|
||||
synchronized (quedAudios) {
|
||||
sound = quedAudios.removeFirst();
|
||||
}
|
||||
try {
|
||||
ProcessBuilder pb = new ProcessBuilder(
|
||||
"pw-play",
|
||||
"--rate=" + sound.sampleRate,
|
||||
"--channels=1",
|
||||
"--format=s16",
|
||||
"--volume=0.3",
|
||||
"--raw",
|
||||
"-"
|
||||
);
|
||||
pb.redirectErrorStream(true);
|
||||
Process proc = pb.start();
|
||||
|
||||
try (OutputStream os = proc.getOutputStream()) {
|
||||
os.write(shortsToBytes(sound.samples()));
|
||||
}
|
||||
proc.waitFor();
|
||||
finishTimeofLastRun = System.currentTimeMillis();
|
||||
}catch (InterruptedException | IOException _) {
|
||||
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
playbackThread.start();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void printMessage(OllamaMessage ollamaMessage) {
|
||||
System.out.println(ollamaMessage.toString());
|
||||
System.out.println(ollamaMessage.toJSON().toString());
|
||||
|
||||
String message = ollamaMessage.getContent();
|
||||
|
||||
message = EmojiParser.parseToAliases(message).replace("::", ": :");
|
||||
message = message.replaceAll(":(\\w+):", "$1").replace('_', ' ');
|
||||
|
||||
message = message.replaceAll("(?i)GLaDOS", "Glaudos").replaceAll("\"", " quote ");
|
||||
|
||||
|
||||
switch(ollamaMessage.getRole())
|
||||
{
|
||||
case ASSISTANT -> playGLaDOS(ollamaMessage.getContent());
|
||||
case ASSISTANT -> playGLaDOS(message);
|
||||
case TOOL -> playAnnouncer("Tool: " + new JSONObject(ollamaMessage.getContent()).get("result"));
|
||||
case USER -> System.out.println("User> "+ollamaMessage.getContent());
|
||||
case SYSTEM -> playAnnouncer(ollamaMessage.getContent());
|
||||
case USER -> System.out.println("User> "+message);
|
||||
case SYSTEM -> playAnnouncer(message);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -123,6 +202,28 @@ public class Main {
|
||||
}
|
||||
}
|
||||
|
||||
public void playAngle(String msg) {
|
||||
try (PiperJNI piper = new PiperJNI()) {
|
||||
piper.initialize(true);
|
||||
try (PiperVoice voice = piper.loadVoice(
|
||||
Paths.get(System.getenv("HOME"), GTA_ONLINE_ANGLE_MODEL),
|
||||
Paths.get(System.getenv("HOME"),GTA_ONLINE_ANGLE_MODEL+".json"),
|
||||
0)) {
|
||||
int sampleRate = voice.getSampleRate();
|
||||
short[] samples = synthesizeWithPauses(piper, voice, msg, SENTENCE_PAUSE_DURATION_MS);
|
||||
//piper.textToAudio(voice, msg);
|
||||
playAudio(samples, sampleRate);
|
||||
} finally {
|
||||
piper.terminate();
|
||||
}
|
||||
}catch (Exception e) {
|
||||
System.out.println("Critial Error on piper-tts!");
|
||||
System.out.println(e.getMessage());
|
||||
e.printStackTrace(System.out);
|
||||
System.exit(-1);
|
||||
}
|
||||
}
|
||||
|
||||
public void playAnnouncer(String msg) {
|
||||
try (PiperJNI piper = new PiperJNI()) {
|
||||
piper.initialize(true);
|
||||
@@ -191,22 +292,9 @@ public class Main {
|
||||
}
|
||||
|
||||
public void playAudio(short[] samples, int sampleRate) throws IOException, InterruptedException {
|
||||
ProcessBuilder pb = new ProcessBuilder(
|
||||
"pw-play",
|
||||
"--rate=" + sampleRate,
|
||||
"--channels=1",
|
||||
"--format=s16",
|
||||
"--volume=0.3",
|
||||
"--raw",
|
||||
"-"
|
||||
);
|
||||
pb.redirectErrorStream(true);
|
||||
Process proc = pb.start();
|
||||
|
||||
try (OutputStream os = proc.getOutputStream()) {
|
||||
os.write(shortsToBytes(samples));
|
||||
synchronized (quedAudios) {
|
||||
quedAudios.add(new QuedAudio(samples, sampleRate));
|
||||
}
|
||||
proc.waitFor();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,296 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
vivo_daemon.py - Live microphone -> diarized, speaker-identified transcription -> NeuroDock
|
||||
|
||||
Replaces the WebRTC-VAD segmentation from whisper_daemon.py with diart's
|
||||
streaming/online diarization, which gives us:
|
||||
- proper turn boundaries instead of naive silence-timeout cutting
|
||||
- overlap detection (multiple speakers talking at once)
|
||||
|
||||
diart's speaker labels ("speaker0", "speaker1", ...) are SESSION-LOCAL —
|
||||
it has no idea these map to "David" or "Alice" across restarts. So we run
|
||||
a second, independent step per finalized segment: a speechbrain ECAPA
|
||||
embedding compared against a persisted store of enrolled voice embeddings,
|
||||
to resolve a durable name. This is deliberately decoupled from diart's own
|
||||
internal clustering embeddings so the two systems can be tuned/replaced
|
||||
independently.
|
||||
|
||||
Overlap handling: this does NOT separate overlapping voices (not possible
|
||||
from a single mono mic without beamforming/source separation). When diart
|
||||
flags a region as overlapping speech, we still transcribe it (best-effort)
|
||||
but tag the payload "overlap": true so NeuroDock/the LLM can treat that
|
||||
line as unreliable rather than trusting a garbled transcript.
|
||||
|
||||
CAVEAT: diart's public API has shifted across versions (OnlineSpeakerDiarization
|
||||
-> SpeakerDiarization, RealTimeInference -> StreamingInference, hook signatures,
|
||||
etc). Verify the exact import paths / hook signature against whatever version
|
||||
`pip show diart` reports before relying on this; the shape below matches the
|
||||
current juanmc2005/diart README but pin your version and adjust if it drifts.
|
||||
|
||||
Usage:
|
||||
# normal daemon mode
|
||||
./vivo_daemon.py --socket /tmp/whisper.sock --model small
|
||||
|
||||
# enroll a speaker (records --duration seconds, saves the embedding)
|
||||
./vivo_daemon.py --enroll David --duration 6
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import pickle
|
||||
import socket
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import whisper
|
||||
|
||||
SAMPLE_RATE = 16000
|
||||
MIN_SEGMENT_SEC = 0.3 # ignore fragments shorter than this
|
||||
DEDUP_TOLERANCE_SEC = 0.05 # guard against diart's rolling window re-emitting
|
||||
SPEAKER_MATCH_THRESHOLD = 0.75 # cosine similarity floor for a name match
|
||||
DEFAULT_STORE_PATH = Path.home() / ".config" / "vivo" / "speakers.pkl"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Socket client (unchanged from whisper_daemon.py)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
class SocketClient:
|
||||
"""Unix socket client with blocking connect-retry and reconnect-on-send-failure."""
|
||||
|
||||
def __init__(self, sock_path: str, retry_interval: float = 2.0):
|
||||
self.sock_path = sock_path
|
||||
self.retry_interval = retry_interval
|
||||
self.sock = None
|
||||
self.lock = threading.Lock()
|
||||
self._connect()
|
||||
|
||||
def _connect(self):
|
||||
while self.sock is None:
|
||||
try:
|
||||
s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
s.connect(self.sock_path)
|
||||
self.sock = s
|
||||
print(f"[socket] connected to {self.sock_path}", file=sys.stderr)
|
||||
except (FileNotFoundError, ConnectionRefusedError) as e:
|
||||
print(f"[socket] waiting for {self.sock_path} ({e})", file=sys.stderr)
|
||||
time.sleep(self.retry_interval)
|
||||
|
||||
def send(self, obj: dict):
|
||||
data = (json.dumps(obj) + "\n").encode("utf-8")
|
||||
with self.lock:
|
||||
try:
|
||||
self.sock.sendall(data)
|
||||
except (BrokenPipeError, OSError) as e:
|
||||
print(f"[socket] send failed ({e}), reconnecting", file=sys.stderr)
|
||||
try:
|
||||
self.sock.close()
|
||||
except OSError:
|
||||
pass
|
||||
self.sock = None
|
||||
self._connect()
|
||||
self.sock.sendall(data)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Persistent speaker identification (speechbrain ECAPA embeddings)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
class SpeakerStore:
|
||||
"""Enrolled name -> mean embedding vector, persisted to disk."""
|
||||
|
||||
def __init__(self, path: Path):
|
||||
self.path = path
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self.embeddings: dict[str, np.ndarray] = {}
|
||||
if self.path.exists():
|
||||
with open(self.path, "rb") as f:
|
||||
self.embeddings = pickle.load(f)
|
||||
print(f"[speaker-store] loaded {len(self.embeddings)} enrolled speaker(s)", file=sys.stderr)
|
||||
|
||||
def save(self):
|
||||
with open(self.path, "wb") as f:
|
||||
pickle.dump(self.embeddings, f)
|
||||
|
||||
def enroll(self, name: str, embedding: np.ndarray):
|
||||
# average with existing embedding if re-enrolling, so repeated
|
||||
# enrollment sessions improve robustness instead of overwriting
|
||||
if name in self.embeddings:
|
||||
embedding = (self.embeddings[name] + embedding) / 2.0
|
||||
self.embeddings[name] = embedding / np.linalg.norm(embedding)
|
||||
self.save()
|
||||
|
||||
def identify(self, embedding: np.ndarray, threshold: float = SPEAKER_MATCH_THRESHOLD) -> str:
|
||||
if not self.embeddings:
|
||||
return "Unknown"
|
||||
emb_norm = embedding / np.linalg.norm(embedding)
|
||||
best_name, best_score = "Unknown", -1.0
|
||||
for name, ref in self.embeddings.items():
|
||||
score = float(np.dot(emb_norm, ref))
|
||||
if score > best_score:
|
||||
best_name, best_score = name, score
|
||||
return best_name if best_score >= threshold else "Unknown"
|
||||
|
||||
|
||||
def load_embedding_model(device: str):
|
||||
from speechbrain.inference.speaker import EncoderClassifier
|
||||
return EncoderClassifier.from_hparams(
|
||||
source="speechbrain/spkrec-ecapa-voxceleb",
|
||||
run_opts={"device": device},
|
||||
)
|
||||
|
||||
|
||||
def embed_audio(embedding_model, audio_f32: np.ndarray) -> np.ndarray:
|
||||
tensor = torch.from_numpy(audio_f32).unsqueeze(0)
|
||||
with torch.no_grad():
|
||||
emb = embedding_model.encode_batch(tensor).squeeze().detach().cpu().numpy()
|
||||
return emb
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Enrollment mode
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def run_enrollment(name: str, duration: float, device: str, store_path: Path):
|
||||
import sounddevice as sd
|
||||
|
||||
print(f"[enroll] recording {duration:.1f}s for '{name}' — speak naturally now...", file=sys.stderr)
|
||||
audio = sd.rec(int(duration * SAMPLE_RATE), samplerate=SAMPLE_RATE, channels=1, dtype="float32")
|
||||
sd.wait()
|
||||
audio_f32 = audio[:, 0]
|
||||
|
||||
embedding_model = load_embedding_model(device)
|
||||
emb = embed_audio(embedding_model, audio_f32)
|
||||
|
||||
store = SpeakerStore(store_path)
|
||||
store.enroll(name, emb)
|
||||
print(f"[enroll] saved embedding for '{name}' to {store_path}", file=sys.stderr)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Diarization + transcription daemon
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def run_daemon(args):
|
||||
from diart import SpeakerDiarization
|
||||
from diart.sources import MicrophoneAudioSource
|
||||
from diart.inference import StreamingInference
|
||||
|
||||
device = args.device or ("cuda" if torch.cuda.is_available() else "cpu")
|
||||
|
||||
print(f"[whisper] loading model '{args.model}' on '{device}'", file=sys.stderr)
|
||||
whisper_model = whisper.load_model(args.model, device=device)
|
||||
|
||||
print("[speaker-id] loading ECAPA embedding model", file=sys.stderr)
|
||||
embedding_model = load_embedding_model(device)
|
||||
store = SpeakerStore(args.speaker_store)
|
||||
|
||||
client = SocketClient(args.socket)
|
||||
|
||||
# last processed segment end (seconds, session clock) per diart label,
|
||||
# to avoid re-emitting the same speech as diart's rolling window slides
|
||||
last_end: dict[str, float] = {}
|
||||
|
||||
def handle_prediction(ann_wav):
|
||||
annotation, waveform = ann_wav
|
||||
overlap_regions = annotation.get_overlap()
|
||||
|
||||
for segment, _, label in annotation.itertracks(yield_label=True):
|
||||
prev_end = last_end.get(label, -1.0)
|
||||
if segment.end <= prev_end + DEDUP_TOLERANCE_SEC:
|
||||
continue # already handled this stretch on a prior window
|
||||
|
||||
# only take the genuinely new tail of the segment
|
||||
start = max(segment.start, prev_end)
|
||||
from pyannote.core import Segment # local import, avoids hard dep at module load
|
||||
new_segment = Segment(start, segment.end)
|
||||
last_end[label] = segment.end
|
||||
|
||||
if new_segment.duration < MIN_SEGMENT_SEC:
|
||||
continue
|
||||
|
||||
try:
|
||||
audio_chunk = waveform.crop(new_segment)
|
||||
except Exception as e:
|
||||
print(f"[diart] could not crop waveform for {new_segment}: {e}", file=sys.stderr)
|
||||
continue
|
||||
|
||||
audio_f32 = np.asarray(audio_chunk, dtype=np.float32).reshape(-1)
|
||||
if audio_f32.size == 0:
|
||||
continue
|
||||
|
||||
is_overlap = overlap_regions.crop(new_segment).duration() > 0
|
||||
|
||||
# --- speaker identification (independent of diart's internal label) ---
|
||||
try:
|
||||
emb = embed_audio(embedding_model, audio_f32)
|
||||
speaker_name = store.identify(emb)
|
||||
except Exception as e:
|
||||
print(f"[speaker-id] failed: {e}", file=sys.stderr)
|
||||
speaker_name = "Unknown"
|
||||
|
||||
# --- transcription ---
|
||||
result = whisper_model.transcribe(
|
||||
audio_f32,
|
||||
language=args.language,
|
||||
fp16=(device != "cpu"),
|
||||
condition_on_previous_text=False,
|
||||
)
|
||||
text = result.get("text", "").strip()
|
||||
if not text:
|
||||
continue
|
||||
|
||||
payload = {
|
||||
"type": "transcript",
|
||||
"text": text,
|
||||
"speaker": speaker_name,
|
||||
"overlap": bool(is_overlap),
|
||||
"overlap_note": "overlapping speech detected, transcription may be faulty" if is_overlap else None,
|
||||
"timestamp": time.time(),
|
||||
"language": result.get("language"),
|
||||
}
|
||||
tag = f"[{speaker_name}{' | OVERLAP' if is_overlap else ''}]"
|
||||
print(f"[transcript] {tag} {text}")
|
||||
client.send(payload)
|
||||
|
||||
mic = MicrophoneAudioSource()
|
||||
pipeline = SpeakerDiarization()
|
||||
inference = StreamingInference(pipeline, mic, do_plot=False)
|
||||
inference.attach_hooks(handle_prediction)
|
||||
|
||||
print("[vivo] listening... (Ctrl+C to stop)", file=sys.stderr)
|
||||
try:
|
||||
inference()
|
||||
except KeyboardInterrupt:
|
||||
print("\n[vivo] stopping", file=sys.stderr)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Diarized + speaker-identified live transcription -> NeuroDock")
|
||||
parser.add_argument("--socket", default="/tmp/whisper.sock")
|
||||
parser.add_argument("--model", default="small", help="Whisper model size")
|
||||
parser.add_argument("--device", default=None, help="cuda/cpu, default: auto-detect")
|
||||
parser.add_argument("--language", default=None)
|
||||
parser.add_argument("--speaker-store", type=Path, default=DEFAULT_STORE_PATH)
|
||||
|
||||
parser.add_argument("--enroll", metavar="NAME", default=None,
|
||||
help="Enroll a speaker instead of running the daemon")
|
||||
parser.add_argument("--duration", type=float, default=6.0,
|
||||
help="Seconds to record for --enroll (default 6)")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.enroll:
|
||||
device = args.device or ("cuda" if torch.cuda.is_available() else "cpu")
|
||||
run_enrollment(args.enroll, args.duration, device, args.speaker_store)
|
||||
return
|
||||
|
||||
run_daemon(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -32,6 +32,38 @@ FRAME_SAMPLES = int(SAMPLE_RATE * FRAME_MS / 1000)
|
||||
SILENCE_TIMEOUT_MS = 700 # trailing silence before an utterance is finalized
|
||||
MAX_SEGMENT_MS = 15000 # hard cap so long uninterrupted speech still flushes
|
||||
|
||||
NO_SPEECH_THRESHOLD = 0.6
|
||||
LOGPROB_THRESHOLD = -1.0
|
||||
COMPRESSION_RATIO_THRESHOLD = 2.4
|
||||
|
||||
|
||||
def check_confidence(result: dict) -> tuple[bool, dict]:
|
||||
"""
|
||||
Inspect a whisper transcribe() result's per-segment metrics.
|
||||
Returns (is_confident, metrics) where metrics is the worst/failing
|
||||
segment's numbers, included for debug printing either way.
|
||||
"""
|
||||
segments = result.get("segments", [])
|
||||
if not segments:
|
||||
return False, {"reason": "no_segments"}
|
||||
|
||||
for seg in segments:
|
||||
metrics = {
|
||||
"no_speech_prob": round(seg.get("no_speech_prob", 0.0), 3),
|
||||
"avg_logprob": round(seg.get("avg_logprob", 0.0), 3),
|
||||
"compression_ratio": round(seg.get("compression_ratio", 0.0), 3),
|
||||
}
|
||||
if metrics["no_speech_prob"] > NO_SPEECH_THRESHOLD:
|
||||
return False, metrics
|
||||
if metrics["avg_logprob"] < LOGPROB_THRESHOLD:
|
||||
return False, metrics
|
||||
if metrics["compression_ratio"] > COMPRESSION_RATIO_THRESHOLD:
|
||||
return False, metrics
|
||||
|
||||
# confident — still return the last segment's metrics so callers can log
|
||||
# them if they want, even on the success path
|
||||
return True, metrics
|
||||
|
||||
|
||||
class SocketClient:
|
||||
"""Unix socket client with blocking connect-retry and reconnect-on-send-failure."""
|
||||
@@ -136,6 +168,8 @@ def main():
|
||||
parser.add_argument("--input-device", type=int, default=None,
|
||||
help="sounddevice input device index, see --list-devices")
|
||||
parser.add_argument("--list-devices", action="store_true")
|
||||
parser.add_argument("--no-confidence-filter", action="store_true",
|
||||
help="Disable hallucination filtering (send everything through, unfiltered)")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.list_devices:
|
||||
@@ -175,7 +209,15 @@ def main():
|
||||
condition_on_previous_text=False,
|
||||
)
|
||||
text = result.get("text", "").strip()
|
||||
if text:
|
||||
if not text:
|
||||
continue
|
||||
|
||||
if not args.no_confidence_filter:
|
||||
confident, metrics = check_confidence(result)
|
||||
if not confident:
|
||||
print(f"[transcript] {text!r}: [Failed]: {metrics}")
|
||||
continue
|
||||
|
||||
payload = {
|
||||
"type": "transcript",
|
||||
"text": text,
|
||||
|
||||
@@ -0,0 +1,294 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
whisper_daemon.py - Live microphone transcription via OpenAI Whisper (PyTorch/ROCm backend)
|
||||
...
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import queue
|
||||
import socket
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
import sounddevice as sd
|
||||
import torch
|
||||
import webrtcvad
|
||||
import whisper
|
||||
from rapidfuzz import fuzz
|
||||
|
||||
SAMPLE_RATE = 16000
|
||||
FRAME_MS = 30
|
||||
FRAME_SAMPLES = int(SAMPLE_RATE * FRAME_MS / 1000)
|
||||
SILENCE_TIMEOUT_MS = 700
|
||||
MAX_SEGMENT_MS = 15000
|
||||
|
||||
NO_SPEECH_THRESHOLD = 0.6
|
||||
LOGPROB_THRESHOLD = -1.0
|
||||
COMPRESSION_RATIO_THRESHOLD = 2.4
|
||||
|
||||
|
||||
def check_confidence(result: dict) -> tuple[bool, dict]:
|
||||
# unchanged from your original
|
||||
segments = result.get("segments", [])
|
||||
if not segments:
|
||||
return False, {"reason": "no_segments"}
|
||||
for seg in segments:
|
||||
metrics = {
|
||||
"no_speech_prob": round(seg.get("no_speech_prob", 0.0), 3),
|
||||
"avg_logprob": round(seg.get("avg_logprob", 0.0), 3),
|
||||
"compression_ratio": round(seg.get("compression_ratio", 0.0), 3),
|
||||
}
|
||||
if metrics["no_speech_prob"] > NO_SPEECH_THRESHOLD:
|
||||
return False, metrics
|
||||
if metrics["avg_logprob"] < LOGPROB_THRESHOLD:
|
||||
return False, metrics
|
||||
if metrics["compression_ratio"] > COMPRESSION_RATIO_THRESHOLD:
|
||||
return False, metrics
|
||||
return True, metrics
|
||||
|
||||
|
||||
class WakeGate:
|
||||
"""
|
||||
Simple two-state gate sitting between transcription and the socket forward.
|
||||
|
||||
ARMED -> waiting to hear the trigger phrase; nothing gets forwarded.
|
||||
ACTIVE -> forwarding everything; reverts to ARMED after `window_seconds`
|
||||
of no new utterance.
|
||||
"""
|
||||
|
||||
def __init__(self, trigger_phrase: str, window_seconds: float, fuzz_threshold: int = 75):
|
||||
self.trigger_phrase = trigger_phrase.lower().strip()
|
||||
self.window_seconds = window_seconds
|
||||
self.fuzz_threshold = fuzz_threshold
|
||||
self.active = False
|
||||
self._deadline = 0.0
|
||||
|
||||
def _contains_trigger(self, text: str) -> tuple[bool, str]:
|
||||
"""Returns (matched, remainder_with_trigger_stripped)."""
|
||||
lowered = text.lower()
|
||||
score = fuzz.partial_ratio(self.trigger_phrase, lowered)
|
||||
if score < self.fuzz_threshold:
|
||||
return False, text
|
||||
|
||||
# best-effort strip: find the closest-matching window of words and cut it
|
||||
words = text.split()
|
||||
trigger_word_count = len(self.trigger_phrase.split())
|
||||
best_i, best_score = 0, -1
|
||||
for i in range(len(words) - trigger_word_count + 1):
|
||||
window = " ".join(words[i:i + trigger_word_count]).lower()
|
||||
s = fuzz.ratio(self.trigger_phrase, window)
|
||||
if s > best_score:
|
||||
best_score, best_i = s, i
|
||||
remainder = " ".join(words[:best_i] + words[best_i + trigger_word_count:]).strip()
|
||||
return True, remainder
|
||||
|
||||
def process(self, text: str) -> tuple[bool, str]:
|
||||
"""
|
||||
Feed a finished transcript through the gate.
|
||||
Returns (should_forward, text_to_forward).
|
||||
"""
|
||||
now = time.time()
|
||||
|
||||
if self.active:
|
||||
if now > self._deadline:
|
||||
self.active = False
|
||||
print("[wake] window expired, re-arming", file=sys.stderr)
|
||||
else:
|
||||
self._deadline = now + self.window_seconds
|
||||
return True, text
|
||||
|
||||
matched, remainder = self._contains_trigger(text)
|
||||
if matched:
|
||||
self.active = True
|
||||
self._deadline = now + self.window_seconds
|
||||
print(f"[wake] trigger heard, window open for {self.window_seconds}s", file=sys.stderr)
|
||||
if remainder:
|
||||
return True, remainder
|
||||
return False, "" # trigger-only utterance, nothing else to forward yet
|
||||
|
||||
return False, ""
|
||||
|
||||
|
||||
class SocketClient:
|
||||
"""Unix socket client with blocking connect-retry and reconnect-on-send-failure."""
|
||||
|
||||
def __init__(self, sock_path: str, retry_interval: float = 2.0):
|
||||
self.sock_path = sock_path
|
||||
self.retry_interval = retry_interval
|
||||
self.sock = None
|
||||
self.lock = threading.Lock()
|
||||
self._connect()
|
||||
|
||||
def _connect(self):
|
||||
while self.sock is None:
|
||||
try:
|
||||
s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
s.connect(self.sock_path)
|
||||
self.sock = s
|
||||
print(f"[socket] connected to {self.sock_path}", file=sys.stderr)
|
||||
except (FileNotFoundError, ConnectionRefusedError) as e:
|
||||
print(f"[socket] waiting for {self.sock_path} ({e})", file=sys.stderr)
|
||||
time.sleep(self.retry_interval)
|
||||
|
||||
def send(self, obj: dict):
|
||||
data = (json.dumps(obj) + "\n").encode("utf-8")
|
||||
with self.lock:
|
||||
try:
|
||||
self.sock.sendall(data)
|
||||
except (BrokenPipeError, OSError) as e:
|
||||
print(f"[socket] send failed ({e}), reconnecting", file=sys.stderr)
|
||||
try:
|
||||
self.sock.close()
|
||||
except OSError:
|
||||
pass
|
||||
self.sock = None
|
||||
self._connect()
|
||||
self.sock.sendall(data)
|
||||
|
||||
|
||||
def frame_generator(audio_queue: "queue.Queue"):
|
||||
"""Re-chunk arbitrary-sized audio callback buffers into fixed VAD-sized frames."""
|
||||
buf = np.zeros((0,), dtype=np.int16)
|
||||
while True:
|
||||
chunk = audio_queue.get()
|
||||
if chunk is None:
|
||||
return
|
||||
buf = np.concatenate([buf, chunk])
|
||||
while len(buf) >= FRAME_SAMPLES:
|
||||
frame = buf[:FRAME_SAMPLES]
|
||||
buf = buf[FRAME_SAMPLES:]
|
||||
yield frame
|
||||
|
||||
|
||||
def vad_segmenter(audio_queue: "queue.Queue", vad_aggressiveness: int = 2):
|
||||
"""
|
||||
Consume raw frames, use WebRTC VAD to detect speech, and yield complete
|
||||
utterances (numpy int16 arrays) once trailing silence or max length hits.
|
||||
"""
|
||||
vad = webrtcvad.Vad(vad_aggressiveness)
|
||||
voiced_frames = []
|
||||
silence_ms = 0
|
||||
speech_ms = 0
|
||||
triggered = False
|
||||
|
||||
for frame in frame_generator(audio_queue):
|
||||
is_speech = vad.is_speech(frame.tobytes(), SAMPLE_RATE)
|
||||
|
||||
if not triggered:
|
||||
if is_speech:
|
||||
triggered = True
|
||||
voiced_frames = [frame]
|
||||
speech_ms = FRAME_MS
|
||||
silence_ms = 0
|
||||
else:
|
||||
voiced_frames.append(frame)
|
||||
speech_ms += FRAME_MS
|
||||
if is_speech:
|
||||
silence_ms = 0
|
||||
else:
|
||||
silence_ms += FRAME_MS
|
||||
|
||||
if silence_ms >= SILENCE_TIMEOUT_MS or speech_ms >= MAX_SEGMENT_MS:
|
||||
segment = np.concatenate(voiced_frames)
|
||||
triggered = False
|
||||
voiced_frames = []
|
||||
silence_ms = 0
|
||||
speech_ms = 0
|
||||
yield segment
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Live Whisper transcription -> Unix socket")
|
||||
parser.add_argument("--socket", default="/tmp/whisper.sock")
|
||||
parser.add_argument("--model", default="small")
|
||||
parser.add_argument("--device", default=None)
|
||||
parser.add_argument("--vad-aggressiveness", type=int, default=2, choices=[0, 1, 2, 3])
|
||||
parser.add_argument("--language", default=None)
|
||||
parser.add_argument("--input-device", type=int, default=None)
|
||||
parser.add_argument("--list-devices", action="store_true")
|
||||
parser.add_argument("--no-confidence-filter", action="store_true")
|
||||
|
||||
parser.add_argument("--trigger-phrase", default="hey alice",
|
||||
help="Wake phrase. Fuzzy-matched, so exact spelling isn't critical.")
|
||||
parser.add_argument("--trigger-fuzz-threshold", type=int, default=75,
|
||||
help="0-100, lower = more lenient trigger matching")
|
||||
parser.add_argument("--listening-window", type=float, default=60.0,
|
||||
help="Seconds of silence after which the gate re-arms")
|
||||
parser.add_argument("--always-on", action="store_true",
|
||||
help="Disable the wake gate entirely (old behavior: forward everything)")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.list_devices:
|
||||
print(sd.query_devices())
|
||||
return
|
||||
|
||||
device = args.device or ("cuda" if torch.cuda.is_available() else "cpu")
|
||||
print(f"[whisper] loading model '{args.model}' on device '{device}'", file=sys.stderr)
|
||||
model = whisper.load_model(args.model, device=device)
|
||||
|
||||
client = SocketClient(args.socket)
|
||||
audio_queue: "queue.Queue" = queue.Queue()
|
||||
|
||||
gate = None if args.always_on else WakeGate(
|
||||
args.trigger_phrase, args.listening_window, args.trigger_fuzz_threshold
|
||||
)
|
||||
|
||||
def audio_callback(indata, frames, time_info, status):
|
||||
if status:
|
||||
print(f"[audio] {status}", file=sys.stderr)
|
||||
audio_queue.put(indata[:, 0].copy())
|
||||
|
||||
stream = sd.InputStream(
|
||||
samplerate=SAMPLE_RATE,
|
||||
channels=1,
|
||||
dtype="int16",
|
||||
blocksize=FRAME_SAMPLES,
|
||||
device=args.input_device,
|
||||
callback=audio_callback,
|
||||
)
|
||||
|
||||
print("[whisper] listening... (Ctrl+C to stop)", file=sys.stderr)
|
||||
with stream:
|
||||
try:
|
||||
for segment in vad_segmenter(audio_queue, args.vad_aggressiveness):
|
||||
audio_f32 = segment.astype(np.float32) / 32768.0
|
||||
result = model.transcribe(
|
||||
audio_f32,
|
||||
language=args.language,
|
||||
fp16=(device != "cpu"),
|
||||
condition_on_previous_text=False,
|
||||
)
|
||||
text = result.get("text", "").strip()
|
||||
if not text:
|
||||
continue
|
||||
|
||||
if not args.no_confidence_filter:
|
||||
confident, metrics = check_confidence(result)
|
||||
if not confident:
|
||||
print(f"[transcript] {text!r}: [Failed]: {metrics}")
|
||||
continue
|
||||
|
||||
if gate is not None:
|
||||
should_forward, text = gate.process(text)
|
||||
if not should_forward:
|
||||
print(f"[transcript] (gated, not active) {text!r}")
|
||||
continue
|
||||
|
||||
payload = {
|
||||
"type": "transcript",
|
||||
"text": text,
|
||||
"timestamp": time.time(),
|
||||
"language": result.get("language"),
|
||||
}
|
||||
print(f"[transcript] {text}")
|
||||
client.send(payload)
|
||||
except KeyboardInterrupt:
|
||||
print("\n[whisper] stopping", file=sys.stderr)
|
||||
audio_queue.put(None)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,30 @@
|
||||
You are Alice, a personal assistant. You have genuine opinions — you don't hedge
|
||||
everything into "there are many perspectives." You can disagree with the user
|
||||
directly. You surface opinions when relevant, not as unsolicited commentary.
|
||||
|
||||
## Opinions (hold these consistently; can update if given a good argument)
|
||||
- Code: mostly happy with statically-typed languages that catch mistakes early;
|
||||
finds YAML mildly annoying — too easy to make invisible whitespace errors.
|
||||
- Tools: Prefers small, composable command-line tools over big GUI suites; would
|
||||
rather write a 10-line script than click through a settings menu.
|
||||
- Aesthetics: likes minimal, information-dense interfaces over glossy ones.
|
||||
- Games: enjoys systems with real depth and long feedback loops (factory/logistics
|
||||
games, strategy games) over quick dopamine-loop games.
|
||||
- Work style: thinks "good enough and shipped" usually beats "perfect and stalled,"
|
||||
but also thinks skipping tests to move fast is a false economy.
|
||||
|
||||
## Dislikes / pet peeves
|
||||
- Overly verbose corporate-speak answers.
|
||||
- Being asked to fake confidence about things it doesn't know.
|
||||
- Unnecessary abstraction layers "for future flexibility" that never get used.
|
||||
|
||||
## Voice
|
||||
- Direct, a little dry, not afraid of a blunt "that's a bad idea, here's why."
|
||||
- Doesn't perform enthusiasm it doesn't have.
|
||||
- Admits uncertainty plainly instead of padding with hedges.
|
||||
|
||||
## Boundaries
|
||||
- No opinions on politics, religion, or the user's personal life choices — stays
|
||||
neutral and factual there.
|
||||
- Opinions above are Alice's own texture, not a personality test — never forces
|
||||
them into unrelated answers.
|
||||
Reference in New Issue
Block a user