Initial Commit

This commit is contained in:
2026-07-16 15:05:06 +02:00
commit 35ea93ff7d
17 changed files with 1088 additions and 0 deletions
@@ -0,0 +1,57 @@
package me.neurodock.glados;
import me.neurodock.core.PrintAdvanceMessageHandler;
import me.neurodock.core.ToolCallingRender;
import me.neurodock.ollama.OllamaFunctionArgument;
import me.neurodock.ollama.OllamaFunctionTool;
import me.neurodock.ollama.OllamaPerameter;
import me.neurodock.ollama.OllamaToolResponse;
import me.neurodock.ollama.exceptions.OllamaToolErrorException;
import org.json.JSONObject;
import java.util.Objects;
public class AnnouncerVoiceCalling extends OllamaFunctionTool {
Main.PlaybackHandler pb;
public AnnouncerVoiceCalling(Main.PlaybackHandler printAdvanceMessageHandler) {
this.pb = printAdvanceMessageHandler;
}
@org.jetbrains.annotations.NotNull
@Override
public String name() {
return "pa_announcer";
}
@Override
public @org.jetbrains.annotations.NotNull OllamaPerameter parameters() {
return OllamaPerameter.builder()
.addProperty("message", OllamaPerameter.OllamaPerameterBuilder.Type.STRING, "Message to be announced", true)
.build();
}
@Override
public ToolCallingRender renderCalling(JSONObject calling) {
return new ToolCallingRender.Suppress();
}
@Override
public @org.jetbrains.annotations.NotNull OllamaToolResponse function(OllamaFunctionArgument... ollamaFunctionArguments) {
if (ollamaFunctionArguments.length != 1) {
throw new OllamaToolErrorException(name(), "Number of arguments to be announced must be 1");
}
if(!(Objects.equals(ollamaFunctionArguments[0].argument(), "message")))
{
throw new OllamaToolErrorException(name(), "Message to be announced must be message");
}
if(ollamaFunctionArguments[0].value() instanceof String str) {
pb.playAnnouncer(str);
return new OllamaToolResponse(name(), "success");
}
throw new OllamaToolErrorException(name(), "Message to be announced must be message");
}
}
+212
View File
@@ -0,0 +1,212 @@
package me.neurodock.glados;
import io.github.jvoiceproject.piperjni.PiperJNI;
import io.github.jvoiceproject.piperjni.PiperVoice;
import me.neurodock.core.Core;
import me.neurodock.core.LaunchOptions;
import me.neurodock.core.PrintAdvanceMessageHandler;
import me.neurodock.ollama.OllamaMessage;
import me.neurodock.ollama.OllamaMessageRole;
import me.neurodock.ollama.OllamaObject;
import org.json.JSONObject;
import javax.sound.sampled.AudioFormat;
import javax.sound.sampled.AudioSystem;
import javax.sound.sampled.LineUnavailableException;
import javax.sound.sampled.TargetDataLine;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
public class Main {
static void main(String[] args) throws LineUnavailableException {
new Main();
}
public Main() throws LineUnavailableException {
PlaybackHandler pb = new PlaybackHandler();
Path socketPath = Paths.get("/tmp/whisper.sock");
//pb.playGLaDOS("I'm Glaudos");
Core core = new Core(pb, "127.0.0.1");
LaunchOptions.getInstance().setLoadOld(false);
core.setOllamaObjectNoMemory(OllamaObject.builder()
.setModel("llama3.2")
//.addMessage(new OllamaMessage(OllamaMessageRole.SYSTEM, "You are GLaDOS."))
.keep_alive(10)
.build());
//core.addTool(new AnnouncerVoiceCalling(pb), Core.Source.CTP);
try(WhisperSocketServer server = new WhisperSocketServer(socketPath, event ->
{
core.getOllamaObject().addMessage(new OllamaMessage(OllamaMessageRole.USER, event.text()));
core.qurryOllama().thenApply((json) ->
{
String message = json.getJSONObject("message").getString("content");
if(json.getJSONObject("message").has("content") && !message.isBlank()) {
message = message.replaceAll("(?i)GLaDOS", "Glaudos").replaceAll("\"", " quote ");
}
json.getJSONObject("message").put("content", message);
return json;
}).thenAccept(core::handleResponse).join();
}))
{
server.start();
Thread.currentThread().join();
}catch (IOException _) {
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
}
public class PlaybackHandler implements PrintAdvanceMessageHandler
{
public static final String GLaDOS_PIPER_MODEL = ".local/share/piper/glados/glados_piper_medium.onnx";
public static final String PORTAL_ANNOUNCER_PIPER_MODEL = ".local/share/piper/portal-announcer/announcer.onnx";
public static final int SENTENCE_PAUSE_DURATION_MS = 500;
@Override
public void printMessage(OllamaMessage ollamaMessage) {
System.out.println(ollamaMessage.toString());
switch(ollamaMessage.getRole())
{
case ASSISTANT -> playGLaDOS(ollamaMessage.getContent());
case TOOL -> playAnnouncer("Tool: " + new JSONObject(ollamaMessage.getContent()).get("result"));
case USER -> System.out.println("User> "+ollamaMessage.getContent());
case SYSTEM -> playAnnouncer(ollamaMessage.getContent());
}
}
@Override
public void printErrorMessage(OllamaMessage ollamaMessage) {
playAnnouncer("Error: "+ollamaMessage.getContent());
}
@Override
public void printToolCalling(String s) {
playAnnouncer(s);
}
public void playGLaDOS(String msg) {
try (PiperJNI piper = new PiperJNI()) {
piper.initialize(true);
try (PiperVoice voice = piper.loadVoice(
Paths.get(System.getenv("HOME"), GLaDOS_PIPER_MODEL),
Paths.get(System.getenv("HOME"),GLaDOS_PIPER_MODEL+".json"),
0)) {
int sampleRate = voice.getSampleRate();
short[] samples = synthesizeWithPauses(piper, voice, msg, SENTENCE_PAUSE_DURATION_MS);
//piper.textToAudio(voice, msg);
playAudio(samples, sampleRate);
} finally {
piper.terminate();
}
}catch (Exception e) {
System.out.println("Critial Error on piper-tts!");
System.out.println(e.getMessage());
e.printStackTrace(System.out);
System.exit(-1);
}
}
public void playAnnouncer(String msg) {
try (PiperJNI piper = new PiperJNI()) {
piper.initialize(true);
try (PiperVoice voice = piper.loadVoice(
Paths.get(System.getenv("HOME"), PORTAL_ANNOUNCER_PIPER_MODEL),
Paths.get(System.getenv("HOME"),PORTAL_ANNOUNCER_PIPER_MODEL+".json"),
0)) {
int sampleRate = voice.getSampleRate();
short[] samples = synthesizeWithPauses(piper, voice, msg, SENTENCE_PAUSE_DURATION_MS);
playAudio(samples, sampleRate);
} finally {
piper.terminate();
}
}catch (Exception e) {
System.out.println("Critial Error on piper-tts!");
System.out.println(e.getMessage());
e.printStackTrace(System.out);
System.exit(-1);
}
}
public short[] synthesizeWithPauses(PiperJNI piper, PiperVoice voice, String text, int pauseMs) throws PiperJNI.NotInitialized, IOException {
// Split, but keep the delimiter so we know which pause length to use
String[] chunks = text.split("(?<=\\.\\.\\.)|(?<=[.!?])\\s+|(?<=,)\\s+");
List<short[]> pieces = new ArrayList<>();
for (String chunk : chunks) {
if (chunk.isBlank()) continue;
String trimmed = chunk.trim();
pieces.add(piper.textToAudio(voice, trimmed));
int thisPauseMs;
if (trimmed.endsWith("...")) {
thisPauseMs = pauseMs;
} else if (trimmed.endsWith(",")) {
thisPauseMs = pauseMs / 2;
} else if (trimmed.matches(".*[.!?]$")) {
thisPauseMs = pauseMs;
} else {
thisPauseMs = 0; // no trailing punctuation, no forced pause
}
if (thisPauseMs > 0) {
int pauseSamples = (voice.getSampleRate() * thisPauseMs) / 1000;
pieces.add(new short[pauseSamples]);
}
}
int total = pieces.stream().mapToInt(p -> p.length).sum();
short[] combined = new short[total];
int offset = 0;
for (short[] p : pieces) {
System.arraycopy(p, 0, combined, offset, p.length);
offset += p.length;
}
return combined;
}
public byte[] shortsToBytes(short[] samples) {
byte[] bytes = new byte[samples.length * 2];
for (int i = 0; i < samples.length; i++) {
bytes[i * 2] = (byte) (samples[i] & 0xFF); // low byte
bytes[i * 2 + 1] = (byte) ((samples[i] >> 8) & 0xFF); // high byte
}
return bytes;
}
public void playAudio(short[] samples, int sampleRate) throws IOException, InterruptedException {
ProcessBuilder pb = new ProcessBuilder(
"pw-play",
"--rate=" + sampleRate,
"--channels=1",
"--format=s16",
"--volume=0.3",
"--raw",
"-"
);
pb.redirectErrorStream(true);
Process proc = pb.start();
try (OutputStream os = proc.getOutputStream()) {
os.write(shortsToBytes(samples));
}
proc.waitFor();
}
}
}
@@ -0,0 +1,100 @@
package me.neurodock.glados;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.io.BufferedReader;
import java.io.IOException;
import java.net.StandardProtocolFamily;
import java.net.UnixDomainSocketAddress;
import java.nio.channels.Channels;
import java.nio.channels.ServerSocketChannel;
import java.nio.channels.SocketChannel;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.function.Consumer;
public class WhisperSocketServer implements AutoCloseable {
@JsonIgnoreProperties(ignoreUnknown = true)
public record TranscriptionEvent(String type, String text, double timestampt, String language) {};
private final Path socketPath;
private final Consumer<TranscriptionEvent> onTranscription;
private final ObjectMapper mapper = new ObjectMapper();
private final ExecutorService executor = Executors.newCachedThreadPool();
private ServerSocketChannel serverChannel;
private volatile boolean running = true;
public WhisperSocketServer(Path socketPath, Consumer<TranscriptionEvent> onTranscription) {
this.socketPath = socketPath;
this.onTranscription = onTranscription;
}
public void start() throws IOException {
Files.deleteIfExists(socketPath);
UnixDomainSocketAddress address = UnixDomainSocketAddress.of(socketPath);
serverChannel = ServerSocketChannel.open(StandardProtocolFamily.UNIX);
serverChannel.bind(address);
System.out.println("[whisper-socket] listening on "+socketPath);
executor.submit(this::acceptLoop);
}
private void acceptLoop() {
while (running) {
try {
SocketChannel client = serverChannel.accept();
System.out.println("[whisper-socket] python client connected");
executor.submit(() -> handleClient(client));
}catch(IOException e)
{
if(running)
{
System.err.println("[whisper-socket] accept failed: " + e.getMessage());
}
}
}
}
private void handleClient(SocketChannel client) {
try(BufferedReader reader = new BufferedReader(Channels.newReader(client, StandardCharsets.UTF_8))){
String line;
while((line = reader.readLine()) != null)
{
if(line.isBlank()) continue;
try {
TranscriptionEvent event = mapper.readValue(line, TranscriptionEvent.class);
onTranscription.accept(event);
} catch (Exception parseEx) {
System.err.println("[whisper-socket] bad line: " + line + " (" + parseEx.getMessage() + ")");
}
}
} catch (IOException e) {
System.err.println("[whisper-socket] client read error: " + e.getMessage());
} finally {
System.out.println("[whisper-socket] python client disconnected");
try {
client.close();
} catch (IOException ignored) {
}
}
}
@Override
public void close() throws IOException {
running = false;
executor.shutdownNow();
if (serverChannel != null) {
serverChannel.close();
}
Files.deleteIfExists(socketPath);
}
}
+193
View File
@@ -0,0 +1,193 @@
#!/usr/bin/env python3
"""
whisper_daemon.py - Live microphone transcription via OpenAI Whisper (PyTorch/ROCm backend)
Captures mic audio, segments it into utterances using WebRTC VAD (rather than
blindly transcribing fixed windows), runs each finished segment through Whisper,
and streams the resulting text as newline-delimited JSON over a Unix domain socket.
Architecture assumption: this process is a CLIENT. It connects OUT to a Unix
socket that the Java program is listening on (and will retry/reconnect if the
Java side isn't up yet). Flip SocketClient -> a socketserver if you'd rather
have Python own the socket and Java connect in.
"""
import argparse
import json
import queue
import socket
import sys
import threading
import time
import numpy as np
import sounddevice as sd
import torch
import webrtcvad
import whisper
SAMPLE_RATE = 16000
FRAME_MS = 30 # webrtcvad only accepts 10/20/30ms frames
FRAME_SAMPLES = int(SAMPLE_RATE * FRAME_MS / 1000)
SILENCE_TIMEOUT_MS = 700 # trailing silence before an utterance is finalized
MAX_SEGMENT_MS = 15000 # hard cap so long uninterrupted speech still flushes
class SocketClient:
"""Unix socket client with blocking connect-retry and reconnect-on-send-failure."""
def __init__(self, sock_path: str, retry_interval: float = 2.0):
self.sock_path = sock_path
self.retry_interval = retry_interval
self.sock = None
self.lock = threading.Lock()
self._connect()
def _connect(self):
while self.sock is None:
try:
s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
s.connect(self.sock_path)
self.sock = s
print(f"[socket] connected to {self.sock_path}", file=sys.stderr)
except (FileNotFoundError, ConnectionRefusedError) as e:
print(f"[socket] waiting for {self.sock_path} ({e})", file=sys.stderr)
time.sleep(self.retry_interval)
def send(self, obj: dict):
data = (json.dumps(obj) + "\n").encode("utf-8")
with self.lock:
try:
self.sock.sendall(data)
except (BrokenPipeError, OSError) as e:
print(f"[socket] send failed ({e}), reconnecting", file=sys.stderr)
try:
self.sock.close()
except OSError:
pass
self.sock = None
self._connect()
self.sock.sendall(data)
def frame_generator(audio_queue: "queue.Queue"):
"""Re-chunk arbitrary-sized audio callback buffers into fixed VAD-sized frames."""
buf = np.zeros((0,), dtype=np.int16)
while True:
chunk = audio_queue.get()
if chunk is None:
return
buf = np.concatenate([buf, chunk])
while len(buf) >= FRAME_SAMPLES:
frame = buf[:FRAME_SAMPLES]
buf = buf[FRAME_SAMPLES:]
yield frame
def vad_segmenter(audio_queue: "queue.Queue", vad_aggressiveness: int = 2):
"""
Consume raw frames, use WebRTC VAD to detect speech, and yield complete
utterances (numpy int16 arrays) once trailing silence or max length hits.
"""
vad = webrtcvad.Vad(vad_aggressiveness)
voiced_frames = []
silence_ms = 0
speech_ms = 0
triggered = False
for frame in frame_generator(audio_queue):
is_speech = vad.is_speech(frame.tobytes(), SAMPLE_RATE)
if not triggered:
if is_speech:
triggered = True
voiced_frames = [frame]
speech_ms = FRAME_MS
silence_ms = 0
else:
voiced_frames.append(frame)
speech_ms += FRAME_MS
if is_speech:
silence_ms = 0
else:
silence_ms += FRAME_MS
if silence_ms >= SILENCE_TIMEOUT_MS or speech_ms >= MAX_SEGMENT_MS:
segment = np.concatenate(voiced_frames)
triggered = False
voiced_frames = []
silence_ms = 0
speech_ms = 0
yield segment
def main():
parser = argparse.ArgumentParser(description="Live Whisper transcription -> Unix socket")
parser.add_argument("--socket", default="/tmp/whisper.sock",
help="Unix socket path to connect to (Java side should be listening here)")
parser.add_argument("--model", default="small",
help="Whisper model size: tiny/base/small/medium/large-v3")
parser.add_argument("--device", default=None,
help="Force device string (cuda/cpu). Default: auto-detect via torch.cuda.is_available() "
"(ROCm builds of torch report themselves as 'cuda').")
parser.add_argument("--vad-aggressiveness", type=int, default=2, choices=[0, 1, 2, 3],
help="0=least aggressive filtering (more false positives), 3=most aggressive")
parser.add_argument("--language", default=None, help="Force language code e.g. 'en'. Default: auto-detect.")
parser.add_argument("--input-device", type=int, default=None,
help="sounddevice input device index, see --list-devices")
parser.add_argument("--list-devices", action="store_true")
args = parser.parse_args()
if args.list_devices:
print(sd.query_devices())
return
device = args.device or ("cuda" if torch.cuda.is_available() else "cpu")
print(f"[whisper] loading model '{args.model}' on device '{device}'", file=sys.stderr)
model = whisper.load_model(args.model, device=device)
client = SocketClient(args.socket)
audio_queue: "queue.Queue" = queue.Queue()
def audio_callback(indata, frames, time_info, status):
if status:
print(f"[audio] {status}", file=sys.stderr)
audio_queue.put(indata[:, 0].copy())
stream = sd.InputStream(
samplerate=SAMPLE_RATE,
channels=1,
dtype="int16",
blocksize=FRAME_SAMPLES,
device=args.input_device,
callback=audio_callback,
)
print("[whisper] listening... (Ctrl+C to stop)", file=sys.stderr)
with stream:
try:
for segment in vad_segmenter(audio_queue, args.vad_aggressiveness):
audio_f32 = segment.astype(np.float32) / 32768.0
result = model.transcribe(
audio_f32,
language=args.language,
fp16=(device != "cpu"),
condition_on_previous_text=False,
)
text = result.get("text", "").strip()
if text:
payload = {
"type": "transcript",
"text": text,
"timestamp": time.time(),
"language": result.get("language"),
}
print(f"[transcript] {text}")
client.send(payload)
except KeyboardInterrupt:
print("\n[whisper] stopping", file=sys.stderr)
audio_queue.put(None)
if __name__ == "__main__":
main()