commit 35ea93ff7dc7f9d43e499f1cb94c23c146b3c5ca Author: Zacharias Date: Thu Jul 16 15:05:06 2026 +0200 Initial Commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ed0aa11 --- /dev/null +++ b/.gitignore @@ -0,0 +1,46 @@ +.gradle +build/ +!gradle/wrapper/gradle-wrapper.jar +!**/src/main/**/build/ +!**/src/test/**/build/ +.kotlin + +### IntelliJ IDEA ### +.idea/modules.xml +.idea/jarRepositories.xml +.idea/compiler.xml +.idea/libraries/ +*.iws +*.iml +*.ipr +out/ +!**/src/main/**/out/ +!**/src/test/**/out/ + +### Eclipse ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache +bin/ +!**/src/main/**/bin/ +!**/src/test/**/bin/ + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ + +### VS Code ### +.vscode/ + +### Mac OS ### +.DS_Store + +### python ### +.venv/ \ No newline at end of file diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..30cf57e --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,10 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ +# Ignored default folder with query files +/queries/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml diff --git a/.idea/gradle.xml b/.idea/gradle.xml new file mode 100644 index 0000000..0ca881b --- /dev/null +++ b/.idea/gradle.xml @@ -0,0 +1,17 @@ + + + + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..95c900b --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..f70ed9a --- /dev/null +++ b/README.md @@ -0,0 +1,2 @@ +# GLaDOS +This is a small example of how to use piper-tts with NeuroDock to get GLaDOS to say what ever the AI responds with \ No newline at end of file diff --git a/build.gradle b/build.gradle new file mode 100644 index 0000000..3d23400 --- /dev/null +++ b/build.gradle @@ -0,0 +1,52 @@ +plugins { + id 'java' + id 'com.gradleup.shadow' version '9.0.0-beta7' +} + +group = 'me.zacharias' +version = '1.0-SNAPSHOT' + +repositories { + mavenCentral() + mavenLocal() +} + +dependencies { + + implementation('me.neurodock:Core:+'); + implementation('io.github.jvoice-project:piper-jni:+') + implementation("org.jetbrains:annotations:23.1.0") + implementation("io.github.givimad:whisper-jni:+") + implementation 'com.fasterxml.jackson.core:jackson-databind:2.17.2' + + testImplementation platform('org.junit:junit-bom:6.0.0') + testImplementation 'org.junit.jupiter:junit-jupiter' + testRuntimeOnly 'org.junit.platform:junit-platform-launcher' +} + +sourceSets { + main { + resources { + srcDirs "src/main/resources" + } + } +} + +tasks.register('copyPythonRuntime', Copy) { + from 'src/main/python' + into "${layout.buildDirectory.get()}/resources/main/python" +} + +tasks.named('processResources') { + dependsOn 'copyPythonRuntime' +} + +jar { + manifest { + attributes 'Main-Class': 'me.neurodock.glados.Main' + } +} + +test { + useJUnitPlatform() +} \ No newline at end of file diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000..249e583 Binary files /dev/null and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 0000000..b61c6b4 --- /dev/null +++ b/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,6 @@ +#Wed Jul 15 15:29:26 CEST 2026 +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-9.4.0-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew new file mode 100755 index 0000000..1b6c787 --- /dev/null +++ b/gradlew @@ -0,0 +1,234 @@ +#!/bin/sh + +# +# Copyright © 2015-2021 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# +############################################################################## + +# Attempt to set APP_HOME + +# Resolve links: $0 may be a link +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit + +APP_NAME="Gradle" +APP_BASE_NAME=${0##*/} + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn () { + echo "$*" +} >&2 + +die () { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + +# Collect all arguments for the java command; +# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of +# shell script including quotes and variable substitutions, so put them in +# double quotes to make sure that they get re-expanded; and +# * put everything else in single quotes, so that it's not re-expanded. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + +exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat new file mode 100644 index 0000000..107acd3 --- /dev/null +++ b/gradlew.bat @@ -0,0 +1,89 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto execute + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..dec1b49 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,50 @@ +certifi==2026.6.17 +cffi==2.1.0 +charset-normalizer==3.4.9 +cuda-bindings==13.3.1 +cuda-pathfinder==1.5.6 +cuda-toolkit==13.0.3.0 +filelock==3.29.0 +fsspec==2026.4.0 +idna==3.18 +Jinja2==3.1.6 +llvmlite==0.48.0 +MarkupSafe==3.0.3 +more-itertools==11.1.0 +mpmath==1.3.0 +networkx==3.6.1 +numba==0.66.0 +numpy==2.4.4 +nvidia-cublas==13.1.1.3 +nvidia-cuda-cupti==13.0.85 +nvidia-cuda-nvrtc==13.0.88 +nvidia-cuda-runtime==13.0.96 +nvidia-cudnn-cu13==9.20.0.48 +nvidia-cufft==12.0.0.61 +nvidia-cufile==1.15.1.6 +nvidia-curand==10.4.0.35 +nvidia-cusolver==12.0.4.66 +nvidia-cusparse==12.6.3.3 +nvidia-cusparselt-cu13==0.8.1 +nvidia-nccl-cu13==2.29.7 +nvidia-nvjitlink==13.3.33 +nvidia-nvshmem-cu13==3.4.5 +nvidia-nvtx==13.0.85 +openai-whisper==20250625 +pillow==12.2.0 +pycparser==3.0 +regex==2026.7.10 +requests==2.34.2 +setuptools==78.1.0 +sounddevice==0.5.5 +sympy==1.14.0 +tiktoken==0.13.0 +torch==2.13.0+rocm7.2 +torchaudio==2.11.0+rocm7.2 +torchvision==0.28.0+rocm7.2 +tqdm==4.68.4 +triton==3.7.1 +triton-rocm==3.7.1 +typing_extensions==4.15.0 +urllib3==2.7.0 +webrtcvad==2.0.10 diff --git a/settings.gradle b/settings.gradle new file mode 100644 index 0000000..028af78 --- /dev/null +++ b/settings.gradle @@ -0,0 +1 @@ +rootProject.name = 'GLaDOS' \ No newline at end of file diff --git a/src/main/java/me/neurodock/glados/AnnouncerVoiceCalling.java b/src/main/java/me/neurodock/glados/AnnouncerVoiceCalling.java new file mode 100644 index 0000000..df10e4a --- /dev/null +++ b/src/main/java/me/neurodock/glados/AnnouncerVoiceCalling.java @@ -0,0 +1,57 @@ +package me.neurodock.glados; + +import me.neurodock.core.PrintAdvanceMessageHandler; +import me.neurodock.core.ToolCallingRender; +import me.neurodock.ollama.OllamaFunctionArgument; +import me.neurodock.ollama.OllamaFunctionTool; +import me.neurodock.ollama.OllamaPerameter; +import me.neurodock.ollama.OllamaToolResponse; +import me.neurodock.ollama.exceptions.OllamaToolErrorException; +import org.json.JSONObject; + +import java.util.Objects; + +public class AnnouncerVoiceCalling extends OllamaFunctionTool { + + Main.PlaybackHandler pb; + public AnnouncerVoiceCalling(Main.PlaybackHandler printAdvanceMessageHandler) { + this.pb = printAdvanceMessageHandler; + } + + @org.jetbrains.annotations.NotNull + @Override + public String name() { + return "pa_announcer"; + } + + @Override + public @org.jetbrains.annotations.NotNull OllamaPerameter parameters() { + return OllamaPerameter.builder() + .addProperty("message", OllamaPerameter.OllamaPerameterBuilder.Type.STRING, "Message to be announced", true) + .build(); + } + + @Override + public ToolCallingRender renderCalling(JSONObject calling) { + return new ToolCallingRender.Suppress(); + } + + @Override + public @org.jetbrains.annotations.NotNull OllamaToolResponse function(OllamaFunctionArgument... ollamaFunctionArguments) { + if (ollamaFunctionArguments.length != 1) { + throw new OllamaToolErrorException(name(), "Number of arguments to be announced must be 1"); + } + + if(!(Objects.equals(ollamaFunctionArguments[0].argument(), "message"))) + { + throw new OllamaToolErrorException(name(), "Message to be announced must be message"); + } + + if(ollamaFunctionArguments[0].value() instanceof String str) { + pb.playAnnouncer(str); + return new OllamaToolResponse(name(), "success"); + } + + throw new OllamaToolErrorException(name(), "Message to be announced must be message"); + } +} diff --git a/src/main/java/me/neurodock/glados/Main.java b/src/main/java/me/neurodock/glados/Main.java new file mode 100644 index 0000000..25d48e9 --- /dev/null +++ b/src/main/java/me/neurodock/glados/Main.java @@ -0,0 +1,212 @@ +package me.neurodock.glados; + +import io.github.jvoiceproject.piperjni.PiperJNI; +import io.github.jvoiceproject.piperjni.PiperVoice; +import me.neurodock.core.Core; +import me.neurodock.core.LaunchOptions; +import me.neurodock.core.PrintAdvanceMessageHandler; +import me.neurodock.ollama.OllamaMessage; +import me.neurodock.ollama.OllamaMessageRole; +import me.neurodock.ollama.OllamaObject; +import org.json.JSONObject; + +import javax.sound.sampled.AudioFormat; +import javax.sound.sampled.AudioSystem; +import javax.sound.sampled.LineUnavailableException; +import javax.sound.sampled.TargetDataLine; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; + +public class Main { + + static void main(String[] args) throws LineUnavailableException { + new Main(); + + } + + public Main() throws LineUnavailableException { + PlaybackHandler pb = new PlaybackHandler(); + + Path socketPath = Paths.get("/tmp/whisper.sock"); + + //pb.playGLaDOS("I'm Glaudos"); + + Core core = new Core(pb, "127.0.0.1"); + + LaunchOptions.getInstance().setLoadOld(false); + + core.setOllamaObjectNoMemory(OllamaObject.builder() + .setModel("llama3.2") + //.addMessage(new OllamaMessage(OllamaMessageRole.SYSTEM, "You are GLaDOS.")) + .keep_alive(10) + .build()); + + //core.addTool(new AnnouncerVoiceCalling(pb), Core.Source.CTP); + + try(WhisperSocketServer server = new WhisperSocketServer(socketPath, event -> + { + core.getOllamaObject().addMessage(new OllamaMessage(OllamaMessageRole.USER, event.text())); + core.qurryOllama().thenApply((json) -> + { + String message = json.getJSONObject("message").getString("content"); + if(json.getJSONObject("message").has("content") && !message.isBlank()) { + message = message.replaceAll("(?i)GLaDOS", "Glaudos").replaceAll("\"", " quote "); + } + + json.getJSONObject("message").put("content", message); + return json; + }).thenAccept(core::handleResponse).join(); + })) + { + server.start(); + Thread.currentThread().join(); + }catch (IOException _) { + + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + + + public class PlaybackHandler implements PrintAdvanceMessageHandler + { + + public static final String GLaDOS_PIPER_MODEL = ".local/share/piper/glados/glados_piper_medium.onnx"; + public static final String PORTAL_ANNOUNCER_PIPER_MODEL = ".local/share/piper/portal-announcer/announcer.onnx"; + public static final int SENTENCE_PAUSE_DURATION_MS = 500; + + @Override + public void printMessage(OllamaMessage ollamaMessage) { + System.out.println(ollamaMessage.toString()); + switch(ollamaMessage.getRole()) + { + case ASSISTANT -> playGLaDOS(ollamaMessage.getContent()); + case TOOL -> playAnnouncer("Tool: " + new JSONObject(ollamaMessage.getContent()).get("result")); + case USER -> System.out.println("User> "+ollamaMessage.getContent()); + case SYSTEM -> playAnnouncer(ollamaMessage.getContent()); + } + } + + @Override + public void printErrorMessage(OllamaMessage ollamaMessage) { + playAnnouncer("Error: "+ollamaMessage.getContent()); + } + + @Override + public void printToolCalling(String s) { + playAnnouncer(s); + } + + public void playGLaDOS(String msg) { + try (PiperJNI piper = new PiperJNI()) { + piper.initialize(true); + try (PiperVoice voice = piper.loadVoice( + Paths.get(System.getenv("HOME"), GLaDOS_PIPER_MODEL), + Paths.get(System.getenv("HOME"),GLaDOS_PIPER_MODEL+".json"), + 0)) { + int sampleRate = voice.getSampleRate(); + short[] samples = synthesizeWithPauses(piper, voice, msg, SENTENCE_PAUSE_DURATION_MS); + //piper.textToAudio(voice, msg); + playAudio(samples, sampleRate); + } finally { + piper.terminate(); + } + }catch (Exception e) { + System.out.println("Critial Error on piper-tts!"); + System.out.println(e.getMessage()); + e.printStackTrace(System.out); + System.exit(-1); + } + } + + public void playAnnouncer(String msg) { + try (PiperJNI piper = new PiperJNI()) { + piper.initialize(true); + try (PiperVoice voice = piper.loadVoice( + Paths.get(System.getenv("HOME"), PORTAL_ANNOUNCER_PIPER_MODEL), + Paths.get(System.getenv("HOME"),PORTAL_ANNOUNCER_PIPER_MODEL+".json"), + 0)) { + int sampleRate = voice.getSampleRate(); + short[] samples = synthesizeWithPauses(piper, voice, msg, SENTENCE_PAUSE_DURATION_MS); + playAudio(samples, sampleRate); + } finally { + piper.terminate(); + } + }catch (Exception e) { + System.out.println("Critial Error on piper-tts!"); + System.out.println(e.getMessage()); + e.printStackTrace(System.out); + System.exit(-1); + } + } + + public short[] synthesizeWithPauses(PiperJNI piper, PiperVoice voice, String text, int pauseMs) throws PiperJNI.NotInitialized, IOException { + // Split, but keep the delimiter so we know which pause length to use + String[] chunks = text.split("(?<=\\.\\.\\.)|(?<=[.!?])\\s+|(?<=,)\\s+"); + List pieces = new ArrayList<>(); + + for (String chunk : chunks) { + if (chunk.isBlank()) continue; + String trimmed = chunk.trim(); + pieces.add(piper.textToAudio(voice, trimmed)); + + int thisPauseMs; + if (trimmed.endsWith("...")) { + thisPauseMs = pauseMs; + } else if (trimmed.endsWith(",")) { + thisPauseMs = pauseMs / 2; + } else if (trimmed.matches(".*[.!?]$")) { + thisPauseMs = pauseMs; + } else { + thisPauseMs = 0; // no trailing punctuation, no forced pause + } + + if (thisPauseMs > 0) { + int pauseSamples = (voice.getSampleRate() * thisPauseMs) / 1000; + pieces.add(new short[pauseSamples]); + } + } + + int total = pieces.stream().mapToInt(p -> p.length).sum(); + short[] combined = new short[total]; + int offset = 0; + for (short[] p : pieces) { + System.arraycopy(p, 0, combined, offset, p.length); + offset += p.length; + } + return combined; + } + + public byte[] shortsToBytes(short[] samples) { + byte[] bytes = new byte[samples.length * 2]; + for (int i = 0; i < samples.length; i++) { + bytes[i * 2] = (byte) (samples[i] & 0xFF); // low byte + bytes[i * 2 + 1] = (byte) ((samples[i] >> 8) & 0xFF); // high byte + } + return bytes; + } + + public void playAudio(short[] samples, int sampleRate) throws IOException, InterruptedException { + ProcessBuilder pb = new ProcessBuilder( + "pw-play", + "--rate=" + sampleRate, + "--channels=1", + "--format=s16", + "--volume=0.3", + "--raw", + "-" + ); + pb.redirectErrorStream(true); + Process proc = pb.start(); + + try (OutputStream os = proc.getOutputStream()) { + os.write(shortsToBytes(samples)); + } + proc.waitFor(); + } + } +} diff --git a/src/main/java/me/neurodock/glados/WhisperSocketServer.java b/src/main/java/me/neurodock/glados/WhisperSocketServer.java new file mode 100644 index 0000000..64312a2 --- /dev/null +++ b/src/main/java/me/neurodock/glados/WhisperSocketServer.java @@ -0,0 +1,100 @@ +package me.neurodock.glados; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.io.BufferedReader; +import java.io.IOException; +import java.net.StandardProtocolFamily; +import java.net.UnixDomainSocketAddress; +import java.nio.channels.Channels; +import java.nio.channels.ServerSocketChannel; +import java.nio.channels.SocketChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.function.Consumer; + +public class WhisperSocketServer implements AutoCloseable { + @JsonIgnoreProperties(ignoreUnknown = true) + public record TranscriptionEvent(String type, String text, double timestampt, String language) {}; + + private final Path socketPath; + + private final Consumer onTranscription; + + private final ObjectMapper mapper = new ObjectMapper(); + + private final ExecutorService executor = Executors.newCachedThreadPool(); + + private ServerSocketChannel serverChannel; + + private volatile boolean running = true; + + public WhisperSocketServer(Path socketPath, Consumer onTranscription) { + this.socketPath = socketPath; + this.onTranscription = onTranscription; + } + + public void start() throws IOException { + Files.deleteIfExists(socketPath); + UnixDomainSocketAddress address = UnixDomainSocketAddress.of(socketPath); + serverChannel = ServerSocketChannel.open(StandardProtocolFamily.UNIX); + serverChannel.bind(address); + + System.out.println("[whisper-socket] listening on "+socketPath); + executor.submit(this::acceptLoop); + } + + private void acceptLoop() { + while (running) { + try { + SocketChannel client = serverChannel.accept(); + System.out.println("[whisper-socket] python client connected"); + executor.submit(() -> handleClient(client)); + }catch(IOException e) + { + if(running) + { + System.err.println("[whisper-socket] accept failed: " + e.getMessage()); + } + } + } + } + + private void handleClient(SocketChannel client) { + try(BufferedReader reader = new BufferedReader(Channels.newReader(client, StandardCharsets.UTF_8))){ + String line; + while((line = reader.readLine()) != null) + { + if(line.isBlank()) continue; + try { + TranscriptionEvent event = mapper.readValue(line, TranscriptionEvent.class); + onTranscription.accept(event); + } catch (Exception parseEx) { + System.err.println("[whisper-socket] bad line: " + line + " (" + parseEx.getMessage() + ")"); + } + } + } catch (IOException e) { + System.err.println("[whisper-socket] client read error: " + e.getMessage()); + } finally { + System.out.println("[whisper-socket] python client disconnected"); + try { + client.close(); + } catch (IOException ignored) { + } + } + } + + @Override + public void close() throws IOException { + running = false; + executor.shutdownNow(); + if (serverChannel != null) { + serverChannel.close(); + } + Files.deleteIfExists(socketPath); + } +} diff --git a/src/main/python/whisper_daemon.py b/src/main/python/whisper_daemon.py new file mode 100644 index 0000000..07e4939 --- /dev/null +++ b/src/main/python/whisper_daemon.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +""" +whisper_daemon.py - Live microphone transcription via OpenAI Whisper (PyTorch/ROCm backend) + +Captures mic audio, segments it into utterances using WebRTC VAD (rather than +blindly transcribing fixed windows), runs each finished segment through Whisper, +and streams the resulting text as newline-delimited JSON over a Unix domain socket. + +Architecture assumption: this process is a CLIENT. It connects OUT to a Unix +socket that the Java program is listening on (and will retry/reconnect if the +Java side isn't up yet). Flip SocketClient -> a socketserver if you'd rather +have Python own the socket and Java connect in. +""" + +import argparse +import json +import queue +import socket +import sys +import threading +import time + +import numpy as np +import sounddevice as sd +import torch +import webrtcvad +import whisper + +SAMPLE_RATE = 16000 +FRAME_MS = 30 # webrtcvad only accepts 10/20/30ms frames +FRAME_SAMPLES = int(SAMPLE_RATE * FRAME_MS / 1000) +SILENCE_TIMEOUT_MS = 700 # trailing silence before an utterance is finalized +MAX_SEGMENT_MS = 15000 # hard cap so long uninterrupted speech still flushes + + +class SocketClient: + """Unix socket client with blocking connect-retry and reconnect-on-send-failure.""" + + def __init__(self, sock_path: str, retry_interval: float = 2.0): + self.sock_path = sock_path + self.retry_interval = retry_interval + self.sock = None + self.lock = threading.Lock() + self._connect() + + def _connect(self): + while self.sock is None: + try: + s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + s.connect(self.sock_path) + self.sock = s + print(f"[socket] connected to {self.sock_path}", file=sys.stderr) + except (FileNotFoundError, ConnectionRefusedError) as e: + print(f"[socket] waiting for {self.sock_path} ({e})", file=sys.stderr) + time.sleep(self.retry_interval) + + def send(self, obj: dict): + data = (json.dumps(obj) + "\n").encode("utf-8") + with self.lock: + try: + self.sock.sendall(data) + except (BrokenPipeError, OSError) as e: + print(f"[socket] send failed ({e}), reconnecting", file=sys.stderr) + try: + self.sock.close() + except OSError: + pass + self.sock = None + self._connect() + self.sock.sendall(data) + + +def frame_generator(audio_queue: "queue.Queue"): + """Re-chunk arbitrary-sized audio callback buffers into fixed VAD-sized frames.""" + buf = np.zeros((0,), dtype=np.int16) + while True: + chunk = audio_queue.get() + if chunk is None: + return + buf = np.concatenate([buf, chunk]) + while len(buf) >= FRAME_SAMPLES: + frame = buf[:FRAME_SAMPLES] + buf = buf[FRAME_SAMPLES:] + yield frame + + +def vad_segmenter(audio_queue: "queue.Queue", vad_aggressiveness: int = 2): + """ + Consume raw frames, use WebRTC VAD to detect speech, and yield complete + utterances (numpy int16 arrays) once trailing silence or max length hits. + """ + vad = webrtcvad.Vad(vad_aggressiveness) + voiced_frames = [] + silence_ms = 0 + speech_ms = 0 + triggered = False + + for frame in frame_generator(audio_queue): + is_speech = vad.is_speech(frame.tobytes(), SAMPLE_RATE) + + if not triggered: + if is_speech: + triggered = True + voiced_frames = [frame] + speech_ms = FRAME_MS + silence_ms = 0 + else: + voiced_frames.append(frame) + speech_ms += FRAME_MS + if is_speech: + silence_ms = 0 + else: + silence_ms += FRAME_MS + + if silence_ms >= SILENCE_TIMEOUT_MS or speech_ms >= MAX_SEGMENT_MS: + segment = np.concatenate(voiced_frames) + triggered = False + voiced_frames = [] + silence_ms = 0 + speech_ms = 0 + yield segment + + +def main(): + parser = argparse.ArgumentParser(description="Live Whisper transcription -> Unix socket") + parser.add_argument("--socket", default="/tmp/whisper.sock", + help="Unix socket path to connect to (Java side should be listening here)") + parser.add_argument("--model", default="small", + help="Whisper model size: tiny/base/small/medium/large-v3") + parser.add_argument("--device", default=None, + help="Force device string (cuda/cpu). Default: auto-detect via torch.cuda.is_available() " + "(ROCm builds of torch report themselves as 'cuda').") + parser.add_argument("--vad-aggressiveness", type=int, default=2, choices=[0, 1, 2, 3], + help="0=least aggressive filtering (more false positives), 3=most aggressive") + parser.add_argument("--language", default=None, help="Force language code e.g. 'en'. Default: auto-detect.") + parser.add_argument("--input-device", type=int, default=None, + help="sounddevice input device index, see --list-devices") + parser.add_argument("--list-devices", action="store_true") + args = parser.parse_args() + + if args.list_devices: + print(sd.query_devices()) + return + + device = args.device or ("cuda" if torch.cuda.is_available() else "cpu") + print(f"[whisper] loading model '{args.model}' on device '{device}'", file=sys.stderr) + model = whisper.load_model(args.model, device=device) + + client = SocketClient(args.socket) + audio_queue: "queue.Queue" = queue.Queue() + + def audio_callback(indata, frames, time_info, status): + if status: + print(f"[audio] {status}", file=sys.stderr) + audio_queue.put(indata[:, 0].copy()) + + stream = sd.InputStream( + samplerate=SAMPLE_RATE, + channels=1, + dtype="int16", + blocksize=FRAME_SAMPLES, + device=args.input_device, + callback=audio_callback, + ) + + print("[whisper] listening... (Ctrl+C to stop)", file=sys.stderr) + with stream: + try: + for segment in vad_segmenter(audio_queue, args.vad_aggressiveness): + audio_f32 = segment.astype(np.float32) / 32768.0 + result = model.transcribe( + audio_f32, + language=args.language, + fp16=(device != "cpu"), + condition_on_previous_text=False, + ) + text = result.get("text", "").strip() + if text: + payload = { + "type": "transcript", + "text": text, + "timestamp": time.time(), + "language": result.get("language"), + } + print(f"[transcript] {text}") + client.send(payload) + except KeyboardInterrupt: + print("\n[whisper] stopping", file=sys.stderr) + audio_queue.put(None) + + +if __name__ == "__main__": + main() \ No newline at end of file