Initial Commit
This commit is contained in:
+46
@@ -0,0 +1,46 @@
|
||||
.gradle
|
||||
build/
|
||||
!gradle/wrapper/gradle-wrapper.jar
|
||||
!**/src/main/**/build/
|
||||
!**/src/test/**/build/
|
||||
.kotlin
|
||||
|
||||
### IntelliJ IDEA ###
|
||||
.idea/modules.xml
|
||||
.idea/jarRepositories.xml
|
||||
.idea/compiler.xml
|
||||
.idea/libraries/
|
||||
*.iws
|
||||
*.iml
|
||||
*.ipr
|
||||
out/
|
||||
!**/src/main/**/out/
|
||||
!**/src/test/**/out/
|
||||
|
||||
### Eclipse ###
|
||||
.apt_generated
|
||||
.classpath
|
||||
.factorypath
|
||||
.project
|
||||
.settings
|
||||
.springBeans
|
||||
.sts4-cache
|
||||
bin/
|
||||
!**/src/main/**/bin/
|
||||
!**/src/test/**/bin/
|
||||
|
||||
### NetBeans ###
|
||||
/nbproject/private/
|
||||
/nbbuild/
|
||||
/dist/
|
||||
/nbdist/
|
||||
/.nb-gradle/
|
||||
|
||||
### VS Code ###
|
||||
.vscode/
|
||||
|
||||
### Mac OS ###
|
||||
.DS_Store
|
||||
|
||||
### python ###
|
||||
.venv/
|
||||
Generated
+10
@@ -0,0 +1,10 @@
|
||||
# Default ignored files
|
||||
/shelf/
|
||||
/workspace.xml
|
||||
# Editor-based HTTP Client requests
|
||||
/httpRequests/
|
||||
# Ignored default folder with query files
|
||||
/queries/
|
||||
# Datasource local storage ignored files
|
||||
/dataSources/
|
||||
/dataSources.local.xml
|
||||
Generated
+17
@@ -0,0 +1,17 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="GradleMigrationSettings" migrationVersion="1" />
|
||||
<component name="GradleSettings">
|
||||
<option name="linkedExternalProjectsSettings">
|
||||
<GradleProjectSettings>
|
||||
<option name="externalProjectPath" value="$PROJECT_DIR$" />
|
||||
<option name="modules">
|
||||
<set>
|
||||
<option value="$PROJECT_DIR$" />
|
||||
</set>
|
||||
</option>
|
||||
<option name="myGradleHome" value="/usr/share/java/gradle" />
|
||||
</GradleProjectSettings>
|
||||
</option>
|
||||
</component>
|
||||
</project>
|
||||
Generated
+13
@@ -0,0 +1,13 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="Black">
|
||||
<option name="sdkName" value="Python 3.14 (GLaDOS)" />
|
||||
</component>
|
||||
<component name="ExternalStorageConfigurationManager" enabled="true" />
|
||||
<component name="FrameworkDetectionExcludesConfiguration">
|
||||
<file type="web" url="file://$PROJECT_DIR$" />
|
||||
</component>
|
||||
<component name="ProjectRootManager" version="2" languageLevel="JDK_26" default="true" project-jdk-name="26" project-jdk-type="JavaSDK">
|
||||
<output url="file://$PROJECT_DIR$/out" />
|
||||
</component>
|
||||
</project>
|
||||
Generated
+6
@@ -0,0 +1,6 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="VcsDirectoryMappings">
|
||||
<mapping directory="$PROJECT_DIR$" vcs="Git" />
|
||||
</component>
|
||||
</project>
|
||||
@@ -0,0 +1,2 @@
|
||||
# GLaDOS
|
||||
This is a small example of how to use piper-tts with NeuroDock to get GLaDOS to say what ever the AI responds with
|
||||
@@ -0,0 +1,52 @@
|
||||
plugins {
|
||||
id 'java'
|
||||
id 'com.gradleup.shadow' version '9.0.0-beta7'
|
||||
}
|
||||
|
||||
group = 'me.zacharias'
|
||||
version = '1.0-SNAPSHOT'
|
||||
|
||||
repositories {
|
||||
mavenCentral()
|
||||
mavenLocal()
|
||||
}
|
||||
|
||||
dependencies {
|
||||
|
||||
implementation('me.neurodock:Core:+');
|
||||
implementation('io.github.jvoice-project:piper-jni:+')
|
||||
implementation("org.jetbrains:annotations:23.1.0")
|
||||
implementation("io.github.givimad:whisper-jni:+")
|
||||
implementation 'com.fasterxml.jackson.core:jackson-databind:2.17.2'
|
||||
|
||||
testImplementation platform('org.junit:junit-bom:6.0.0')
|
||||
testImplementation 'org.junit.jupiter:junit-jupiter'
|
||||
testRuntimeOnly 'org.junit.platform:junit-platform-launcher'
|
||||
}
|
||||
|
||||
sourceSets {
|
||||
main {
|
||||
resources {
|
||||
srcDirs "src/main/resources"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tasks.register('copyPythonRuntime', Copy) {
|
||||
from 'src/main/python'
|
||||
into "${layout.buildDirectory.get()}/resources/main/python"
|
||||
}
|
||||
|
||||
tasks.named('processResources') {
|
||||
dependsOn 'copyPythonRuntime'
|
||||
}
|
||||
|
||||
jar {
|
||||
manifest {
|
||||
attributes 'Main-Class': 'me.neurodock.glados.Main'
|
||||
}
|
||||
}
|
||||
|
||||
test {
|
||||
useJUnitPlatform()
|
||||
}
|
||||
Vendored
BIN
Binary file not shown.
+6
@@ -0,0 +1,6 @@
|
||||
#Wed Jul 15 15:29:26 CEST 2026
|
||||
distributionBase=GRADLE_USER_HOME
|
||||
distributionPath=wrapper/dists
|
||||
distributionUrl=https\://services.gradle.org/distributions/gradle-9.4.0-bin.zip
|
||||
zipStoreBase=GRADLE_USER_HOME
|
||||
zipStorePath=wrapper/dists
|
||||
@@ -0,0 +1,234 @@
|
||||
#!/bin/sh
|
||||
|
||||
#
|
||||
# Copyright © 2015-2021 the original authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# https://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
##############################################################################
|
||||
#
|
||||
# Gradle start up script for POSIX generated by Gradle.
|
||||
#
|
||||
# Important for running:
|
||||
#
|
||||
# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is
|
||||
# noncompliant, but you have some other compliant shell such as ksh or
|
||||
# bash, then to run this script, type that shell name before the whole
|
||||
# command line, like:
|
||||
#
|
||||
# ksh Gradle
|
||||
#
|
||||
# Busybox and similar reduced shells will NOT work, because this script
|
||||
# requires all of these POSIX shell features:
|
||||
# * functions;
|
||||
# * expansions «$var», «${var}», «${var:-default}», «${var+SET}»,
|
||||
# «${var#prefix}», «${var%suffix}», and «$( cmd )»;
|
||||
# * compound commands having a testable exit status, especially «case»;
|
||||
# * various built-in commands including «command», «set», and «ulimit».
|
||||
#
|
||||
# Important for patching:
|
||||
#
|
||||
# (2) This script targets any POSIX shell, so it avoids extensions provided
|
||||
# by Bash, Ksh, etc; in particular arrays are avoided.
|
||||
#
|
||||
# The "traditional" practice of packing multiple parameters into a
|
||||
# space-separated string is a well documented source of bugs and security
|
||||
# problems, so this is (mostly) avoided, by progressively accumulating
|
||||
# options in "$@", and eventually passing that to Java.
|
||||
#
|
||||
# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS,
|
||||
# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly;
|
||||
# see the in-line comments for details.
|
||||
#
|
||||
# There are tweaks for specific operating systems such as AIX, CygWin,
|
||||
# Darwin, MinGW, and NonStop.
|
||||
#
|
||||
# (3) This script is generated from the Groovy template
|
||||
# https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
|
||||
# within the Gradle project.
|
||||
#
|
||||
# You can find Gradle at https://github.com/gradle/gradle/.
|
||||
#
|
||||
##############################################################################
|
||||
|
||||
# Attempt to set APP_HOME
|
||||
|
||||
# Resolve links: $0 may be a link
|
||||
app_path=$0
|
||||
|
||||
# Need this for daisy-chained symlinks.
|
||||
while
|
||||
APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path
|
||||
[ -h "$app_path" ]
|
||||
do
|
||||
ls=$( ls -ld "$app_path" )
|
||||
link=${ls#*' -> '}
|
||||
case $link in #(
|
||||
/*) app_path=$link ;; #(
|
||||
*) app_path=$APP_HOME$link ;;
|
||||
esac
|
||||
done
|
||||
|
||||
APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit
|
||||
|
||||
APP_NAME="Gradle"
|
||||
APP_BASE_NAME=${0##*/}
|
||||
|
||||
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
||||
DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
|
||||
|
||||
# Use the maximum available, or set MAX_FD != -1 to use that value.
|
||||
MAX_FD=maximum
|
||||
|
||||
warn () {
|
||||
echo "$*"
|
||||
} >&2
|
||||
|
||||
die () {
|
||||
echo
|
||||
echo "$*"
|
||||
echo
|
||||
exit 1
|
||||
} >&2
|
||||
|
||||
# OS specific support (must be 'true' or 'false').
|
||||
cygwin=false
|
||||
msys=false
|
||||
darwin=false
|
||||
nonstop=false
|
||||
case "$( uname )" in #(
|
||||
CYGWIN* ) cygwin=true ;; #(
|
||||
Darwin* ) darwin=true ;; #(
|
||||
MSYS* | MINGW* ) msys=true ;; #(
|
||||
NONSTOP* ) nonstop=true ;;
|
||||
esac
|
||||
|
||||
CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
|
||||
|
||||
|
||||
# Determine the Java command to use to start the JVM.
|
||||
if [ -n "$JAVA_HOME" ] ; then
|
||||
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
|
||||
# IBM's JDK on AIX uses strange locations for the executables
|
||||
JAVACMD=$JAVA_HOME/jre/sh/java
|
||||
else
|
||||
JAVACMD=$JAVA_HOME/bin/java
|
||||
fi
|
||||
if [ ! -x "$JAVACMD" ] ; then
|
||||
die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
|
||||
|
||||
Please set the JAVA_HOME variable in your environment to match the
|
||||
location of your Java installation."
|
||||
fi
|
||||
else
|
||||
JAVACMD=java
|
||||
which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
|
||||
|
||||
Please set the JAVA_HOME variable in your environment to match the
|
||||
location of your Java installation."
|
||||
fi
|
||||
|
||||
# Increase the maximum file descriptors if we can.
|
||||
if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
|
||||
case $MAX_FD in #(
|
||||
max*)
|
||||
MAX_FD=$( ulimit -H -n ) ||
|
||||
warn "Could not query maximum file descriptor limit"
|
||||
esac
|
||||
case $MAX_FD in #(
|
||||
'' | soft) :;; #(
|
||||
*)
|
||||
ulimit -n "$MAX_FD" ||
|
||||
warn "Could not set maximum file descriptor limit to $MAX_FD"
|
||||
esac
|
||||
fi
|
||||
|
||||
# Collect all arguments for the java command, stacking in reverse order:
|
||||
# * args from the command line
|
||||
# * the main class name
|
||||
# * -classpath
|
||||
# * -D...appname settings
|
||||
# * --module-path (only if needed)
|
||||
# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables.
|
||||
|
||||
# For Cygwin or MSYS, switch paths to Windows format before running java
|
||||
if "$cygwin" || "$msys" ; then
|
||||
APP_HOME=$( cygpath --path --mixed "$APP_HOME" )
|
||||
CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" )
|
||||
|
||||
JAVACMD=$( cygpath --unix "$JAVACMD" )
|
||||
|
||||
# Now convert the arguments - kludge to limit ourselves to /bin/sh
|
||||
for arg do
|
||||
if
|
||||
case $arg in #(
|
||||
-*) false ;; # don't mess with options #(
|
||||
/?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath
|
||||
[ -e "$t" ] ;; #(
|
||||
*) false ;;
|
||||
esac
|
||||
then
|
||||
arg=$( cygpath --path --ignore --mixed "$arg" )
|
||||
fi
|
||||
# Roll the args list around exactly as many times as the number of
|
||||
# args, so each arg winds up back in the position where it started, but
|
||||
# possibly modified.
|
||||
#
|
||||
# NB: a `for` loop captures its iteration list before it begins, so
|
||||
# changing the positional parameters here affects neither the number of
|
||||
# iterations, nor the values presented in `arg`.
|
||||
shift # remove old arg
|
||||
set -- "$@" "$arg" # push replacement arg
|
||||
done
|
||||
fi
|
||||
|
||||
# Collect all arguments for the java command;
|
||||
# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of
|
||||
# shell script including quotes and variable substitutions, so put them in
|
||||
# double quotes to make sure that they get re-expanded; and
|
||||
# * put everything else in single quotes, so that it's not re-expanded.
|
||||
|
||||
set -- \
|
||||
"-Dorg.gradle.appname=$APP_BASE_NAME" \
|
||||
-classpath "$CLASSPATH" \
|
||||
org.gradle.wrapper.GradleWrapperMain \
|
||||
"$@"
|
||||
|
||||
# Use "xargs" to parse quoted args.
|
||||
#
|
||||
# With -n1 it outputs one arg per line, with the quotes and backslashes removed.
|
||||
#
|
||||
# In Bash we could simply go:
|
||||
#
|
||||
# readarray ARGS < <( xargs -n1 <<<"$var" ) &&
|
||||
# set -- "${ARGS[@]}" "$@"
|
||||
#
|
||||
# but POSIX shell has neither arrays nor command substitution, so instead we
|
||||
# post-process each arg (as a line of input to sed) to backslash-escape any
|
||||
# character that might be a shell metacharacter, then use eval to reverse
|
||||
# that process (while maintaining the separation between arguments), and wrap
|
||||
# the whole thing up as a single "set" statement.
|
||||
#
|
||||
# This will of course break if any of these variables contains a newline or
|
||||
# an unmatched quote.
|
||||
#
|
||||
|
||||
eval "set -- $(
|
||||
printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" |
|
||||
xargs -n1 |
|
||||
sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' |
|
||||
tr '\n' ' '
|
||||
)" '"$@"'
|
||||
|
||||
exec "$JAVACMD" "$@"
|
||||
Vendored
+89
@@ -0,0 +1,89 @@
|
||||
@rem
|
||||
@rem Copyright 2015 the original author or authors.
|
||||
@rem
|
||||
@rem Licensed under the Apache License, Version 2.0 (the "License");
|
||||
@rem you may not use this file except in compliance with the License.
|
||||
@rem You may obtain a copy of the License at
|
||||
@rem
|
||||
@rem https://www.apache.org/licenses/LICENSE-2.0
|
||||
@rem
|
||||
@rem Unless required by applicable law or agreed to in writing, software
|
||||
@rem distributed under the License is distributed on an "AS IS" BASIS,
|
||||
@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
@rem See the License for the specific language governing permissions and
|
||||
@rem limitations under the License.
|
||||
@rem
|
||||
|
||||
@if "%DEBUG%" == "" @echo off
|
||||
@rem ##########################################################################
|
||||
@rem
|
||||
@rem Gradle startup script for Windows
|
||||
@rem
|
||||
@rem ##########################################################################
|
||||
|
||||
@rem Set local scope for the variables with windows NT shell
|
||||
if "%OS%"=="Windows_NT" setlocal
|
||||
|
||||
set DIRNAME=%~dp0
|
||||
if "%DIRNAME%" == "" set DIRNAME=.
|
||||
set APP_BASE_NAME=%~n0
|
||||
set APP_HOME=%DIRNAME%
|
||||
|
||||
@rem Resolve any "." and ".." in APP_HOME to make it shorter.
|
||||
for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi
|
||||
|
||||
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
||||
set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m"
|
||||
|
||||
@rem Find java.exe
|
||||
if defined JAVA_HOME goto findJavaFromJavaHome
|
||||
|
||||
set JAVA_EXE=java.exe
|
||||
%JAVA_EXE% -version >NUL 2>&1
|
||||
if "%ERRORLEVEL%" == "0" goto execute
|
||||
|
||||
echo.
|
||||
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
|
||||
echo.
|
||||
echo Please set the JAVA_HOME variable in your environment to match the
|
||||
echo location of your Java installation.
|
||||
|
||||
goto fail
|
||||
|
||||
:findJavaFromJavaHome
|
||||
set JAVA_HOME=%JAVA_HOME:"=%
|
||||
set JAVA_EXE=%JAVA_HOME%/bin/java.exe
|
||||
|
||||
if exist "%JAVA_EXE%" goto execute
|
||||
|
||||
echo.
|
||||
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
|
||||
echo.
|
||||
echo Please set the JAVA_HOME variable in your environment to match the
|
||||
echo location of your Java installation.
|
||||
|
||||
goto fail
|
||||
|
||||
:execute
|
||||
@rem Setup the command line
|
||||
|
||||
set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
|
||||
|
||||
|
||||
@rem Execute Gradle
|
||||
"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %*
|
||||
|
||||
:end
|
||||
@rem End local scope for the variables with windows NT shell
|
||||
if "%ERRORLEVEL%"=="0" goto mainEnd
|
||||
|
||||
:fail
|
||||
rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
|
||||
rem the _cmd.exe /c_ return code!
|
||||
if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
|
||||
exit /b 1
|
||||
|
||||
:mainEnd
|
||||
if "%OS%"=="Windows_NT" endlocal
|
||||
|
||||
:omega
|
||||
@@ -0,0 +1,50 @@
|
||||
certifi==2026.6.17
|
||||
cffi==2.1.0
|
||||
charset-normalizer==3.4.9
|
||||
cuda-bindings==13.3.1
|
||||
cuda-pathfinder==1.5.6
|
||||
cuda-toolkit==13.0.3.0
|
||||
filelock==3.29.0
|
||||
fsspec==2026.4.0
|
||||
idna==3.18
|
||||
Jinja2==3.1.6
|
||||
llvmlite==0.48.0
|
||||
MarkupSafe==3.0.3
|
||||
more-itertools==11.1.0
|
||||
mpmath==1.3.0
|
||||
networkx==3.6.1
|
||||
numba==0.66.0
|
||||
numpy==2.4.4
|
||||
nvidia-cublas==13.1.1.3
|
||||
nvidia-cuda-cupti==13.0.85
|
||||
nvidia-cuda-nvrtc==13.0.88
|
||||
nvidia-cuda-runtime==13.0.96
|
||||
nvidia-cudnn-cu13==9.20.0.48
|
||||
nvidia-cufft==12.0.0.61
|
||||
nvidia-cufile==1.15.1.6
|
||||
nvidia-curand==10.4.0.35
|
||||
nvidia-cusolver==12.0.4.66
|
||||
nvidia-cusparse==12.6.3.3
|
||||
nvidia-cusparselt-cu13==0.8.1
|
||||
nvidia-nccl-cu13==2.29.7
|
||||
nvidia-nvjitlink==13.3.33
|
||||
nvidia-nvshmem-cu13==3.4.5
|
||||
nvidia-nvtx==13.0.85
|
||||
openai-whisper==20250625
|
||||
pillow==12.2.0
|
||||
pycparser==3.0
|
||||
regex==2026.7.10
|
||||
requests==2.34.2
|
||||
setuptools==78.1.0
|
||||
sounddevice==0.5.5
|
||||
sympy==1.14.0
|
||||
tiktoken==0.13.0
|
||||
torch==2.13.0+rocm7.2
|
||||
torchaudio==2.11.0+rocm7.2
|
||||
torchvision==0.28.0+rocm7.2
|
||||
tqdm==4.68.4
|
||||
triton==3.7.1
|
||||
triton-rocm==3.7.1
|
||||
typing_extensions==4.15.0
|
||||
urllib3==2.7.0
|
||||
webrtcvad==2.0.10
|
||||
@@ -0,0 +1 @@
|
||||
rootProject.name = 'GLaDOS'
|
||||
@@ -0,0 +1,57 @@
|
||||
package me.neurodock.glados;
|
||||
|
||||
import me.neurodock.core.PrintAdvanceMessageHandler;
|
||||
import me.neurodock.core.ToolCallingRender;
|
||||
import me.neurodock.ollama.OllamaFunctionArgument;
|
||||
import me.neurodock.ollama.OllamaFunctionTool;
|
||||
import me.neurodock.ollama.OllamaPerameter;
|
||||
import me.neurodock.ollama.OllamaToolResponse;
|
||||
import me.neurodock.ollama.exceptions.OllamaToolErrorException;
|
||||
import org.json.JSONObject;
|
||||
|
||||
import java.util.Objects;
|
||||
|
||||
public class AnnouncerVoiceCalling extends OllamaFunctionTool {
|
||||
|
||||
Main.PlaybackHandler pb;
|
||||
public AnnouncerVoiceCalling(Main.PlaybackHandler printAdvanceMessageHandler) {
|
||||
this.pb = printAdvanceMessageHandler;
|
||||
}
|
||||
|
||||
@org.jetbrains.annotations.NotNull
|
||||
@Override
|
||||
public String name() {
|
||||
return "pa_announcer";
|
||||
}
|
||||
|
||||
@Override
|
||||
public @org.jetbrains.annotations.NotNull OllamaPerameter parameters() {
|
||||
return OllamaPerameter.builder()
|
||||
.addProperty("message", OllamaPerameter.OllamaPerameterBuilder.Type.STRING, "Message to be announced", true)
|
||||
.build();
|
||||
}
|
||||
|
||||
@Override
|
||||
public ToolCallingRender renderCalling(JSONObject calling) {
|
||||
return new ToolCallingRender.Suppress();
|
||||
}
|
||||
|
||||
@Override
|
||||
public @org.jetbrains.annotations.NotNull OllamaToolResponse function(OllamaFunctionArgument... ollamaFunctionArguments) {
|
||||
if (ollamaFunctionArguments.length != 1) {
|
||||
throw new OllamaToolErrorException(name(), "Number of arguments to be announced must be 1");
|
||||
}
|
||||
|
||||
if(!(Objects.equals(ollamaFunctionArguments[0].argument(), "message")))
|
||||
{
|
||||
throw new OllamaToolErrorException(name(), "Message to be announced must be message");
|
||||
}
|
||||
|
||||
if(ollamaFunctionArguments[0].value() instanceof String str) {
|
||||
pb.playAnnouncer(str);
|
||||
return new OllamaToolResponse(name(), "success");
|
||||
}
|
||||
|
||||
throw new OllamaToolErrorException(name(), "Message to be announced must be message");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,212 @@
|
||||
package me.neurodock.glados;
|
||||
|
||||
import io.github.jvoiceproject.piperjni.PiperJNI;
|
||||
import io.github.jvoiceproject.piperjni.PiperVoice;
|
||||
import me.neurodock.core.Core;
|
||||
import me.neurodock.core.LaunchOptions;
|
||||
import me.neurodock.core.PrintAdvanceMessageHandler;
|
||||
import me.neurodock.ollama.OllamaMessage;
|
||||
import me.neurodock.ollama.OllamaMessageRole;
|
||||
import me.neurodock.ollama.OllamaObject;
|
||||
import org.json.JSONObject;
|
||||
|
||||
import javax.sound.sampled.AudioFormat;
|
||||
import javax.sound.sampled.AudioSystem;
|
||||
import javax.sound.sampled.LineUnavailableException;
|
||||
import javax.sound.sampled.TargetDataLine;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class Main {
|
||||
|
||||
static void main(String[] args) throws LineUnavailableException {
|
||||
new Main();
|
||||
|
||||
}
|
||||
|
||||
public Main() throws LineUnavailableException {
|
||||
PlaybackHandler pb = new PlaybackHandler();
|
||||
|
||||
Path socketPath = Paths.get("/tmp/whisper.sock");
|
||||
|
||||
//pb.playGLaDOS("I'm Glaudos");
|
||||
|
||||
Core core = new Core(pb, "127.0.0.1");
|
||||
|
||||
LaunchOptions.getInstance().setLoadOld(false);
|
||||
|
||||
core.setOllamaObjectNoMemory(OllamaObject.builder()
|
||||
.setModel("llama3.2")
|
||||
//.addMessage(new OllamaMessage(OllamaMessageRole.SYSTEM, "You are GLaDOS."))
|
||||
.keep_alive(10)
|
||||
.build());
|
||||
|
||||
//core.addTool(new AnnouncerVoiceCalling(pb), Core.Source.CTP);
|
||||
|
||||
try(WhisperSocketServer server = new WhisperSocketServer(socketPath, event ->
|
||||
{
|
||||
core.getOllamaObject().addMessage(new OllamaMessage(OllamaMessageRole.USER, event.text()));
|
||||
core.qurryOllama().thenApply((json) ->
|
||||
{
|
||||
String message = json.getJSONObject("message").getString("content");
|
||||
if(json.getJSONObject("message").has("content") && !message.isBlank()) {
|
||||
message = message.replaceAll("(?i)GLaDOS", "Glaudos").replaceAll("\"", " quote ");
|
||||
}
|
||||
|
||||
json.getJSONObject("message").put("content", message);
|
||||
return json;
|
||||
}).thenAccept(core::handleResponse).join();
|
||||
}))
|
||||
{
|
||||
server.start();
|
||||
Thread.currentThread().join();
|
||||
}catch (IOException _) {
|
||||
|
||||
} catch (InterruptedException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public class PlaybackHandler implements PrintAdvanceMessageHandler
|
||||
{
|
||||
|
||||
public static final String GLaDOS_PIPER_MODEL = ".local/share/piper/glados/glados_piper_medium.onnx";
|
||||
public static final String PORTAL_ANNOUNCER_PIPER_MODEL = ".local/share/piper/portal-announcer/announcer.onnx";
|
||||
public static final int SENTENCE_PAUSE_DURATION_MS = 500;
|
||||
|
||||
@Override
|
||||
public void printMessage(OllamaMessage ollamaMessage) {
|
||||
System.out.println(ollamaMessage.toString());
|
||||
switch(ollamaMessage.getRole())
|
||||
{
|
||||
case ASSISTANT -> playGLaDOS(ollamaMessage.getContent());
|
||||
case TOOL -> playAnnouncer("Tool: " + new JSONObject(ollamaMessage.getContent()).get("result"));
|
||||
case USER -> System.out.println("User> "+ollamaMessage.getContent());
|
||||
case SYSTEM -> playAnnouncer(ollamaMessage.getContent());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void printErrorMessage(OllamaMessage ollamaMessage) {
|
||||
playAnnouncer("Error: "+ollamaMessage.getContent());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void printToolCalling(String s) {
|
||||
playAnnouncer(s);
|
||||
}
|
||||
|
||||
public void playGLaDOS(String msg) {
|
||||
try (PiperJNI piper = new PiperJNI()) {
|
||||
piper.initialize(true);
|
||||
try (PiperVoice voice = piper.loadVoice(
|
||||
Paths.get(System.getenv("HOME"), GLaDOS_PIPER_MODEL),
|
||||
Paths.get(System.getenv("HOME"),GLaDOS_PIPER_MODEL+".json"),
|
||||
0)) {
|
||||
int sampleRate = voice.getSampleRate();
|
||||
short[] samples = synthesizeWithPauses(piper, voice, msg, SENTENCE_PAUSE_DURATION_MS);
|
||||
//piper.textToAudio(voice, msg);
|
||||
playAudio(samples, sampleRate);
|
||||
} finally {
|
||||
piper.terminate();
|
||||
}
|
||||
}catch (Exception e) {
|
||||
System.out.println("Critial Error on piper-tts!");
|
||||
System.out.println(e.getMessage());
|
||||
e.printStackTrace(System.out);
|
||||
System.exit(-1);
|
||||
}
|
||||
}
|
||||
|
||||
public void playAnnouncer(String msg) {
|
||||
try (PiperJNI piper = new PiperJNI()) {
|
||||
piper.initialize(true);
|
||||
try (PiperVoice voice = piper.loadVoice(
|
||||
Paths.get(System.getenv("HOME"), PORTAL_ANNOUNCER_PIPER_MODEL),
|
||||
Paths.get(System.getenv("HOME"),PORTAL_ANNOUNCER_PIPER_MODEL+".json"),
|
||||
0)) {
|
||||
int sampleRate = voice.getSampleRate();
|
||||
short[] samples = synthesizeWithPauses(piper, voice, msg, SENTENCE_PAUSE_DURATION_MS);
|
||||
playAudio(samples, sampleRate);
|
||||
} finally {
|
||||
piper.terminate();
|
||||
}
|
||||
}catch (Exception e) {
|
||||
System.out.println("Critial Error on piper-tts!");
|
||||
System.out.println(e.getMessage());
|
||||
e.printStackTrace(System.out);
|
||||
System.exit(-1);
|
||||
}
|
||||
}
|
||||
|
||||
public short[] synthesizeWithPauses(PiperJNI piper, PiperVoice voice, String text, int pauseMs) throws PiperJNI.NotInitialized, IOException {
|
||||
// Split, but keep the delimiter so we know which pause length to use
|
||||
String[] chunks = text.split("(?<=\\.\\.\\.)|(?<=[.!?])\\s+|(?<=,)\\s+");
|
||||
List<short[]> pieces = new ArrayList<>();
|
||||
|
||||
for (String chunk : chunks) {
|
||||
if (chunk.isBlank()) continue;
|
||||
String trimmed = chunk.trim();
|
||||
pieces.add(piper.textToAudio(voice, trimmed));
|
||||
|
||||
int thisPauseMs;
|
||||
if (trimmed.endsWith("...")) {
|
||||
thisPauseMs = pauseMs;
|
||||
} else if (trimmed.endsWith(",")) {
|
||||
thisPauseMs = pauseMs / 2;
|
||||
} else if (trimmed.matches(".*[.!?]$")) {
|
||||
thisPauseMs = pauseMs;
|
||||
} else {
|
||||
thisPauseMs = 0; // no trailing punctuation, no forced pause
|
||||
}
|
||||
|
||||
if (thisPauseMs > 0) {
|
||||
int pauseSamples = (voice.getSampleRate() * thisPauseMs) / 1000;
|
||||
pieces.add(new short[pauseSamples]);
|
||||
}
|
||||
}
|
||||
|
||||
int total = pieces.stream().mapToInt(p -> p.length).sum();
|
||||
short[] combined = new short[total];
|
||||
int offset = 0;
|
||||
for (short[] p : pieces) {
|
||||
System.arraycopy(p, 0, combined, offset, p.length);
|
||||
offset += p.length;
|
||||
}
|
||||
return combined;
|
||||
}
|
||||
|
||||
public byte[] shortsToBytes(short[] samples) {
|
||||
byte[] bytes = new byte[samples.length * 2];
|
||||
for (int i = 0; i < samples.length; i++) {
|
||||
bytes[i * 2] = (byte) (samples[i] & 0xFF); // low byte
|
||||
bytes[i * 2 + 1] = (byte) ((samples[i] >> 8) & 0xFF); // high byte
|
||||
}
|
||||
return bytes;
|
||||
}
|
||||
|
||||
public void playAudio(short[] samples, int sampleRate) throws IOException, InterruptedException {
|
||||
ProcessBuilder pb = new ProcessBuilder(
|
||||
"pw-play",
|
||||
"--rate=" + sampleRate,
|
||||
"--channels=1",
|
||||
"--format=s16",
|
||||
"--volume=0.3",
|
||||
"--raw",
|
||||
"-"
|
||||
);
|
||||
pb.redirectErrorStream(true);
|
||||
Process proc = pb.start();
|
||||
|
||||
try (OutputStream os = proc.getOutputStream()) {
|
||||
os.write(shortsToBytes(samples));
|
||||
}
|
||||
proc.waitFor();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
package me.neurodock.glados;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.IOException;
|
||||
import java.net.StandardProtocolFamily;
|
||||
import java.net.UnixDomainSocketAddress;
|
||||
import java.nio.channels.Channels;
|
||||
import java.nio.channels.ServerSocketChannel;
|
||||
import java.nio.channels.SocketChannel;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.function.Consumer;
|
||||
|
||||
public class WhisperSocketServer implements AutoCloseable {
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
public record TranscriptionEvent(String type, String text, double timestampt, String language) {};
|
||||
|
||||
private final Path socketPath;
|
||||
|
||||
private final Consumer<TranscriptionEvent> onTranscription;
|
||||
|
||||
private final ObjectMapper mapper = new ObjectMapper();
|
||||
|
||||
private final ExecutorService executor = Executors.newCachedThreadPool();
|
||||
|
||||
private ServerSocketChannel serverChannel;
|
||||
|
||||
private volatile boolean running = true;
|
||||
|
||||
public WhisperSocketServer(Path socketPath, Consumer<TranscriptionEvent> onTranscription) {
|
||||
this.socketPath = socketPath;
|
||||
this.onTranscription = onTranscription;
|
||||
}
|
||||
|
||||
public void start() throws IOException {
|
||||
Files.deleteIfExists(socketPath);
|
||||
UnixDomainSocketAddress address = UnixDomainSocketAddress.of(socketPath);
|
||||
serverChannel = ServerSocketChannel.open(StandardProtocolFamily.UNIX);
|
||||
serverChannel.bind(address);
|
||||
|
||||
System.out.println("[whisper-socket] listening on "+socketPath);
|
||||
executor.submit(this::acceptLoop);
|
||||
}
|
||||
|
||||
private void acceptLoop() {
|
||||
while (running) {
|
||||
try {
|
||||
SocketChannel client = serverChannel.accept();
|
||||
System.out.println("[whisper-socket] python client connected");
|
||||
executor.submit(() -> handleClient(client));
|
||||
}catch(IOException e)
|
||||
{
|
||||
if(running)
|
||||
{
|
||||
System.err.println("[whisper-socket] accept failed: " + e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void handleClient(SocketChannel client) {
|
||||
try(BufferedReader reader = new BufferedReader(Channels.newReader(client, StandardCharsets.UTF_8))){
|
||||
String line;
|
||||
while((line = reader.readLine()) != null)
|
||||
{
|
||||
if(line.isBlank()) continue;
|
||||
try {
|
||||
TranscriptionEvent event = mapper.readValue(line, TranscriptionEvent.class);
|
||||
onTranscription.accept(event);
|
||||
} catch (Exception parseEx) {
|
||||
System.err.println("[whisper-socket] bad line: " + line + " (" + parseEx.getMessage() + ")");
|
||||
}
|
||||
}
|
||||
} catch (IOException e) {
|
||||
System.err.println("[whisper-socket] client read error: " + e.getMessage());
|
||||
} finally {
|
||||
System.out.println("[whisper-socket] python client disconnected");
|
||||
try {
|
||||
client.close();
|
||||
} catch (IOException ignored) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
running = false;
|
||||
executor.shutdownNow();
|
||||
if (serverChannel != null) {
|
||||
serverChannel.close();
|
||||
}
|
||||
Files.deleteIfExists(socketPath);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,193 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
whisper_daemon.py - Live microphone transcription via OpenAI Whisper (PyTorch/ROCm backend)
|
||||
|
||||
Captures mic audio, segments it into utterances using WebRTC VAD (rather than
|
||||
blindly transcribing fixed windows), runs each finished segment through Whisper,
|
||||
and streams the resulting text as newline-delimited JSON over a Unix domain socket.
|
||||
|
||||
Architecture assumption: this process is a CLIENT. It connects OUT to a Unix
|
||||
socket that the Java program is listening on (and will retry/reconnect if the
|
||||
Java side isn't up yet). Flip SocketClient -> a socketserver if you'd rather
|
||||
have Python own the socket and Java connect in.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import queue
|
||||
import socket
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
import sounddevice as sd
|
||||
import torch
|
||||
import webrtcvad
|
||||
import whisper
|
||||
|
||||
SAMPLE_RATE = 16000
|
||||
FRAME_MS = 30 # webrtcvad only accepts 10/20/30ms frames
|
||||
FRAME_SAMPLES = int(SAMPLE_RATE * FRAME_MS / 1000)
|
||||
SILENCE_TIMEOUT_MS = 700 # trailing silence before an utterance is finalized
|
||||
MAX_SEGMENT_MS = 15000 # hard cap so long uninterrupted speech still flushes
|
||||
|
||||
|
||||
class SocketClient:
|
||||
"""Unix socket client with blocking connect-retry and reconnect-on-send-failure."""
|
||||
|
||||
def __init__(self, sock_path: str, retry_interval: float = 2.0):
|
||||
self.sock_path = sock_path
|
||||
self.retry_interval = retry_interval
|
||||
self.sock = None
|
||||
self.lock = threading.Lock()
|
||||
self._connect()
|
||||
|
||||
def _connect(self):
|
||||
while self.sock is None:
|
||||
try:
|
||||
s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
s.connect(self.sock_path)
|
||||
self.sock = s
|
||||
print(f"[socket] connected to {self.sock_path}", file=sys.stderr)
|
||||
except (FileNotFoundError, ConnectionRefusedError) as e:
|
||||
print(f"[socket] waiting for {self.sock_path} ({e})", file=sys.stderr)
|
||||
time.sleep(self.retry_interval)
|
||||
|
||||
def send(self, obj: dict):
|
||||
data = (json.dumps(obj) + "\n").encode("utf-8")
|
||||
with self.lock:
|
||||
try:
|
||||
self.sock.sendall(data)
|
||||
except (BrokenPipeError, OSError) as e:
|
||||
print(f"[socket] send failed ({e}), reconnecting", file=sys.stderr)
|
||||
try:
|
||||
self.sock.close()
|
||||
except OSError:
|
||||
pass
|
||||
self.sock = None
|
||||
self._connect()
|
||||
self.sock.sendall(data)
|
||||
|
||||
|
||||
def frame_generator(audio_queue: "queue.Queue"):
|
||||
"""Re-chunk arbitrary-sized audio callback buffers into fixed VAD-sized frames."""
|
||||
buf = np.zeros((0,), dtype=np.int16)
|
||||
while True:
|
||||
chunk = audio_queue.get()
|
||||
if chunk is None:
|
||||
return
|
||||
buf = np.concatenate([buf, chunk])
|
||||
while len(buf) >= FRAME_SAMPLES:
|
||||
frame = buf[:FRAME_SAMPLES]
|
||||
buf = buf[FRAME_SAMPLES:]
|
||||
yield frame
|
||||
|
||||
|
||||
def vad_segmenter(audio_queue: "queue.Queue", vad_aggressiveness: int = 2):
|
||||
"""
|
||||
Consume raw frames, use WebRTC VAD to detect speech, and yield complete
|
||||
utterances (numpy int16 arrays) once trailing silence or max length hits.
|
||||
"""
|
||||
vad = webrtcvad.Vad(vad_aggressiveness)
|
||||
voiced_frames = []
|
||||
silence_ms = 0
|
||||
speech_ms = 0
|
||||
triggered = False
|
||||
|
||||
for frame in frame_generator(audio_queue):
|
||||
is_speech = vad.is_speech(frame.tobytes(), SAMPLE_RATE)
|
||||
|
||||
if not triggered:
|
||||
if is_speech:
|
||||
triggered = True
|
||||
voiced_frames = [frame]
|
||||
speech_ms = FRAME_MS
|
||||
silence_ms = 0
|
||||
else:
|
||||
voiced_frames.append(frame)
|
||||
speech_ms += FRAME_MS
|
||||
if is_speech:
|
||||
silence_ms = 0
|
||||
else:
|
||||
silence_ms += FRAME_MS
|
||||
|
||||
if silence_ms >= SILENCE_TIMEOUT_MS or speech_ms >= MAX_SEGMENT_MS:
|
||||
segment = np.concatenate(voiced_frames)
|
||||
triggered = False
|
||||
voiced_frames = []
|
||||
silence_ms = 0
|
||||
speech_ms = 0
|
||||
yield segment
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Live Whisper transcription -> Unix socket")
|
||||
parser.add_argument("--socket", default="/tmp/whisper.sock",
|
||||
help="Unix socket path to connect to (Java side should be listening here)")
|
||||
parser.add_argument("--model", default="small",
|
||||
help="Whisper model size: tiny/base/small/medium/large-v3")
|
||||
parser.add_argument("--device", default=None,
|
||||
help="Force device string (cuda/cpu). Default: auto-detect via torch.cuda.is_available() "
|
||||
"(ROCm builds of torch report themselves as 'cuda').")
|
||||
parser.add_argument("--vad-aggressiveness", type=int, default=2, choices=[0, 1, 2, 3],
|
||||
help="0=least aggressive filtering (more false positives), 3=most aggressive")
|
||||
parser.add_argument("--language", default=None, help="Force language code e.g. 'en'. Default: auto-detect.")
|
||||
parser.add_argument("--input-device", type=int, default=None,
|
||||
help="sounddevice input device index, see --list-devices")
|
||||
parser.add_argument("--list-devices", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.list_devices:
|
||||
print(sd.query_devices())
|
||||
return
|
||||
|
||||
device = args.device or ("cuda" if torch.cuda.is_available() else "cpu")
|
||||
print(f"[whisper] loading model '{args.model}' on device '{device}'", file=sys.stderr)
|
||||
model = whisper.load_model(args.model, device=device)
|
||||
|
||||
client = SocketClient(args.socket)
|
||||
audio_queue: "queue.Queue" = queue.Queue()
|
||||
|
||||
def audio_callback(indata, frames, time_info, status):
|
||||
if status:
|
||||
print(f"[audio] {status}", file=sys.stderr)
|
||||
audio_queue.put(indata[:, 0].copy())
|
||||
|
||||
stream = sd.InputStream(
|
||||
samplerate=SAMPLE_RATE,
|
||||
channels=1,
|
||||
dtype="int16",
|
||||
blocksize=FRAME_SAMPLES,
|
||||
device=args.input_device,
|
||||
callback=audio_callback,
|
||||
)
|
||||
|
||||
print("[whisper] listening... (Ctrl+C to stop)", file=sys.stderr)
|
||||
with stream:
|
||||
try:
|
||||
for segment in vad_segmenter(audio_queue, args.vad_aggressiveness):
|
||||
audio_f32 = segment.astype(np.float32) / 32768.0
|
||||
result = model.transcribe(
|
||||
audio_f32,
|
||||
language=args.language,
|
||||
fp16=(device != "cpu"),
|
||||
condition_on_previous_text=False,
|
||||
)
|
||||
text = result.get("text", "").strip()
|
||||
if text:
|
||||
payload = {
|
||||
"type": "transcript",
|
||||
"text": text,
|
||||
"timestamp": time.time(),
|
||||
"language": result.get("language"),
|
||||
}
|
||||
print(f"[transcript] {text}")
|
||||
client.send(payload)
|
||||
except KeyboardInterrupt:
|
||||
print("\n[whisper] stopping", file=sys.stderr)
|
||||
audio_queue.put(None)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user