From 140f8c446620096ac1c2a94c887f7ab87a977643 Mon Sep 17 00:00:00 2001 From: Zacharias Date: Mon, 19 Oct 2026 22:41:06 +0200 Subject: [PATCH] Upddated things to match newer version of NeuroDock --- build.gradle | 10 + .../glados/AnnouncerVoiceCalling.java | 40 ++- .../java/me/neurodock/glados/KokoroVoice.java | 2 +- src/main/java/me/neurodock/glados/Main.java | 244 ++++++++++++------ .../me/neurodock/glados/SentenceChunker.java | 54 ++-- .../neurodock/glados/WhisperSocketServer.java | 3 +- .../glados/rest/controller/Message.java | 39 ++- .../glados/rest/payloads/MessagePayload.java | 7 +- 8 files changed, 255 insertions(+), 144 deletions(-) diff --git a/build.gradle b/build.gradle index aef75c7..7cf25ed 100644 --- a/build.gradle +++ b/build.gradle @@ -6,6 +6,8 @@ plugins { id 'org.springframework.boot' version '4.1.0-M4' id 'io.spring.dependency-management' version '1.1.4' + + id 'application' } group = 'me.zacharias' @@ -70,6 +72,14 @@ tasks.named('processResources') { dependsOn 'copyPythonRuntime' } +application { + mainClass = 'me.neurodock.glados.Main' +} + +tasks.named('run') { + standardInput = System.in +} + jar { manifest { attributes 'Main-Class': 'me.neurodock.glados.Main' diff --git a/src/main/java/me/neurodock/glados/AnnouncerVoiceCalling.java b/src/main/java/me/neurodock/glados/AnnouncerVoiceCalling.java index 1cceedc..0254457 100644 --- a/src/main/java/me/neurodock/glados/AnnouncerVoiceCalling.java +++ b/src/main/java/me/neurodock/glados/AnnouncerVoiceCalling.java @@ -1,16 +1,16 @@ package me.neurodock.glados; import me.neurodock.core.ToolCallingRender; -import me.neurodock.ollama.OllamaFunctionArgument; -import me.neurodock.ollama.OllamaFunctionTool; -import me.neurodock.ollama.OllamaPerameter; -import me.neurodock.ollama.OllamaToolResponse; -import me.neurodock.ollama.exceptions.OllamaToolErrorException; +import me.neurodock.llm.exceptions.ToolException; +import me.neurodock.llm.tools.FunctionTool; +import me.neurodock.llm.tools.ToolArguments; +import me.neurodock.llm.tools.ToolParameters; +import me.neurodock.llm.tools.ToolResponse; import org.json.JSONObject; import java.util.Objects; -public class AnnouncerVoiceCalling extends OllamaFunctionTool { +public class AnnouncerVoiceCalling extends FunctionTool { Main.PlaybackHandlerPiper pb; public AnnouncerVoiceCalling(Main.PlaybackHandlerPiper printAdvanceMessageHandler) { @@ -24,9 +24,9 @@ public class AnnouncerVoiceCalling extends OllamaFunctionTool { } @Override - public @org.jetbrains.annotations.NotNull OllamaPerameter parameters() { - return OllamaPerameter.builder() - .addProperty("message", OllamaPerameter.OllamaPerameterBuilder.Type.STRING, "Message to be announced", true) + public @org.jetbrains.annotations.NotNull ToolParameters parameters() { + return ToolParameters.builder() + .addProperty("message", ToolParameters.ToolParametersBuilder.Type.STRING, "Message to be announced", true) .build(); } @@ -36,21 +36,15 @@ public class AnnouncerVoiceCalling extends OllamaFunctionTool { } @Override - public @org.jetbrains.annotations.NotNull OllamaToolResponse function(OllamaFunctionArgument... ollamaFunctionArguments) { - if (ollamaFunctionArguments.length != 1) { - throw new OllamaToolErrorException(name(), "Number of arguments to be announced must be 1"); + public @org.jetbrains.annotations.NotNull ToolResponse function(ToolArguments ollamaFunctionArguments) { + + String line = ollamaFunctionArguments.optArgument("message", String.class); + + if(line == null) { + throw new ToolException(this, "Please provide a message"); } - if(!(Objects.equals(ollamaFunctionArguments[0].argument(), "message"))) - { - throw new OllamaToolErrorException(name(), "Message to be announced must be message"); - } - - if(ollamaFunctionArguments[0].value() instanceof String str) { - pb.playAnnouncer(str); - return new OllamaToolResponse(name(), "success"); - } - - throw new OllamaToolErrorException(name(), "Message to be announced must be message"); + pb.playAnnouncer(line); + return new ToolResponse(name(), "success"); } } diff --git a/src/main/java/me/neurodock/glados/KokoroVoice.java b/src/main/java/me/neurodock/glados/KokoroVoice.java index 8af4da4..51c081f 100644 --- a/src/main/java/me/neurodock/glados/KokoroVoice.java +++ b/src/main/java/me/neurodock/glados/KokoroVoice.java @@ -84,7 +84,7 @@ public class KokoroVoice implements AutoCloseable { LongBuffer.wrap(tokenIds), new long[]{1, tokenIds.length}); OnnxTensor styleTensor = OnnxTensor.createTensor(env, FloatBuffer.wrap(style), new long[]{1, style.length}); - OnnxTensor speed = OnnxTensor.createTensor(env, new float[]{1.0f})) { + OnnxTensor speed = OnnxTensor.createTensor(env, new float[]{1.2f})) { Map inputs = Map.of( "input_ids", inputIds, "style", styleTensor, "speed", speed); diff --git a/src/main/java/me/neurodock/glados/Main.java b/src/main/java/me/neurodock/glados/Main.java index 65cbaae..8b53522 100644 --- a/src/main/java/me/neurodock/glados/Main.java +++ b/src/main/java/me/neurodock/glados/Main.java @@ -3,16 +3,23 @@ package me.neurodock.glados; import com.vdurmont.emoji.EmojiParser; import io.github.jvoiceproject.piperjni.PiperJNI; import io.github.jvoiceproject.piperjni.PiperVoice; +import me.neurodock.backend.open.ai.OpenAIModel; +import me.neurodock.backend.open.ai.OpenAIThinking; import me.neurodock.core.*; -import me.neurodock.core.files.FileHandlerLocation; import me.neurodock.core.memory.CoreMemory; import me.neurodock.glados.rest.APIApplication; -import me.neurodock.ollama.*; +import me.neurodock.llm.Message; +import me.neurodock.llm.tools.FunctionTool; +import me.neurodock.llm.tools.Tool; +import me.neurodock.llm.tools.ToolParameters; +import me.neurodock.core.LLMSystemPrompt.OutputFormat; +import org.json.JSONArray; import org.json.JSONObject; import picocli.CommandLine; import picocli.CommandLine.Option; import picocli.CommandLine.Command; +import javax.naming.OperationNotSupportedException; import javax.sound.sampled.LineUnavailableException; import java.io.BufferedReader; import java.io.IOException; @@ -23,7 +30,10 @@ import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.function.Consumer; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import static me.neurodock.core.Core.writeLog; @@ -38,6 +48,9 @@ public class Main implements Runnable { @Option(names = {"-c", "--cli"}, description = "Weather a CLI input should be ran") boolean withCLI = true; + @Option(names = {"-v", "--verbose"}, description = "Weather this run shuld be verbose or not") + boolean verbose = false; + public PlaybackHandlerPiper pb; public PlaybackHandlerKokoro pbk; @@ -46,13 +59,16 @@ public class Main implements Runnable { SentenceChunker chunker = new SentenceChunker(); Consumer onStreamChunkConsumer = (chunk) -> { - JSONObject message = chunk.getJSONObject("message"); - String content = message.optString("content", ""); - if (!content.isEmpty()) { + if(verbose) + { + System.out.println("Chunk: " + chunk); + } + String content = (String) chunk.optJSONArray("choices", new JSONArray()).optJSONObject(0, new JSONObject()).optJSONObject("delta", new JSONObject()).optString("content", ""); + if (content != null && !content.isEmpty()) { for (String sentence : chunker.appendAndExtract(content)) { System.out.println("Sentance:> " + sentence); //pb.playGLaDOS(sentence); - pbk.playVoice(sentence); + pbk.playVoice(stripMarkdownForTts(sentence)); } } }; @@ -66,34 +82,11 @@ public class Main implements Runnable { public void run() { - APIApplication.start(); - APIApplication.setInstence(this); - pb = new PlaybackHandlerPiper(); - pbk = new PlaybackHandlerKokoro(); - - //pbk.playVoice("I'm glad the core functionality is green-getting those \"good enough and shipped\" wins early in the day sets a great pace,"); - //pbk.playVoice("especially when tests are finally passing without drama."); - //pbk.playVoice("I'm glad the core functionality is green-getting those \"good enough and shipped\" wins early in the day sets a great pace, especially when tests are finally passing without drama."); - //pbk.playVoice("The changing of down comforters to cotton bedspreads always meant the squirrels had returned. She couldn't decide of the glass was half empty or half full so she drank it."); - - Path socketPath = Paths.get("/tmp/whisper.sock"); - - //pbk.playVoice("I'm Glaudos"); - - Core.setDataDirectory("GLaDOS",false); - - core = new Core(pb, "127.0.0.1"); - - LaunchOptions.getInstance().setLoadOld(false); - - core.setOllamaObjectNoMemory(OllamaObject.builder() - .setModel("qwen3.5") - //.addMessage(new OllamaMessage(OllamaMessageRole.SYSTEM, "You are GLaDOS.")) - .setSystemPrompt(LLMSystemPrompt.builder() - .identity(new LLMSystemPrompt.Identity("Alice", "Personal assistant", null)) - .context(new LLMSystemPrompt.Context() - .addFact(""" + LLMSystemPrompt identity = LLMSystemPrompt.builder() + .identity(new LLMSystemPrompt.Identity("Alice", "Personal assistant", null)) + .context(new LLMSystemPrompt.Context() + .addFact(""" Language policy (highest priority): - Every response MUST be written in English. - Never respond in Swedish. @@ -101,7 +94,7 @@ public class Main implements Runnable { - If the user writes in Swedish, answer in English. - Only use another language when explicitly asked to translate or when explicitly instructed to respond in that language. """) - .addFact(""" + .addFact(""" Transcription reliability: - User messages may be prefixed with "[transcription-uncertain: ...]" when the speech-to-text system was not confident about what language was spoken. @@ -113,19 +106,39 @@ public class Main implements Runnable { - Never mention this tag's literal text to the user — react to its meaning, not its syntax. """) - .build()) - .loadBehaviorFromFile("Alice.txt") - .capabilities(new LLMSystemPrompt.Capabilities() - .use(AnnouncerVoiceCalling.class, "Use this to make an announcement. Not for regular messages.")) .build()) - .keep_alive(10) - .stream(true) - .setThinkingTier(OllamaObject.Thinking.FALSE) - .build()); + .loadBehaviorFromFile("Alice.txt") + .capabilities(new LLMSystemPrompt.Capabilities() + .use(AnnouncerVoiceCalling.class, "Use this to make an announcement. Not for regular messages.")) + .outputFormat(new OutputFormat(""" + When writing mathematical expressions, always wrap them in $...$ (inline) or $$...$$ (display), and use proper LaTeX commands (\\times, \\cdot) instead of * for multiplication. Never use bare * or _ outside of markdown emphasis or math delimiters.""")) + .build(); + + Message prompt = identity.generateSystemPrompt(); + + pb = new PlaybackHandlerPiper(); + pbk = new PlaybackHandlerKokoro(); + + Path socketPath = Paths.get("/tmp/whisper.sock"); + + Options.getInstance().setDataDir(Path.of("GLaDOS"), false); + + core = new Core(pb); + + LaunchOptions.getInstance().setLoadOld(false); + + String llamaAddress = System.getenv().getOrDefault("LLAMA_ADDRESS", "http://127.0.0.1:8080"); + + core.setModelNoMemory(new OpenAIModel(llamaAddress, "unsloth/Qwen3.5", OpenAIThinking.LOW)); //core.addTool(new AnnouncerVoiceCalling(pb), Core.Source.CTP); - core.getOllamaObject().reGenerateSystemPrompt(); + //core.getOllamaObject().reGenerateSystemPrompt(); + + core.addMessage(prompt); + + APIApplication.start(); + APIApplication.setInstence(this); try(WhisperSocketServer server = new WhisperSocketServer(socketPath, event -> { @@ -140,7 +153,7 @@ public class Main implements Runnable { event.getKey().runner_up(), event.getKey().runner_up_confidence() * 100, event.getKey().text()) : event.getKey().text(); - sendAndHandleMessage(fixedMessage.toString(), true); + sendAndHandleMessage(fixedMessage.toString(), false); })) { server.start(); @@ -164,6 +177,7 @@ public class Main implements Runnable { /corelist Lists all tools according to the OllamaObject. /working Prints the current working directories. /peek Peeks the current memory. + /speek Speaks out the last message again. """); break; case "bye": @@ -192,40 +206,39 @@ public class Main implements Runnable { case "list": writeLog("Tools installed in this instance"); - for (Pair funtion : core.getFuntionTools()) { - StringBuilder args = new StringBuilder(); - OllamaPerameter perameter = funtion.getKey().parameters(); - if (perameter != null) { - JSONObject obj = perameter.getProperties(); - for (String name : obj.keySet()) { - args.append(args.toString().isBlank() ? "" : ", ").append(obj.getJSONObject(name).getString("type")).append(Arrays.stream(perameter.getRequired()).anyMatch(str -> str.equalsIgnoreCase(name)) ? "" : "?").append(" ").append(name); + for (Tool tool : core.getTools()) { + if(tool instanceof FunctionTool function) { + StringBuilder args = new StringBuilder(); + ToolParameters perameter = function.parameters(); + if (perameter != null) { + Map obj = perameter.getProperties(); + for (String name : obj.keySet()) { + args.append(args.toString().isBlank() ? "" : ", ").append(obj.get(name).type()).append(perameter.getRequired().stream().anyMatch(str -> str.equalsIgnoreCase(name)) ? "" : "?").append(" ").append(name); + } } + + System.out.println("> Function: " + function.name() + "(" + args + ")"); + writeLog("Function: " + function.name() + "(" + args + ")"); } - - System.out.println("> Function: " + funtion.getKey().name() + "(" + args + ") [" + funtion.getValue() + "]"); - writeLog("Function: " + funtion.getKey().name() + "(" + args + ") [" + funtion.getValue() + "]"); - } - break; - case "corelist": - writeLog("Tools installed in this instance acording to the coire OllamaObject"); - - for (Pair funtion : core.getOllamaObject().getTools()) { - System.out.println("> Function: " + funtion.getKey().toJSON()); - writeLog("Function: " + funtion.getKey().toJSON()); } break; case "working": System.out.println("Working directories:\n" + - " Data: " + Core.DATA_DIR.getAbsolutePath() + "\n" + - " DateFiles: " + FileHandlerLocation.DATA_FILES + "\n" + - " Plugins: " + Core.PLUGIN_DIRECTORY.getAbsolutePath()); + " Data: " + Options.getInstance().getDataDir().getAbsolutePath() + "\n" + + " DateFiles: " + Options.getInstance().getFileHandlerDataLocation() + "\n" + + " Plugins: " + Options.getInstance().getPluginDirectory().getAbsolutePath()); break; + case "speek": + for(String str : chunker.appendAndExtract((String) core.getChatObject().getConversation().getLast().getContent())) + { + pbk.playVoice(stripMarkdownForTts(str)); + } default: System.out.println("Unknown command: " + message); } } else { writeLog("User: " + message); - sendAndHandleMessage(message.toString(), true); + sendAndHandleMessage(message.toString(), false); } } } catch (Exception e) { @@ -245,25 +258,96 @@ public class Main implements Runnable { } } - public JSONObject sendAndHandleMessage(String message, boolean noVoice) { - core.getOllamaObject().addMessage(new OllamaMessage(OllamaMessageRole.USER, message)); - return core.qurryOllama((noVoice?((_)->{}):onStreamChunkConsumer)).thenApply(json -> { - core.handleResponse(json); - return json; - }).join(); + private static final Pattern MATH_OR_CODE = Pattern.compile( + "```.*?```" // fenced code blocks (multiline) + + "|`[^`\\n]+?`" // inline code spans + + "|\\$\\$.*?\\$\\$" // display math $$...$$ + + "|\\$[^$\\n]+?\\$" // inline math $...$ + + "|\\\\\\[.*?\\\\\\]" // display math \[...\] + + "|\\\\\\(.*?\\\\\\)", // inline math \(...\) + Pattern.DOTALL + ); + + private static final Pattern MARKDOWN_EMPHASIS = Pattern.compile( + "\\*\\*\\*(.+?)\\*\\*\\*" // ***bold italic*** + + "|\\*\\*(.+?)\\*\\*" // **bold** + + "|\\*(.+?)\\*" // *italic* + + "|___(.+?)___" // ___bold italic___ + + "|__(.+?)__" // __bold__ + + "|_(.+?)_" // _italic_ + + "|~~(.+?)~~" // ~~strikethrough~~ + ); + + /** + * Strips markdown emphasis syntax (*, _, **, __, ~~) from text intended for + * TTS, while leaving math and code regions untouched, since those characters + * are semantically meaningful there (multiplication, subscripts, etc.). + */ + public String stripMarkdownForTts(String input) { + // Step 1: pull out math/code spans, replace with placeholders + List protectedSpans = new ArrayList<>(); + Matcher protectMatcher = MATH_OR_CODE.matcher(input); + StringBuilder withPlaceholders = new StringBuilder(); + int lastEnd = 0; + + while (protectMatcher.find()) { + withPlaceholders.append(input, lastEnd, protectMatcher.start()); + String placeholder = "\u0000" + protectedSpans.size() + "\u0000"; + protectedSpans.add(protectMatcher.group()); + withPlaceholders.append(placeholder); + lastEnd = protectMatcher.end(); + } + withPlaceholders.append(input.substring(lastEnd)); + + // Step 2: strip markdown emphasis from what's left + String stripped = MARKDOWN_EMPHASIS.matcher(withPlaceholders.toString()) + .replaceAll(mr -> { + for (int i = 1; i <= mr.groupCount(); i++) { + if (mr.group(i) != null) return mr.group(i); + } + return mr.group(); + }); + + // Step 3: restore protected spans + StringBuilder result = new StringBuilder(stripped); + for (int i = 0; i < protectedSpans.size(); i++) { + String placeholder = "\u0000" + i + "\u0000"; + int idx = result.indexOf(placeholder); + if (idx != -1) { + result.replace(idx, idx + placeholder.length(), protectedSpans.get(i)); + } + } + + return result.toString(); + } + + public Message sendAndHandleMessage(String message, boolean noVoice) { + core.addMessage(new Message(Message.Role.USER, message)); + try { + return core.queryModel((noVoice?((_)->{}):onStreamChunkConsumer)).thenApply(json -> { + //onStreamChunkConsumer.accept(json); + pbk.playVoice(stripMarkdownForTts(chunker.flush())); + core.handleResponse(json); + return json; + }).join(); + } catch (OperationNotSupportedException e) { + throw new RuntimeException(e); + } } public void sendAndHandleMessageNonBlocking(String message) { - core.getOllamaObject().addMessage(new OllamaMessage(OllamaMessageRole.USER, message)); - core.qurryOllama(onStreamChunkConsumer).thenAccept(core::handleResponse); + core.addMessage(new Message(Message.Role.USER, message)); + try { + core.queryModel(onStreamChunkConsumer).thenAccept(core::handleResponse); + }catch (Exception _) {} } public class PlaybackHandlerKokoro { - public static final Path KOKORO_82M_MODEL_FP16 = Path.of("/home/zacharias/.local/share/onnxruntime/models/kokoro-82M-model_fp16.onnx"); - public static final Path AF_HEART_VOICE = Path.of("/home/zacharias/.local/share/onnxruntime/voices/af_heart.bin"); + public static final Path KOKORO_82M_MODEL_FP16 = Path.of("~/.local/share/onnxruntime/models/kokoro-82M-model_fp16.onnx"); + public static final Path AF_HEART_VOICE = Path.of("~/.local/share/onnxruntime/voices/af_heart.bin"); public void playVoice(String msg) { @@ -353,10 +437,10 @@ public class Main implements Runnable { } @Override - public void printMessage(OllamaMessage ollamaMessage) { - System.out.println(ollamaMessage.toJSON().toString()); + public void printMessage(Message ollamaMessage) { + System.out.println(ollamaMessage.toString()); - String message = ollamaMessage.getContent(); + String message = ollamaMessage.getContent().toString(); message = EmojiParser.parseToAliases(message).replace("::", ": :"); message = message.replaceAll(":(\\w+):", "$1").replace('_', ' '); @@ -374,7 +458,7 @@ public class Main implements Runnable { } @Override - public void printErrorMessage(OllamaMessage ollamaMessage) { + public void printErrorMessage(Message ollamaMessage) { System.out.println("Error: "+ollamaMessage.getContent()); playAnnouncer("Error: "+ollamaMessage.getContent()); } diff --git a/src/main/java/me/neurodock/glados/SentenceChunker.java b/src/main/java/me/neurodock/glados/SentenceChunker.java index 0a2add1..d780907 100644 --- a/src/main/java/me/neurodock/glados/SentenceChunker.java +++ b/src/main/java/me/neurodock/glados/SentenceChunker.java @@ -9,53 +9,57 @@ public class SentenceChunker { private final StringBuilder buffer = new StringBuilder(); - // Matches a sentence-ending punctuation mark followed by whitespace (or end of buffer), - // but only greedily captures the punctuation itself so we can inspect context around it. - private static final Pattern BOUNDARY = Pattern.compile("[.!?]+[\"')\\]]?(?=\\s|$)"); + // Sentence-ending punctuation followed by real whitespace (not end-of-buffer, since more + // text might still arrive: "3." + "14"), or a line break. + private static final Pattern BOUNDARY = Pattern.compile("[.!?]+[\"')\\]]?(?=\\s)|\\n"); + + // Just a 1-3 digit number (optionally indented): the text before the dot in "1." / "12." + private static final Pattern LIST_MARKER = Pattern.compile("\\s*\\d{1,3}"); + + private static final String[] ABBREVIATIONS = {"Mr", "Mrs", "Ms", "Dr", "vs", "etc", "e.g", "i.e"}; - /** - * Appends newly streamed text and extracts any complete sentences now available. - * Incomplete trailing text remains buffered for the next call. - */ public List appendAndExtract(String delta) { buffer.append(delta); List sentences = new ArrayList<>(); Matcher m = BOUNDARY.matcher(buffer); - int lastCut = 0; + int sentenceStart = 0; // where the current sentence begins + int searchFrom = 0; // where to resume scanning - while (m.find(lastCut)) { - int boundaryEnd = m.end(); + while (m.find(searchFrom)) { + int start = m.start(); + int end = m.end(); + searchFrom = end; - if (isFalsePositive(buffer, m.start(), boundaryEnd)) { - lastCut = boundaryEnd; // skip past it, keep scanning the same buffer - continue; - } + if (isFalsePositive(buffer, start, end, sentenceStart)) continue; - String sentence = buffer.substring(lastCut, boundaryEnd).trim(); + String sentence = buffer.substring(sentenceStart, end).trim(); if (!sentence.isEmpty()) sentences.add(sentence); - lastCut = boundaryEnd; + sentenceStart = end; } - buffer.delete(0, lastCut); // keep only the unconsumed remainder + buffer.delete(0, sentenceStart); // keep only the unconsumed remainder return sentences; } - /** Call once generation is done to flush whatever's left, even without terminal punctuation. */ public String flush() { String remainder = buffer.toString().trim(); buffer.setLength(0); return remainder; } - private boolean isFalsePositive(CharSequence buf, int start, int end) { - // Guard: decimal numbers, e.g. "3.14" — digit immediately before and after the dot - if (buf.charAt(start) == '.' && start > 0 && Character.isDigit(buf.charAt(start - 1))) { - if (end < buf.length() && Character.isDigit(buf.charAt(end))) return true; + private boolean isFalsePositive(StringBuilder buf, int start, int end, int sentenceStart) { + if (buf.charAt(start) != '.') return false; // only dots are ambiguous + + // Numbered list marker: a bare "1." at the start of a line (or of the current sentence) + if (end - start == 1) { + int lineStart = Math.max(sentenceStart, buf.lastIndexOf("\n", start - 1) + 1); + if (LIST_MARKER.matcher(buf.subSequence(lineStart, start)).matches()) return true; } - // Guard: common abbreviations right before the dot (extend as needed) - String before = buf.subSequence(Math.max(0, start - 4), start).toString(); - for (String abbr : new String[]{"Mr", "Mrs", "Ms", "Dr", "vs", "etc", "e.g", "i.e"}) { + + // Abbreviations right before the dot + String before = buf.substring(Math.max(0, start - 4), start); + for (String abbr : ABBREVIATIONS) { if (before.endsWith(abbr)) return true; } return false; diff --git a/src/main/java/me/neurodock/glados/WhisperSocketServer.java b/src/main/java/me/neurodock/glados/WhisperSocketServer.java index 82bb0ff..ba7c2ad 100644 --- a/src/main/java/me/neurodock/glados/WhisperSocketServer.java +++ b/src/main/java/me/neurodock/glados/WhisperSocketServer.java @@ -3,7 +3,6 @@ package me.neurodock.glados; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.databind.ObjectMapper; import me.neurodock.core.Pair; -import org.json.JSONObject; import java.io.BufferedReader; import java.io.BufferedWriter; @@ -25,7 +24,7 @@ import java.util.function.Consumer; public class WhisperSocketServer implements AutoCloseable { @JsonIgnoreProperties(ignoreUnknown = true) - public record TranscriptionEvent(String type, String text, double timestampt, String language, + public record TranscriptionEvent(String type, String text, double timestamp, String language, double confidence, String runner_up, double runner_up_confidence) { public boolean isAmbiguous(double minGap) { diff --git a/src/main/java/me/neurodock/glados/rest/controller/Message.java b/src/main/java/me/neurodock/glados/rest/controller/Message.java index 95096ea..291fb38 100644 --- a/src/main/java/me/neurodock/glados/rest/controller/Message.java +++ b/src/main/java/me/neurodock/glados/rest/controller/Message.java @@ -4,10 +4,9 @@ import io.swagger.v3.oas.annotations.media.Content; import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.responses.ApiResponses; +import me.neurodock.backend.open.ai.OpenAIAssistentMessage; import me.neurodock.glados.rest.APIApplication; import me.neurodock.glados.rest.payloads.MessagePayload; -import me.neurodock.ollama.OllamaMessageRole; -import org.json.JSONObject; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.*; @@ -38,13 +37,28 @@ public ResponseEntity send(@RequestBody MessagePayload payload) return ResponseEntity.ok("success"); } else { - JSONObject body = APIApplication.getInstence().sendAndHandleMessage(payload.message(), payload.noVoice()); - MessagePayload response = new MessagePayload( - body.getJSONObject("message").optString("content", ""), - OllamaMessageRole.ASSISTANT, - body.optBoolean("done", false), - payload.noVoice(), - false); + me.neurodock.llm.Message message = APIApplication.getInstence().sendAndHandleMessage(payload.message(), payload.noVoice()); + MessagePayload response; + if(message instanceof OpenAIAssistentMessage msg) + { + response = new MessagePayload( + (String) message.getContent(), + me.neurodock.llm.Message.Role.ASSISTANT, + true, + payload.noVoice(), + false, + msg.getEndReason()); + } + else + { + response = new MessagePayload( + (String) message.getContent(), + me.neurodock.llm.Message.Role.ASSISTANT, + true, + payload.noVoice(), + false, + OpenAIAssistentMessage.Reason.NULL); + } return ResponseEntity.ok(response); } } @@ -57,8 +71,11 @@ public ResponseEntity send(@RequestBody MessagePayload payload) { ArrayList body = new ArrayList<>(); - APIApplication.getInstence().getCore().getOllamaObject().getMessages().forEach(mess -> { - body.add(new MessagePayload(mess.getContent(), mess.getRole(), false, false, false)); + APIApplication.getInstence().getCore().getChatObject().getConversation().forEach(mess -> { + if (mess instanceof OpenAIAssistentMessage msg) + body.add(new MessagePayload((String) mess.getContent(), mess.getRole(), false, false, false, msg.getEndReason())); + else + body.add(new MessagePayload((String) mess.getContent(), mess.getRole(), false, false, false, OpenAIAssistentMessage.Reason.STOP)); }); return ResponseEntity.ok(body); diff --git a/src/main/java/me/neurodock/glados/rest/payloads/MessagePayload.java b/src/main/java/me/neurodock/glados/rest/payloads/MessagePayload.java index 7d332e3..9c1c85c 100644 --- a/src/main/java/me/neurodock/glados/rest/payloads/MessagePayload.java +++ b/src/main/java/me/neurodock/glados/rest/payloads/MessagePayload.java @@ -1,6 +1,9 @@ package me.neurodock.glados.rest.payloads; -import me.neurodock.ollama.OllamaMessageRole; -public record MessagePayload(String message, OllamaMessageRole role, boolean finished, boolean noVoice, boolean stream) { +import me.neurodock.backend.open.ai.OpenAIAssistentMessage; +import me.neurodock.llm.Message; + +public record MessagePayload(String message, Message.Role role, boolean finished, boolean noVoice, boolean stream, + OpenAIAssistentMessage.Reason endReason) { }