Stuff :)

.gitignore

@@ -0,0 +1,11 @@
+/tmp
+/out-tsc
+
+/node_modules
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+/.pnp
+.pnp.js
+
+.vscode/*

.idea/.gitignore

@@ -0,0 +1,10 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Ignored default folder with query files
+/queries/
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml
+# Editor-based HTTP Client requests
+/httpRequests/

package.json

@@ -0,0 +1,19 @@
+{
+  "name": "neuro-test",
+  "version": "1.0.0",
+  "description": "",
+  "main": "dist/index.js",
+  "scripts": {
+    "build": "tsc"
+  },
+  "dependencies": {
+    "@types/jsdom": "^27.0.0",
+    "@types/node": "^25.2.3",
+    "jsdom": "^28.0.0",
+    "xmlbuilder2": "^4.0.3"
+  },
+  "devDependencies": {
+    "typescript": "^5.5.3"
+  },
+  "private": true
+}

src/index.ts

@@ -0,0 +1,280 @@
+import * as fs from "fs";
+import { JSDOM } from "jsdom";
+import { spawn } from "child_process";
+import path from "path";
+import os from "os";
+import {Readable} from "node:stream";
+import {createWriteStream} from "node:fs";
+import crypto from "crypto";
+import {hash} from "node:crypto";
+import {it} from "node:test";
+import {fetchLyrics, writeToFile} from "./lyrics";
+
+type genre = {
+    // To be filled out
+}
+
+type coverArt = {
+    id: string;
+    fileName: string;
+    contentType: string;
+    description: string | null;
+    credit: string | null;
+    cloudflareId: string;
+    mediaStorageType: number;
+    absolutePath: string;
+    artist: {} | null;
+    "artist.id": string;
+    "artist.name": string;
+    "artist.socialLink": string | null;
+    upvotes: number;
+    tagString: string;
+}
+
+type thumbnailArt = {
+    // To be filled out
+}
+
+type song = {
+    id: string;
+    title: string;
+    absolutePath: string;
+    playCount: number;
+    duration: number;
+    streamData: string | null;
+    dateAdded: string | null;
+    coverArtists: string[];
+    originalArtists: string[];
+    genres: genre[] | null;
+    coverArt: coverArt | null;
+    thumbnailArt: thumbnailArt | null;
+    order: number;
+    hasLyrics: boolean;
+    userUploaded: boolean;
+    videoId: string | null;
+    hls: string | null;
+    karaokeDate: string;
+}
+
+type artist = {
+    id: string;
+    name: string;
+    summary: string | null;
+    content: string | null;
+    imagePath: string | null;
+    mediaID: string | null;
+    songListDTOs: string | null;
+    songCount: number;
+    newImageBytes: string | null
+}
+
+let fails: song[] = [];
+
+const date = new Date();
+
+const outputDir = path.join(
+    os.homedir(),
+    //`Neuro_dump/Neuro-snapshot-live-dev1`
+    `Neuro_dump/Neuro-snapshot-${date.toLocaleDateString().replace(/\//g, "_")}`
+);
+
+const downloadedPath = path.join(outputDir, "../.downloaded.json");
+
+fs.mkdir(outputDir, { recursive: true }, (err) => {
+    if (err) {
+        console.log(err);
+        process.exit(1);
+    }
+});
+
+let downloaded: Set<string> = new Set(
+    fs.existsSync(downloadedPath)
+        ? JSON.parse(fs.readFileSync(downloadedPath, "utf-8"))
+        : []
+);
+
+function genHash(name: string, artists: string[], length: number) {
+    const str = JSON.stringify({
+        'name': name,
+        'artists': artists,
+        'length': length,
+    });
+    return crypto.createHash('sha256').update(str).digest('hex');
+}
+
+async function sleep(ms: number): Promise<void> {
+    return new Promise(
+        (resolve)=> setTimeout(resolve, ms));
+}
+
+function runProcess(cmd: string, args: string[]): Promise<"ok" | "skipped"> {
+    return new Promise((resolve, reject) => {
+        const proc = spawn(cmd, args);
+
+        let stderr = "";
+
+        proc.stdout.pipe(process.stdout);
+        proc.stderr.on("data", data => {
+            const text = data.toString();
+            stderr += text;
+            process.stderr.write(text);
+        });
+
+        proc.on("error", reject);
+
+        proc.on("close", (code) => {
+            // yt-dlp uses non-zero for unavailable videos
+            if (
+                code !== 0 &&
+                /video unavailable|removed by the uploader|private video/i.test(stderr)
+            ) {
+                console.warn("⚠️ Video unavailable, skipping.");
+                resolve("skipped");
+                return;
+            }
+
+            if (code === 0) {
+                resolve("ok");
+            } else {
+                reject(new Error(`${cmd} exited with code ${code}`));
+            }
+        });
+    });
+}
+
+async function run()
+{
+    //const artists: artist[] = JSON.parse(await (await fetch("https://api.neurokaraoke.com/api/artists?page=0&pageSize=99999")).text())
+
+    // Get all the songs
+    let songs: {
+        items: song[];
+        totalCount: number;
+        page: number;
+        pageSize: number;
+    } = JSON.parse(await (await fetch("https://api.neurokaraoke.com/api/songs", {
+        method: "POST",
+        body: JSON.stringify({
+            "search": "",
+            "page": 0,
+            "pageSize": 9999,
+            "sortBy": "KaraokeDate",
+            "sortDesc": true,
+            "genreIds": null,
+            "themeIds": null,
+            "moodIds": null,
+            "artistIds": null,
+            "coverArtistIds": null,
+            "energyLevel": null,
+            "tempo": null,
+            "key": null,
+            "karaokeStart": null,
+            "karaokeEnd": null
+        }),
+        headers: {
+            "Content-Type": "application/json",
+            "User-Agent": "Mozilla/5.0 (compatible; neuro-snapshot/1.0)",
+        }
+    })).text());
+
+
+    let avgMs = 0;
+    let num = 0;
+    let skipped = 0;
+    // Loop thru them
+    for (let item of songs.items) {
+        const start = Date.now();
+
+        // UUID for songs
+        const fullID = item.id ?? genHash(item.title, item.originalArtists, item.duration);
+        const shortID = fullID.slice(0, 8);
+
+        // Extract info
+        const songUrl = `https://storage.neurokaraoke.com/${item.absolutePath}`
+        const coverArtUrl = item.coverArt ? `https://images.neurokaraoke.com/${item.coverArt.cloudflareId}/format=png` : null;
+        const songName = item.title || item.title;
+        let coverArtists = item.coverArtists.join(", ").replace(/ & /, ", ");
+        const formatedName = `${songName} - ${item.originalArtists.join(", ")} - ${coverArtists} [${shortID}].mp3`.replace(/\//g, "／").replace(/[/\\:*?"<>|]/g, "-");
+        const oldFormatedName = `${songName} - ${item.originalArtists.join(", ")} - ${coverArtists}.mp3`.replace(/\//g, "／").replace(/[/\\:*?"<>|]/g, "-");
+        const artistsMetadata = `${coverArtists}, ${item.originalArtists.join(", ")}`;
+
+
+        console.log(`Downloading song ${item.title} by ${item.originalArtists.join(", ")} covered by ${coverArtists}`);
+
+        await fetchLyrics(fullID, item.duration, songName).then(lyric => {
+            if(!lyric) return;
+            writeToFile(formatedName, lyric);
+            let delay = 1000 + (Math.random() * 500) - 250
+            console.log(`>> Waiting ${(delay/1000.0).toFixed(1)} seconds :)`)
+            sleep(delay)
+        });
+
+        if(downloaded.has(item.id)) {
+            console.log(`>> Skipping ${item.title}, already downloaded`);
+            num++;
+            skipped++;
+            continue;
+        }
+
+        const isWav = item.absolutePath.endsWith(".wav");
+
+        // Download the song file, and inject the metadata and the cover art
+        const ffmpegArgs = [
+            "-loglevel", "error",
+            "-i", songUrl,
+            ...(coverArtUrl ? ["-i", coverArtUrl] : [] ),
+            "-map", "0:a:0",
+            ...(coverArtUrl ? ["-map", "1"] : []),
+            ...(isWav ? ["-c:a", "libmp3lame", "-q:a", "2"] : ["-c:a", "copy"]),
+            "-c:v", "copy",
+            "-metadata", `title=${item.title}`,
+            "-metadata", `artist=${coverArtists}`,
+            "-metadata", `date=${item.karaokeDate?.split("T")[0] ?? date.getFullYear()}`,
+            "-metadata", `album_artist=${item.originalArtists.join(", ")}`,
+            "-metadata", `album=${item.originalArtists.join(", ")} covers`,
+            "-metadata", `uid=${fullID}`,
+            path.join(outputDir, formatedName),
+        ];
+
+        let res = await runProcess("ffmpeg", ffmpegArgs)
+
+        console.log(res)
+
+        if(res === "ok")
+        {
+            downloaded.add(item.id);
+            fs.writeFileSync(downloadedPath, JSON.stringify([...downloaded]));
+        }
+
+        let delay = 5000 + (Math.random() * 4000)
+        console.log(`>> Waiting ${(delay/1000.0).toFixed(1)} seconds :)`)
+
+        await sleep(delay);
+
+        const duration = Date.now() - start;
+        avgMs = num === 0 ? duration : avgMs * 0.8 + duration * 0.2;
+
+        const remaining = songs.totalCount - (num + 1);
+        const etaMs = avgMs * remaining;
+
+        console.log(`>> Finished ${num + 1}/${songs.totalCount} (${(((num + 1) / songs.totalCount) * 100).toFixed(1)}%)`);
+        console.log(`>> ETA: ${(etaMs / 1000 / 60).toFixed(1)} minutes remaining`);
+
+        num++;
+        if(num >= 10)
+        {
+            //break;
+        }
+    }
+
+    console.log(`Skipped ${skipped}/${songs.totalCount} songs`);
+}
+
+console.log(genHash("LIFE", ["Neuro Sama"], 3*60+32).slice(0,8))
+
+/**
+run().then(r => {
+    fs.writeFile(`./failes-${date.toLocaleDateString().replace(/\//g, "_")}.json`, JSON.stringify(fails, null, 2), () => {});
+})
+/**/
+//fetchData().then(fetched => {console.log(fetched); console.log("Failed songs: ", fails)});

src/lyrics.ts

@@ -0,0 +1,154 @@
+import * as fs from "node:fs";
+import path from "path";
+import os from "os";
+import crypto from "crypto";
+import { create } from "xmlbuilder2";
+import {XMLBuilder} from "xmlbuilder2/lib/interfaces";
+
+interface Caption {
+    begin: string;   // e.g. "00:00:01.000"
+    end: string;
+    text: string;
+    region?: string;
+}
+
+class TtmlDocument {
+    private doc: XMLBuilder;
+    private body: XMLBuilder;
+    private div: XMLBuilder;
+
+    constructor(lang = 'en') {
+        this.doc = create({ version: '1.0', encoding: 'UTF-8' })
+            .ele('tt', {
+                xmlns: 'http://www.w3.org/ns/ttml',
+                'xmlns:tts': 'http://www.w3.org/ns/ttml#styling',
+                'xmlns:ttm': 'http://www.w3.org/ns/ttml#metadata',
+                'xml:lang': lang,
+            });
+
+        // head is optional but commonly holds styling/layout
+        const head = this.doc.ele('head');
+        head.ele('styling')
+            .ele('style', {
+                'xml:id': 'defaultStyle',
+                'tts:fontSize': '100%',
+                'tts:textAlign': 'center',
+            });
+        head.ele('layout')
+            .ele('region', {
+                'xml:id': 'bottom',
+                'tts:displayAlign': 'after',
+            });
+
+        this.body = this.doc.ele('body');
+        this.div = this.body.ele('div');
+    }
+
+    addCaption({ begin, end, text, region = 'bottom' }: Caption): XMLBuilder {
+        return this.div.ele('p', {
+            begin,
+            end,
+            region,
+            style: 'defaultStyle',
+        }).txt(text);
+    }
+
+    // for arbitrary/generic tag insertion anywhere in the tree
+    addTag(
+        parent: XMLBuilder,
+        name: string,
+        attrs: Record<string, string> = {},
+        text?: string
+    ): XMLBuilder {
+        const el = parent.ele(name, attrs);
+        if (text !== undefined) el.txt(text);
+        return el;
+    }
+
+    get root(): XMLBuilder {
+        return this.doc;
+    }
+
+    toString(pretty = true): string {
+        return this.doc.end({ prettyPrint: pretty });
+    }
+}
+
+interface Lyrics {
+    text: string,
+    start: string,
+    end: string
+}
+
+const date = new Date();
+
+const outputDir = path.join(
+    os.homedir(),
+    `Neuro_dump/Neuro-lyrics-snapshot-${date.toLocaleDateString().replace(/\//g, "_")}`
+);
+
+const downloadedPath = path.join(outputDir, "../.downloaded-lyrics.json");
+
+fs.mkdir(outputDir, { recursive: true }, (err) => {
+    if (err) {
+        console.log(err);
+        process.exit(1);
+    }
+});
+
+let downloaded: Set<string> = new Set(
+    fs.existsSync(downloadedPath)
+        ? JSON.parse(fs.readFileSync(downloadedPath, "utf-8"))
+        : []
+);
+
+function formatEndTime(time: number): string
+{
+    const date = new Date(time * 1000);
+    const hours = String(date.getUTCHours()).padStart(2, '0');
+    const minutes = String(date.getUTCMinutes()).padStart(2, '0');
+    const secs = String(date.getUTCSeconds()).padStart(2, '0');
+    const millis = String(date.getUTCMilliseconds()).padStart(3, '0');
+
+    return `${hours}:${minutes}:${secs}.${millis}`;
+}
+
+export async function fetchLyrics(songID: string, totalLength: number, title: string = "unkown"): Promise<string | undefined>
+{
+    if(downloaded.has(songID)) return undefined;
+
+    const rawLyrics: {time: string, text: string}[] = JSON.parse(await (await fetch(`https://api.neurokaraoke.com/api/songs/${songID}/lyrics`)).text());
+    const ttml = new TtmlDocument('en');
+
+    for(let i of rawLyrics.keys())
+    {
+        let lyricsBlock = rawLyrics[i];
+        let nextLyricsBlockEnd = rawLyrics[i+1]?.time ?? formatEndTime(totalLength);
+
+        ttml.addCaption({
+            text: lyricsBlock.text,
+            begin: lyricsBlock.time,
+            end: nextLyricsBlockEnd
+        })
+    }
+
+    downloaded.add(songID);
+    fs.writeFileSync(downloadedPath, JSON.stringify([...downloaded]));
+
+    console.log(`Fetched Lyrics for ${title}`)
+
+    return ttml.toString();
+}
+
+export function writeToFile(formatedName: string, ttml: string)
+{
+    if(formatedName.endsWith(".mp3"))
+    {
+        formatedName = formatedName.replace(".mp3", "");
+    }
+    if(!formatedName.endsWith(".ttml"))
+    {
+        formatedName = formatedName+".ttml";
+    }
+    fs.writeFileSync(outputDir+"/"+formatedName, ttml);
+}

tsconfig.json

@@ -0,0 +1,13 @@
+{
+  "compilerOptions": {
+    "target": "es2016",
+    "module": "commonjs",
+    "esModuleInterop": true,
+    "forceConsistentCasingInFileNames": true,
+    "strict": true,
+    "skipLibCheck": true,
+    "outDir": "dist",
+    "sourceMap": true
+  },
+  "include": ["src"]
+}