commit 6073cbfa972719502d92ee8ae760f9da4eb67d81 Author: Zacharias Date: Fri Sep 4 17:04:20 2026 +0200 Stuff :) diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..32b877b --- /dev/null +++ b/.gitignore @@ -0,0 +1,11 @@ +/tmp +/out-tsc + +/node_modules +npm-debug.log* +yarn-debug.log* +yarn-error.log* +/.pnp +.pnp.js + +.vscode/* \ No newline at end of file diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..ab1f416 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,10 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Ignored default folder with query files +/queries/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml +# Editor-based HTTP Client requests +/httpRequests/ diff --git a/package.json b/package.json new file mode 100644 index 0000000..fcee613 --- /dev/null +++ b/package.json @@ -0,0 +1,19 @@ +{ + "name": "neuro-test", + "version": "1.0.0", + "description": "", + "main": "dist/index.js", + "scripts": { + "build": "tsc" + }, + "dependencies": { + "@types/jsdom": "^27.0.0", + "@types/node": "^25.2.3", + "jsdom": "^28.0.0", + "xmlbuilder2": "^4.0.3" + }, + "devDependencies": { + "typescript": "^5.5.3" + }, + "private": true +} diff --git a/src/index.ts b/src/index.ts new file mode 100644 index 0000000..ec40ab2 --- /dev/null +++ b/src/index.ts @@ -0,0 +1,280 @@ +import * as fs from "fs"; +import { JSDOM } from "jsdom"; +import { spawn } from "child_process"; +import path from "path"; +import os from "os"; +import {Readable} from "node:stream"; +import {createWriteStream} from "node:fs"; +import crypto from "crypto"; +import {hash} from "node:crypto"; +import {it} from "node:test"; +import {fetchLyrics, writeToFile} from "./lyrics"; + +type genre = { + // To be filled out +} + +type coverArt = { + id: string; + fileName: string; + contentType: string; + description: string | null; + credit: string | null; + cloudflareId: string; + mediaStorageType: number; + absolutePath: string; + artist: {} | null; + "artist.id": string; + "artist.name": string; + "artist.socialLink": string | null; + upvotes: number; + tagString: string; +} + +type thumbnailArt = { + // To be filled out +} + +type song = { + id: string; + title: string; + absolutePath: string; + playCount: number; + duration: number; + streamData: string | null; + dateAdded: string | null; + coverArtists: string[]; + originalArtists: string[]; + genres: genre[] | null; + coverArt: coverArt | null; + thumbnailArt: thumbnailArt | null; + order: number; + hasLyrics: boolean; + userUploaded: boolean; + videoId: string | null; + hls: string | null; + karaokeDate: string; +} + +type artist = { + id: string; + name: string; + summary: string | null; + content: string | null; + imagePath: string | null; + mediaID: string | null; + songListDTOs: string | null; + songCount: number; + newImageBytes: string | null +} + +let fails: song[] = []; + +const date = new Date(); + +const outputDir = path.join( + os.homedir(), + //`Neuro_dump/Neuro-snapshot-live-dev1` + `Neuro_dump/Neuro-snapshot-${date.toLocaleDateString().replace(/\//g, "_")}` +); + +const downloadedPath = path.join(outputDir, "../.downloaded.json"); + +fs.mkdir(outputDir, { recursive: true }, (err) => { + if (err) { + console.log(err); + process.exit(1); + } +}); + +let downloaded: Set = new Set( + fs.existsSync(downloadedPath) + ? JSON.parse(fs.readFileSync(downloadedPath, "utf-8")) + : [] +); + +function genHash(name: string, artists: string[], length: number) { + const str = JSON.stringify({ + 'name': name, + 'artists': artists, + 'length': length, + }); + return crypto.createHash('sha256').update(str).digest('hex'); +} + +async function sleep(ms: number): Promise { + return new Promise( + (resolve)=> setTimeout(resolve, ms)); +} + +function runProcess(cmd: string, args: string[]): Promise<"ok" | "skipped"> { + return new Promise((resolve, reject) => { + const proc = spawn(cmd, args); + + let stderr = ""; + + proc.stdout.pipe(process.stdout); + proc.stderr.on("data", data => { + const text = data.toString(); + stderr += text; + process.stderr.write(text); + }); + + proc.on("error", reject); + + proc.on("close", (code) => { + // yt-dlp uses non-zero for unavailable videos + if ( + code !== 0 && + /video unavailable|removed by the uploader|private video/i.test(stderr) + ) { + console.warn("⚠️ Video unavailable, skipping."); + resolve("skipped"); + return; + } + + if (code === 0) { + resolve("ok"); + } else { + reject(new Error(`${cmd} exited with code ${code}`)); + } + }); + }); +} + +async function run() +{ + //const artists: artist[] = JSON.parse(await (await fetch("https://api.neurokaraoke.com/api/artists?page=0&pageSize=99999")).text()) + + // Get all the songs + let songs: { + items: song[]; + totalCount: number; + page: number; + pageSize: number; + } = JSON.parse(await (await fetch("https://api.neurokaraoke.com/api/songs", { + method: "POST", + body: JSON.stringify({ + "search": "", + "page": 0, + "pageSize": 9999, + "sortBy": "KaraokeDate", + "sortDesc": true, + "genreIds": null, + "themeIds": null, + "moodIds": null, + "artistIds": null, + "coverArtistIds": null, + "energyLevel": null, + "tempo": null, + "key": null, + "karaokeStart": null, + "karaokeEnd": null + }), + headers: { + "Content-Type": "application/json", + "User-Agent": "Mozilla/5.0 (compatible; neuro-snapshot/1.0)", + } + })).text()); + + + let avgMs = 0; + let num = 0; + let skipped = 0; + // Loop thru them + for (let item of songs.items) { + const start = Date.now(); + + // UUID for songs + const fullID = item.id ?? genHash(item.title, item.originalArtists, item.duration); + const shortID = fullID.slice(0, 8); + + // Extract info + const songUrl = `https://storage.neurokaraoke.com/${item.absolutePath}` + const coverArtUrl = item.coverArt ? `https://images.neurokaraoke.com/${item.coverArt.cloudflareId}/format=png` : null; + const songName = item.title || item.title; + let coverArtists = item.coverArtists.join(", ").replace(/ & /, ", "); + const formatedName = `${songName} - ${item.originalArtists.join(", ")} - ${coverArtists} [${shortID}].mp3`.replace(/\//g, "／").replace(/[/\\:*?"<>|]/g, "-"); + const oldFormatedName = `${songName} - ${item.originalArtists.join(", ")} - ${coverArtists}.mp3`.replace(/\//g, "／").replace(/[/\\:*?"<>|]/g, "-"); + const artistsMetadata = `${coverArtists}, ${item.originalArtists.join(", ")}`; + + + console.log(`Downloading song ${item.title} by ${item.originalArtists.join(", ")} covered by ${coverArtists}`); + + await fetchLyrics(fullID, item.duration, songName).then(lyric => { + if(!lyric) return; + writeToFile(formatedName, lyric); + let delay = 1000 + (Math.random() * 500) - 250 + console.log(`>> Waiting ${(delay/1000.0).toFixed(1)} seconds :)`) + sleep(delay) + }); + + if(downloaded.has(item.id)) { + console.log(`>> Skipping ${item.title}, already downloaded`); + num++; + skipped++; + continue; + } + + const isWav = item.absolutePath.endsWith(".wav"); + + // Download the song file, and inject the metadata and the cover art + const ffmpegArgs = [ + "-loglevel", "error", + "-i", songUrl, + ...(coverArtUrl ? ["-i", coverArtUrl] : [] ), + "-map", "0:a:0", + ...(coverArtUrl ? ["-map", "1"] : []), + ...(isWav ? ["-c:a", "libmp3lame", "-q:a", "2"] : ["-c:a", "copy"]), + "-c:v", "copy", + "-metadata", `title=${item.title}`, + "-metadata", `artist=${coverArtists}`, + "-metadata", `date=${item.karaokeDate?.split("T")[0] ?? date.getFullYear()}`, + "-metadata", `album_artist=${item.originalArtists.join(", ")}`, + "-metadata", `album=${item.originalArtists.join(", ")} covers`, + "-metadata", `uid=${fullID}`, + path.join(outputDir, formatedName), + ]; + + let res = await runProcess("ffmpeg", ffmpegArgs) + + console.log(res) + + if(res === "ok") + { + downloaded.add(item.id); + fs.writeFileSync(downloadedPath, JSON.stringify([...downloaded])); + } + + let delay = 5000 + (Math.random() * 4000) + console.log(`>> Waiting ${(delay/1000.0).toFixed(1)} seconds :)`) + + await sleep(delay); + + const duration = Date.now() - start; + avgMs = num === 0 ? duration : avgMs * 0.8 + duration * 0.2; + + const remaining = songs.totalCount - (num + 1); + const etaMs = avgMs * remaining; + + console.log(`>> Finished ${num + 1}/${songs.totalCount} (${(((num + 1) / songs.totalCount) * 100).toFixed(1)}%)`); + console.log(`>> ETA: ${(etaMs / 1000 / 60).toFixed(1)} minutes remaining`); + + num++; + if(num >= 10) + { + //break; + } + } + + console.log(`Skipped ${skipped}/${songs.totalCount} songs`); +} + +console.log(genHash("LIFE", ["Neuro Sama"], 3*60+32).slice(0,8)) + +/** +run().then(r => { + fs.writeFile(`./failes-${date.toLocaleDateString().replace(/\//g, "_")}.json`, JSON.stringify(fails, null, 2), () => {}); +}) +/**/ +//fetchData().then(fetched => {console.log(fetched); console.log("Failed songs: ", fails)}); \ No newline at end of file diff --git a/src/lyrics.ts b/src/lyrics.ts new file mode 100644 index 0000000..d75cad3 --- /dev/null +++ b/src/lyrics.ts @@ -0,0 +1,154 @@ +import * as fs from "node:fs"; +import path from "path"; +import os from "os"; +import crypto from "crypto"; +import { create } from "xmlbuilder2"; +import {XMLBuilder} from "xmlbuilder2/lib/interfaces"; + +interface Caption { + begin: string; // e.g. "00:00:01.000" + end: string; + text: string; + region?: string; +} + +class TtmlDocument { + private doc: XMLBuilder; + private body: XMLBuilder; + private div: XMLBuilder; + + constructor(lang = 'en') { + this.doc = create({ version: '1.0', encoding: 'UTF-8' }) + .ele('tt', { + xmlns: 'http://www.w3.org/ns/ttml', + 'xmlns:tts': 'http://www.w3.org/ns/ttml#styling', + 'xmlns:ttm': 'http://www.w3.org/ns/ttml#metadata', + 'xml:lang': lang, + }); + + // head is optional but commonly holds styling/layout + const head = this.doc.ele('head'); + head.ele('styling') + .ele('style', { + 'xml:id': 'defaultStyle', + 'tts:fontSize': '100%', + 'tts:textAlign': 'center', + }); + head.ele('layout') + .ele('region', { + 'xml:id': 'bottom', + 'tts:displayAlign': 'after', + }); + + this.body = this.doc.ele('body'); + this.div = this.body.ele('div'); + } + + addCaption({ begin, end, text, region = 'bottom' }: Caption): XMLBuilder { + return this.div.ele('p', { + begin, + end, + region, + style: 'defaultStyle', + }).txt(text); + } + + // for arbitrary/generic tag insertion anywhere in the tree + addTag( + parent: XMLBuilder, + name: string, + attrs: Record = {}, + text?: string + ): XMLBuilder { + const el = parent.ele(name, attrs); + if (text !== undefined) el.txt(text); + return el; + } + + get root(): XMLBuilder { + return this.doc; + } + + toString(pretty = true): string { + return this.doc.end({ prettyPrint: pretty }); + } +} + +interface Lyrics { + text: string, + start: string, + end: string +} + +const date = new Date(); + +const outputDir = path.join( + os.homedir(), + `Neuro_dump/Neuro-lyrics-snapshot-${date.toLocaleDateString().replace(/\//g, "_")}` +); + +const downloadedPath = path.join(outputDir, "../.downloaded-lyrics.json"); + +fs.mkdir(outputDir, { recursive: true }, (err) => { + if (err) { + console.log(err); + process.exit(1); + } +}); + +let downloaded: Set = new Set( + fs.existsSync(downloadedPath) + ? JSON.parse(fs.readFileSync(downloadedPath, "utf-8")) + : [] +); + +function formatEndTime(time: number): string +{ + const date = new Date(time * 1000); + const hours = String(date.getUTCHours()).padStart(2, '0'); + const minutes = String(date.getUTCMinutes()).padStart(2, '0'); + const secs = String(date.getUTCSeconds()).padStart(2, '0'); + const millis = String(date.getUTCMilliseconds()).padStart(3, '0'); + + return `${hours}:${minutes}:${secs}.${millis}`; +} + +export async function fetchLyrics(songID: string, totalLength: number, title: string = "unkown"): Promise +{ + if(downloaded.has(songID)) return undefined; + + const rawLyrics: {time: string, text: string}[] = JSON.parse(await (await fetch(`https://api.neurokaraoke.com/api/songs/${songID}/lyrics`)).text()); + const ttml = new TtmlDocument('en'); + + for(let i of rawLyrics.keys()) + { + let lyricsBlock = rawLyrics[i]; + let nextLyricsBlockEnd = rawLyrics[i+1]?.time ?? formatEndTime(totalLength); + + ttml.addCaption({ + text: lyricsBlock.text, + begin: lyricsBlock.time, + end: nextLyricsBlockEnd + }) + } + + downloaded.add(songID); + fs.writeFileSync(downloadedPath, JSON.stringify([...downloaded])); + + console.log(`Fetched Lyrics for ${title}`) + + return ttml.toString(); +} + +export function writeToFile(formatedName: string, ttml: string) +{ + if(formatedName.endsWith(".mp3")) + { + formatedName = formatedName.replace(".mp3", ""); + } + if(!formatedName.endsWith(".ttml")) + { + formatedName = formatedName+".ttml"; + } + fs.writeFileSync(outputDir+"/"+formatedName, ttml); +} \ No newline at end of file diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..e09cfff --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "target": "es2016", + "module": "commonjs", + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "strict": true, + "skipLibCheck": true, + "outDir": "dist", + "sourceMap": true + }, + "include": ["src"] +}