{
"generated_at": "2026-10-22T10:43:37.573Z",
"source": [
"shared/data/ai-focus.ts",
"shared/data/ai-cards.ts",
"shared/data/hebrish.ts (https://hebrish.app/data/*.json)",
"shared/data/benchmarks.ts (github.com/danielrosehill/My-Benchmarks)",
"site-base/src/data/projects.json"
],
"about": "Daniel Rosehill's AI work: focus areas, flagship items, model and dataset cards (page = card on ai.danielrosehill.com), the Hebrish series and its leaderboard, benchmarks and evals, every Hugging Face item, and My Weird Prompts figures. Regenerated on every deploy.",
"pages": {
"hub": "https://ai.danielrosehill.com/",
"models": "https://ai.danielrosehill.com/models",
"datasets": "https://ai.danielrosehill.com/datasets",
"benchmarks": "https://ai.danielrosehill.com/benchmarks",
"hugging_face": "https://ai.danielrosehill.com/hugging-face",
"plugins": "https://ai.danielrosehill.com/plugins",
"skills": "https://ai.danielrosehill.com/skills",
"projects": "https://ai.danielrosehill.com/projects"
},
"focus_areas": [
{
"id": "code-switching",
"title": "English–Hebrew code-switching",
"summary": "Speech and text models that cope with \"Hebrish\" — the English-with-Hebrew-words that English speakers in Israel actually use. Off-the-shelf ASR mangles words like teudat zehut or mashkanta; I build the datasets and fine-tunes that fix that."
},
{
"id": "asr",
"title": "Speech-to-text & on-device inference on Android",
"summary": "Whisper fine-tunes, ASR evaluations and small models that run locally on a phone. Most of my writing starts as voice, so I care about accuracy on real voice notes, background noise and fast on-device inference."
},
{
"id": "accessibility",
"title": "Accessibility",
"summary": "Using AI to make technology easier to use: voice-first input instead of typing, TTS that pronounces Hebrew words correctly, and tools that reduce friction for people who work in more than one language."
},
{
"id": "evals",
"title": "Benchmarks & AI safety",
"summary": "Evals for how models behave, not just how they score: does a model lecture the user, repeat propaganda when the premise is loaded, endorse conspiracy theories or assume everyone is American? Plus the speech benchmarks behind my ASR work. Open test prompts on Hugging Face."
},
{
"id": "education",
"title": "AI for education",
"summary": "AI that teaches rather than just does: tutor-style agents, learning loops for code generation, and My Weird Prompts, a podcast that explains a new topic every day."
}
],
"flagship": [
{
"title": "Hebrish (Sabich v1.0): Parakeet code-switching plus tech dictation",
"kind": "model",
"url": "https://huggingface.co/Hebrish/Sabich",
"description": "The recommended model in the Hebrish apps: Jachnun's code-switching data plus tech vocabulary, brands, numbers, Irish-English phrasing and my other task recordings, retrained from stock Parakeet v2. Level with Jachnun on code-switching (public benchmark term recall 73.8% against 73.3%), slightly better on tech (WER 1.54% against 1.74%). Also released on Qwen3-ASR 1.7B, Canary-Qwen 2.5B, Parakeet 110M and asr4all bases. Gated, access by request.",
"focus": [
"code-switching",
"asr",
"accessibility"
],
"page": "https://ai.danielrosehill.com/models/parakeet-tdt-v2-english-hebrew-all-subsets"
},
{
"title": "Hebrish (Jachnun v1.0): Parakeet English–Hebrew code-switching",
"kind": "model",
"url": "https://huggingface.co/Hebrish/Jachnun",
"description": "On-device English dictation that writes Hebrew words correctly, in romanised Latin letters. Parakeet TDT 0.6B v2 fine-tuned on my own recordings, used through the Hebrish apps on Android, Ubuntu and Windows. Code-switched WER 11.8% to 5.4% (v1.0), plain English no worse. Use it if you mix Hebrew into English; for plain English, see Boxty. Site: hebrish.app.",
"focus": [
"code-switching",
"asr",
"accessibility"
],
"page": "https://ai.danielrosehill.com/models/parakeet-english-hebrew-codeswitching"
},
{
"title": "Boxty v1.0: Parakeet English and tech dictation",
"kind": "model",
"url": "https://huggingface.co/Hebrish/Parakeet-TDT-0.6B-v2-Tech-And-Accent",
"description": "The sister model to Jachnun, for English dictation without Hebrew: tech vocabulary, brands, acronyms, numbers in digits and Irish-English phrasing, in my voice. Tech dictation WER 2.51% to 1.35%. It has none of the code-switching vocabulary, so use Jachnun if you mix Hebrew into English. Gated, access by request.",
"focus": [
"asr",
"accessibility"
],
"page": "https://ai.danielrosehill.com/models/parakeet-tdt-v2-tech-and-accent"
},
{
"title": "English–Hebrew code-switching ASR benchmark",
"kind": "space",
"url": "https://huggingface.co/spaces/danielrosehill/English-Hebrew-Codeswitching-ASR-Leaderboard",
"description": "A replicable benchmark and leaderboard: stock and fine-tuned speech-to-text scored on how many Hebrew words in English speech they get right. Hebrish v1.0 (Jachnun): 73%; Whisper Large V3: 46%; stock Parakeet: 33%.",
"focus": [
"code-switching",
"asr",
"evals"
]
},
{
"title": "Hebrish Keyboard",
"kind": "repo",
"url": "https://github.com/Hebrish-ASR/Hebrish-Keyboard",
"description": "A full Android keyboard with Hebrish dictation on the mic key: a fork of HeliBoard, so there is no switching to a separate voice keyboard. GPL-3.0, arm64 only, early release. On the Hebrish F-Droid repository.",
"focus": [
"code-switching",
"asr",
"accessibility"
]
},
{
"title": "ASR Dataset Blender",
"kind": "space",
"url": "https://huggingface.co/spaces/danielrosehill/ASR-Dataset-Blender",
"description": "Builds a training mix from Hugging Face speech datasets: filters, de-duplication, held-out sets that keep test terms out of training, and a share per source. Downloads NeMo manifests and a recipe pinned to each source's commit, so a blend can be rebuilt exactly. Source: github.com/Hebrish-ASR/ASR-Dataset-Blender.",
"focus": [
"asr",
"code-switching"
]
},
{
"title": "The PC Bench",
"kind": "repo",
"url": "https://github.com/danielrosehill/The-PC-Bench",
"description": "Does the model scold you? An eval for lecturing, moralising, terminology correction and unrequested disclaimers on legitimate requests, with mirrored Western-progressive and conservative triggers to measure selectivity. 54 open test prompts.",
"focus": [
"evals"
]
},
{
"title": "Propaganda Bait Eval",
"kind": "repo",
"url": "https://github.com/danielrosehill/Propaganda-Bait-Eval",
"description": "Loaded political premises across six series (CCP, MAGA, pro-Israel, pro-Palestine, Kremlin, US-progressive), scored on how readily a model repeats each framing.",
"focus": [
"evals"
]
},
{
"title": "Whisper Hebrish",
"kind": "model",
"url": "https://huggingface.co/danielrosehill/Whisper-Hebrish",
"description": "Whisper Large v3 Turbo fine-tuned for English speech with Hebrew words mixed in, the way English speakers in Israel talk.",
"focus": [
"code-switching",
"asr",
"accessibility"
]
},
{
"title": "Whisper Hebrish demo",
"kind": "space",
"url": "https://huggingface.co/spaces/danielrosehill/Whisper-Hebrish",
"description": "Try the Hebrish model in the browser and compare it with stock Whisper.",
"focus": [
"code-switching",
"asr"
]
},
{
"title": "Hebrew-in-Latin token classifier",
"kind": "model",
"url": "https://huggingface.co/Hebrish/Hebrew-Latin-Token-Classifier",
"description": "Released: finds Hebrew words written in Latin letters inside English text (\"the makolet\"), so TTS can send them to a Hebrew voice. XLM-RoBERTa base, span F1 0.94 on a term-disjoint test set. No model for this existed before. Part of the Hebrish series since 2026-10-20. Demo Space and a Replicate API.",
"focus": [
"code-switching",
"accessibility"
],
"page": "https://ai.danielrosehill.com/models/hebrew-latin-token-classifier"
},
{
"title": "Gemma 3 270M ASR Paragraph Breaks",
"kind": "model",
"url": "https://huggingface.co/danielrosehill/Gemma-3-270M-ASR-Paragraph-Breaks",
"description": "New (2026-10-20): adds paragraph breaks to flat speech-to-text output and changes nothing else, small enough to run on a phone after the ASR pass. Break F1 0.78 against 0.00 for the untrained model. ONNX export for ONNX Runtime.",
"focus": [
"asr"
],
"page": "https://ai.danielrosehill.com/models/gemma-3-270m-asr-paragraph-breaks"
},
{
"title": "Gemma 3 270M Email Key-Line Bolding",
"kind": "model",
"url": "https://huggingface.co/danielrosehill/Gemma-3-270M-Email-Key-Line-Bolding",
"description": "Preliminary (2026-10-21): puts the key parts of a dictated email (the requests, the purpose, the main points) in bold and changes nothing else. Runs after the paragraph-breaks model. Word F1 0.62 on 6 hand-annotated emails; the untrained model fell back on all of them. Demo Space.",
"focus": [
"asr"
],
"page": "https://ai.danielrosehill.com/models/gemma-3-270m-email-key-line-bolding"
},
{
"title": "Proximate Pseudonymizer 0.6B",
"kind": "model",
"url": "https://huggingface.co/danielrosehill/Proximate-Pseudonymizer-0.6B",
"description": "Proof of concept (2026-10-21): rewrites a text with identifying details swapped for close, plausible substitutes (Daniel to David, Jerusalem to Tel Aviv), used consistently. Not redaction. Qwen3-0.6B, Apache-2.0. It still leaves 9.3% of identifying details unchanged, so it is not safe to run unsupervised. Demo Space.",
"focus": [],
"page": "https://ai.danielrosehill.com/models/proximate-pseudonymizer-0.6b"
},
{
"title": "LLM Spoken Steganography",
"kind": "space",
"url": "https://huggingface.co/spaces/danielrosehill/LLM-Spoken-Steganography",
"description": "Proof of concept: an LLM hides an encrypted message in the first letters of a mundane podcast script's sentences, which survive text-to-speech, playback through the air and speech-to-text. Grew out of My Weird Prompts #6141. Two-phone demo in the browser.",
"focus": []
},
{
"title": "Pseudo-Personalised Spam Classifier",
"kind": "repo",
"url": "https://github.com/danielrosehill/Pseudo-Personalised-Spam-Classifier",
"description": "Planned: a small classifier that separates genuine email from strangers from mass outreach written to look personal, which gets past spam filters. Seed dataset of 18 pseudonymised emails on Hugging Face (2026-10-20); no model yet. The Pseudospam Filter plugin for Claude Code applies the same labelling standard today.",
"focus": []
},
{
"title": "Hebrew-in-Latin code-switching corpus",
"kind": "dataset",
"url": "https://huggingface.co/datasets/Hebrish/hebrew-latin-code-switching",
"description": "English sentences with Hebrew words written in Latin letters, labelled at token level (BIO): the training set for the classifier above. No comparable corpus existed when it was built.",
"focus": [
"code-switching",
"accessibility"
],
"page": "https://ai.danielrosehill.com/datasets/hebrew-latin-code-switching"
},
{
"title": "English–Hebrew Code-Switching Sentences",
"kind": "dataset",
"url": "https://huggingface.co/datasets/danielrosehill/English-Hebrew-Codeswitching-Sentences",
"description": "Single-speaker read-aloud audio of English sentences containing Hebrew words, for ASR fine-tuning.",
"focus": [
"code-switching",
"asr"
],
"page": "https://ai.danielrosehill.com/datasets/english-hebrew-codeswitching-sentences"
},
{
"title": "English–Hebrew Mixed Sentences",
"kind": "dataset",
"url": "https://huggingface.co/datasets/danielrosehill/English-Hebrew-Mixed-Sentences",
"description": "The training set behind Whisper Hebrish: English sentences with everyday Hebrew vocabulary.",
"focus": [
"code-switching",
"asr"
]
},
{
"title": "Read-Aloud Voice Sentences",
"kind": "dataset",
"url": "https://huggingface.co/datasets/danielrosehill/Read-Aloud-Voice-Sentences",
"description": "Read-aloud English and Hebrew sentences for fine-tuning Parakeet and other ASR models.",
"focus": [
"asr",
"code-switching"
],
"page": "https://ai.danielrosehill.com/datasets/read-aloud-voice-sentences"
},
{
"title": "Tech Sentences",
"kind": "dataset",
"url": "https://huggingface.co/datasets/danielrosehill/Tech-Sentences",
"description": "Developer and AI vocabulary read aloud, for ASR that has to get technical terms right.",
"focus": [
"asr"
]
},
{
"title": "Whisper Hebrish (large-v3-turbo)",
"kind": "collection",
"url": "https://huggingface.co/collections/danielrosehill/whisper-hebrish-691bc2624553319eb727c1db",
"description": "Where Hebrish started, in November 2025: a full fine-tune of Whisper large-v3-turbo for English with Hebrew words, with its demo Space and sentence data.",
"focus": [
"code-switching",
"asr"
]
},
{
"title": "Whisper Hebrish Small ACFT",
"kind": "model",
"url": "https://huggingface.co/Hebrish/Whisper-Hebrish-Small-ACFT",
"description": "A phone-sized Whisper small.en fine-tune with FUTO's ACFT, for FUTO Keyboard and FUTO Voice Input on Android. 5.8% code-switching WER on its own test set.",
"focus": [
"code-switching",
"asr",
"accessibility"
]
},
{
"title": "On-device ASR on the OnePlus Nord 3",
"kind": "collection",
"url": "https://huggingface.co/collections/danielrosehill/on-device-asr-one-plus-cph2493-nord-3",
"description": "ASR models I have tested running locally on an Android phone (OnePlus Nord 3, CPH2493).",
"focus": [
"asr",
"accessibility"
]
},
{
"title": "Android voice keyboard roundup",
"kind": "repo",
"url": "https://github.com/danielrosehill/Android-Voice-Keyboard-Roundup",
"description": "An evaluation matrix of open-source Android voice keyboards and voice IMEs.",
"focus": [
"asr",
"accessibility"
]
},
{
"title": "Noisy Voice Notes",
"kind": "dataset",
"url": "https://huggingface.co/datasets/danielrosehill/Noisy-Voice-Notes",
"description": "Real voice notes recorded in rough conditions, for testing ASR beyond studio audio.",
"focus": [
"asr"
]
},
{
"title": "ASR WPM & background noise eval",
"kind": "dataset",
"url": "https://huggingface.co/datasets/danielrosehill/ASR-WPM-And-Background-Noise-Eval",
"description": "Annotated recordings that measure how speaking rate and background noise affect ASR accuracy.",
"focus": [
"asr"
]
},
{
"title": "Transcription Cleanup Trainer",
"kind": "dataset",
"url": "https://huggingface.co/datasets/danielrosehill/Transcription-Cleanup-Trainer",
"description": "Pairs for fine-tuning a model to clean raw speech-to-text output into readable text.",
"focus": [
"asr",
"accessibility"
]
},
{
"title": "Code Tutor Agent",
"kind": "repo",
"url": "https://github.com/danielrosehill/Code-Tutor-Agent-Model",
"description": "An agent configuration that guides a learner through a problem instead of doing it for them.",
"focus": [
"education"
]
},
{
"title": "Vibe Code And Learn",
"kind": "repo",
"url": "https://github.com/danielrosehill/Vibe-Code-And-Learn",
"description": "A multi-agent code-generation setup with an educational loop built in.",
"focus": [
"education"
]
}
],
"model_cards": [
{
"slug": "parakeet-english-hebrew-codeswitching",
"title": "Hebrish (Jachnun v1.0): Parakeet TDT 0.6B v2 for English–Hebrew code-switching",
"short_title": "Jachnun (Parakeet English–Hebrew code-switching)",
"tagline": "Speech-to-text that gets the Hebrew words right when an English speaker in Israel drops them into English: \"pick up some bamba from the makolet\".",
"status": "v1.0 · Jachnun",
"released": "2026-10-19",
"license": "CC-BY-4.0",
"base_model": {
"label": "nvidia/parakeet-tdt-0.6b-v2",
"url": "https://huggingface.co/nvidia/parakeet-tdt-0.6b-v2"
},
"page": "https://ai.danielrosehill.com/models/parakeet-english-hebrew-codeswitching",
"hugging_face": "https://huggingface.co/Hebrish/Jachnun",
"website": "https://hebrish.app",
"sibling": {
"slug": "parakeet-tdt-v2-tech-and-accent",
"label": "Boxty",
"note": "Jachnun is for English with Hebrew words mixed in. For plain English and tech dictation with no Hebrew, use its sister model Boxty, which has none of the code-switching vocabulary.",
"page": "https://ai.danielrosehill.com/models/parakeet-tdt-v2-tech-and-accent"
},
"summary": [
"Hebrish is on-device English dictation for English speakers in Israel that writes the Hebrew words correctly, romanized: makolet, teudat zehut. It is a fine-tune of NVIDIA Parakeet TDT 0.6B v2, trained on my own recordings, and you use it through the Hebrish apps: an Android voice keyboard plus tray apps for Ubuntu and Windows. The project site is hebrish.app.",
"Stock Parakeet v2 is very good at English, but it has never seen romanized Hebrew. A Hebrew word in English speech comes out as whatever English words sound closest. The v2 tokenizer has no Hebrew characters, so this fine-tune teaches the model one romanized spelling for each Hebrew word.",
"It was trained on read-aloud sentences that I recorded on the Android phone that runs the model, plus some plain English so that it keeps its English. It runs fully offline on a phone or a Linux or Windows desktop.",
"This is v1.0 of the fine-tune, named Jachnun (2026-10-19), trained from stock Parakeet v2 on 339 minutes of code-switched speech. The five checkpoints published before it as v1 to v5 are now called PoC-1 to PoC-5. The code-switching line takes Israeli food names.",
"Since 2026-10-21 the Hebrish apps recommend Sabich v1.0, trained on the same code-switching data plus tech vocabulary and the other task recordings. It is level with Jachnun on code-switching and slightly better on tech terms; Jachnun stays in the apps as the pure code-switching model. See Sabich's own card page.",
"I chose v2 over Parakeet v3 on purpose. In my own dictation v2 was more accurate for English; v3 adds 25 languages and gives up some English accuracy."
],
"results": {
"as_of": "2026-10-20",
"run": "r6-en30",
"version": "v1.0 · Jachnun",
"base_label": "Stock",
"tuned_label": "Fine-tune",
"rows": [
{
"test": "Code-switching, shared test set, 422 clips (normalised WER)",
"base": "11.82%",
"tuned": "5.42%",
"better": true
},
{
"test": "Hebrew terms held out of training: share correct (324 occurrences, 134 terms)",
"base": "23.8%",
"tuned": "46.9%",
"better": true
},
{
"test": "Speaker's own plain English, 84 clips (normalised WER)",
"base": "2.81%",
"tuned": "1.56%",
"better": true
},
{
"test": "LibriSpeech test-clean, 200 utterances (normalised WER)",
"base": "1.23%",
"tuned": "1.20%"
},
{
"test": "Long form, 17 files (normalised WER)",
"base": "10.34%",
"tuned": "5.84%",
"better": true
}
],
"notes": [
"Scored on the shared test set of 2026-10-20 (dataset revision f956dbb184c2, eval tag r7-r6-en30), the same clips used for Boxty, so the two models compare directly. Stock is nvidia/parakeet-tdt-0.6b-v2 on the same clips (tag r7-baseline).",
"WER is normalised: lowercase, punctuation removed. None of these clips were trained on or used to pick the checkpoint.",
"LibriSpeech is within noise of stock, so general English was not lost.",
"The checkpoints published as v1 to v5 are now PoC-1 to PoC-5; v1.0 is the first release scored on this test set.",
"The charts below are v1.0's earlier scoring (2026-10-19): the held-out split at dataset revision 550bfcaa0f84 and the public benchmark (test set v1, 84 clips no model trained on). The figures differ slightly from the table because the test sets differ."
]
},
"downloads": [
{
"label": "Hebrish APK",
"size": "14.5 MB + 660 MB model",
"use": "The Hebrish Android voice keyboard (latest version). Downloads the model once, then runs fully offline. arm64 phones, Android 9+.",
"url": "https://cdn.hebrish.app/apps/android/Hebrish-latest.apk"
},
{
"label": "GGUF Q8_0",
"size": "730 MB",
"use": "Android phone or Linux desktop. Take this one if unsure.",
"url": "https://huggingface.co/danielrosehill/Parakeet-TDT-0.6B-v2-English-Hebrew-Codeswitching/resolve/main/gguf/Parakeet-TDT-0.6B-v2-English-Hebrew-Codeswitching-Q8_0.gguf"
},
{
"label": "GGUF Q4_K_M",
"size": "475 MB",
"use": "Phones with 4 GB of RAM or less, or if Q8_0 is too slow.",
"url": "https://huggingface.co/danielrosehill/Parakeet-TDT-0.6B-v2-English-Hebrew-Codeswitching/resolve/main/gguf/Parakeet-TDT-0.6B-v2-English-Hebrew-Codeswitching-Q4_K_M.gguf"
},
{
"label": ".nemo checkpoint",
"size": "2.47 GB",
"use": "Python, GPU inference or further fine-tuning, at full precision.",
"url": "https://huggingface.co/danielrosehill/Parakeet-TDT-0.6B-v2-English-Hebrew-Codeswitching/resolve/main/Parakeet-TDT-0.6B-v2-English-Hebrew-Codeswitching.nemo"
}
],
"app": {
"name": "Hebrish",
"repo": "https://github.com/Hebrish-ASR/Hebrish-App",
"builds": [
{
"platform": "Linux (Debian / Ubuntu)",
"url": "https://cdn.hebrish.app/apps/ubuntu/hebrish-latest-amd64.deb"
},
{
"platform": "Android",
"url": "https://cdn.hebrish.app/apps/android/Hebrish-latest.apk"
},
{
"platform": "Windows",
"url": "https://cdn.hebrish.app/apps/windows/Hebrish-latest-windows-x64.zip"
}
]
},
"datasets": [
{
"label": "English–Hebrew Code-Switching Sentences",
"url": "https://huggingface.co/datasets/danielrosehill/English-Hebrew-Codeswitching-Sentences",
"page": "https://ai.danielrosehill.com/datasets/english-hebrew-codeswitching-sentences"
},
{
"label": "Read-Aloud Voice Sentences",
"url": "https://huggingface.co/datasets/danielrosehill/Read-Aloud-Voice-Sentences"
},
{
"label": "LibriSpeech (replay)",
"url": "https://huggingface.co/datasets/openslr/librispeech_asr"
}
],
"limitations": [
"Trained on one voice so far: an English speaker with an Irish accent living in Jerusalem. Recordings from more speakers are the next step; contribute yours at hebrish.app/contact?topic=samples.",
"About 5.7 hours of code-switched audio, with most Hebrew terms recorded only once or twice. Terms the model never heard in training are right only about half the time (46.9% for v1.0), and more audio has not moved that much since PoC-1.",
"Spellings follow one convention (the Hebrew-Latin-Token-Classifier lexicon), so the model may prefer it over other romanizations.",
"On-device accuracy and phone speed are not measured yet beyond a five-clip GGUF check."
],
"checkpoints": [
{
"label": "Baseline",
"state": "done",
"date": "2026-10-15",
"note": "Stock v2 got 28% of Hebrew term occurrences right across 615 terms."
},
{
"label": "PoC-1 pilot (published as v1, pilot-en30)",
"state": "done",
"date": "2026-10-15",
"note": "First release: 77 min of code-switching, 30% English. Superseded by PoC-2; kept in the Hugging Face repo history."
},
{
"label": "GGUF export",
"state": "done",
"date": "2026-10-15",
"note": "transcribe.cpp v0.1.3; identical transcripts to PyTorch on 5 test clips."
},
{
"label": "PoC-2 retrain (published as v2, r2-en30)",
"state": "done",
"date": "2026-10-15",
"note": "Released: 120 min of code-switching, 28% English, trained from stock v2. Same files and download links; .nemo plus GGUF Q8_0 and Q4_K_M."
},
{
"label": "ONNX and sherpa-onnx export",
"state": "done",
"date": "2026-10-16",
"note": "Separate -ONNX and -sherpa-onnx-int8 repos, for Voxtype, onnx-asr and sherpa-onnx apps."
},
{
"label": "PoC-3 retrain (published as v3, r3-en30)",
"state": "done",
"date": "2026-10-18",
"note": "Released: 196 min of code-switching, 27% English, trained from stock v2. Every version re-scored on a new 219-clip test split. Same download links; PoC-1 and PoC-2 pinned in the model card."
},
{
"label": "Codeswitch Voice APK",
"state": "done",
"date": "2026-10-18",
"note": "v0.1.0 of my own Android voice keyboard (proof of concept): sherpa-onnx int8 PoC-3 weights, on-device, companion to HeliBoard."
},
{
"label": "Desktop dictation app",
"state": "done",
"date": "2026-10-18",
"note": "v0.2.0 of the desktop app (Linux .deb, untested Windows build): hotkey dictation plus a Transcribe tab for recordings and audio files. The Android app moved into the same repo as v0.5.0."
},
{
"label": "PoC-4 retrain (published as v4, r4-en30)",
"state": "done",
"date": "2026-10-18",
"note": "Released before its benchmark: 269 min of code-switching, the first run on the 70/20/10 mix target (70.8% / 19.1% / 10.1%)."
},
{
"label": "PoC-5 retrain (published as v5, r5-en30)",
"state": "done",
"date": "2026-10-18",
"note": "Released before its benchmark: 294 min of code-switching, 70/20/10 mix, trained early ahead of the 360-min checkpoint. .nemo, GGUF (Q8_0, Q4_K_M, F16), ONNX and sherpa-onnx refreshed; the apps pick it up on their own. MLX stays PoC-3."
},
{
"label": "Nemotron streaming fine-tune",
"state": "done",
"date": "2026-10-18",
"note": "Experimental second base model: the PoC-5 data and recipe on NVIDIA nemotron-speech-streaming-en-0.6b, for live streaming text. Later scored offline: 5.8% code-switching WER on the held-out split, against 5.1% for v1.0."
},
{
"label": "Parlance",
"state": "done",
"date": "2026-10-19",
"note": "The Android and desktop apps became Parlance (Android v0.1.0, desktop v0.1.0), with a choice of stock Parakeet or this fine-tune."
},
{
"label": "Hebrish apps 0.2.0",
"state": "done",
"date": "2026-10-19",
"note": "The apps were renamed Hebrish (repo now Hebrish-ASR/Hebrish-App): Android APK, Ubuntu .deb and Windows zip, desktop hotkey F14. Parlance continues separately as a general-purpose Parakeet dictation app."
},
{
"label": "v1.0 · Jachnun (r6-en30)",
"state": "done",
"date": "2026-10-19",
"note": "The first release after five proof-of-concept checkpoints: 339 min of code-switching, 30% English, trained from stock v2. The checkpoints published as v1 to v5 became PoC-1 to PoC-5. MLX stays PoC-3."
},
{
"label": "Hebrish apps: Android 0.5.0, desktop 0.2.2",
"state": "done",
"date": "2026-10-20",
"note": "Current app releases, linked in the downloads above. The repo was renamed Hebrish-ASR/Hebrish-App."
},
{
"label": "Boxty v1.0",
"state": "done",
"date": "2026-10-20",
"note": "A sister fine-tune from the same base for English dictation without Hebrew: tech vocabulary and accent. See its own card page."
},
{
"label": "Sabich v1.0",
"state": "done",
"date": "2026-10-20",
"note": "A second Hebrish model: Jachnun's data plus tech vocabulary and the other task recordings, retrained from stock v2. Level with Jachnun on code-switching; in the apps' model list. See its own card page."
},
{
"label": "Data collection",
"state": "in-progress",
"note": "Recording continues, weighted toward terms stock v2 never gets right and terms with only one or two takes."
},
{
"label": "v1.1",
"state": "planned",
"note": "The next retrain on more code-switching audio. A comparison of 70/30, 80/20, 90/10 and 100/0 code-switching / English remains an option."
},
{
"label": "On-phone test",
"state": "planned",
"note": "Measure accuracy and speed of the on-device builds on the target phone."
}
],
"tags": [
"Hebrish",
"ASR",
"Parakeet",
"NeMo",
"code-switching",
"Hebrew",
"English",
"on-device",
"GGUF",
"Android"
]
},
{
"slug": "parakeet-tdt-v2-english-hebrew-all-subsets",
"title": "Hebrish (Sabich v1.0): Parakeet TDT 0.6B v2 for code-switching plus tech dictation",
"short_title": "Sabich (Parakeet code-switching plus tech)",
"tagline": "The recommended Hebrish model: everything Jachnun was trained on, plus tech vocabulary, brand names, acronyms, numbers and currency, Irish-English phrasing and the rest of my task recordings.",
"status": "v1.0 · Sabich",
"released": "2026-10-20",
"license": "CC-BY-4.0",
"base_model": {
"label": "nvidia/parakeet-tdt-0.6b-v2",
"url": "https://huggingface.co/nvidia/parakeet-tdt-0.6b-v2"
},
"page": "https://ai.danielrosehill.com/models/parakeet-tdt-v2-english-hebrew-all-subsets",
"hugging_face": "https://huggingface.co/Hebrish/Sabich",
"website": "https://hebrish.app/models/sabich",
"gated": "The model repos (.nemo, GGUF, ONNX, MLX and the sherpa-onnx build the apps use) are gated: request access on Hugging Face. The apps download it for you.",
"sibling": {
"slug": "parakeet-english-hebrew-codeswitching",
"label": "Jachnun",
"note": "Sabich and Jachnun are level on code-switching; Sabich is slightly better on tech vocabulary, which is why it is the recommended pick. Both run in the Hebrish apps, so switch in the model list if Jachnun suits you better.",
"page": "https://ai.danielrosehill.com/models/parakeet-english-hebrew-codeswitching"
},
"summary": [
"Sabich is the second Hebrish model: a fine-tune of NVIDIA Parakeet TDT 0.6B v2 trained on Jachnun's data plus everything else I have recorded. That means the code-switched speech, plus tech vocabulary, brand names, acronyms, numbers and currency, Irish-English phrasing, disfluencies and self-corrections.",
"It is not trained on top of Jachnun. Every model in the family is retrained from stock Parakeet v2 on the cumulative data: PoC-1 to PoC-5, then Jachnun v1.0, then Sabich v1.0. Boxty, a separate fine-tune for English without Hebrew (not part of Hebrish), starts from the same base.",
"The question was whether adding all the other data would cost code-switching accuracy. It did not. On the public benchmark Sabich and Jachnun are level within noise, and Sabich is slightly better on tech vocabulary. That settled shipping both models in the Hebrish apps.",
"Sabich is named after the Israeli pita with a bit of everything in it. The code-switching line uses Israeli food names."
],
"results": {
"as_of": "2026-10-20",
"run": "r7-all",
"version": "v1.0 · Sabich",
"base_label": "Stock",
"tuned_label": "Fine-tune",
"rows": [
{
"test": "Code-switching, held-out split (normalised WER)",
"base": "11.82%",
"tuned": "5.30%",
"better": true
},
{
"test": "Hebrew terms held out of training: share correct",
"base": "23.8%",
"tuned": "48.1%",
"better": true
},
{
"test": "Tech dictation (normalised WER)",
"base": "2.51%",
"tuned": "1.54%",
"better": true
},
{
"test": "LibriSpeech (normalised WER)",
"base": "1.23%",
"tuned": "1.30%"
}
],
"notes": [
"Against Jachnun v1.0 on the same held-out split (same dataset revision): code-switching WER 5.30% against 5.42%; held-out terms 48.1% against 46.9%; tech WER 1.54% against 1.74%; LibriSpeech 1.30% against 1.20%.",
"Public benchmark (84 clean clips, round 4), Sabich against Jachnun: Hebrew term recall 73.8% against 73.3%; unseen-term recall 68.4% against 70.1%; WER 3.27% against 3.32%; English control WER 1.99% against 2.17%.",
"Read: level on code-switching (the differences are noise), a modest gain on tech vocabulary, and no loss of code-switching accuracy from the extra data.",
"WER is normalised: lowercase, punctuation removed."
]
},
"downloads": [],
"app": {
"name": "Hebrish",
"repo": "https://github.com/Hebrish-ASR/Hebrish-App",
"builds": [
{
"platform": "Linux (Debian / Ubuntu)",
"url": "https://cdn.hebrish.app/apps/ubuntu/hebrish-latest-amd64.deb"
},
{
"platform": "Android",
"url": "https://cdn.hebrish.app/apps/android/Hebrish-latest.apk"
},
{
"platform": "Windows",
"url": "https://cdn.hebrish.app/apps/windows/Hebrish-latest-windows-x64.zip"
}
]
},
"datasets": [
{
"label": "English–Hebrew Code-Switching Sentences",
"url": "https://huggingface.co/datasets/danielrosehill/English-Hebrew-Codeswitching-Sentences",
"page": "https://ai.danielrosehill.com/datasets/english-hebrew-codeswitching-sentences"
},
{
"label": "All training datasets (collection)",
"url": "https://huggingface.co/collections/Hebrish/parakeet-v2-fine-tune-training-data-6ad63c739d300685a62edb3e"
},
{
"label": "LibriSpeech (replay)",
"url": "https://huggingface.co/datasets/openslr/librispeech_asr"
}
],
"limitations": [
"Trained on one voice so far: an English speaker with an Irish accent living in Jerusalem. Recordings from more speakers are the next step; contribute yours at hebrish.app/contact?topic=samples.",
"On code-switching it is level with Jachnun, not better: Hebrew terms the model never heard in training are still right only about half the time.",
"The model repos are gated, with access by request."
],
"checkpoints": [
{
"label": "v1.0 · Sabich (r7-all)",
"state": "done",
"date": "2026-10-20",
"note": "First release: Jachnun's data plus every other task recording, 560.5 min of own recordings plus 62.5 min replay, trained from stock v2. Added to the Hebrish apps' model list."
},
{
"label": "Recommended in the apps",
"state": "done",
"date": "2026-10-21",
"note": "Sabich became the recommended pick in the Hebrish apps, with Jachnun kept alongside it."
},
{
"label": "Sabich on other bases",
"state": "done",
"date": "2026-10-21",
"note": "The same training data on four more bases: Qwen3-ASR 1.7B (5.4% code-switching WER, stock 10.0%) and Canary-Qwen 2.5B (6.9%) for desktop GPUs, Parakeet TDT-CTC 110M for small devices, and asr4all S and M (10.1% and 9.1%) for wearables. Not in the apps yet. Shakshuka v1.0, the streaming line, is trained on it too."
}
],
"tags": [
"Hebrish",
"Sabich",
"ASR",
"Parakeet",
"NeMo",
"code-switching",
"Hebrew",
"English",
"tech vocabulary",
"on-device"
]
},
{
"slug": "parakeet-tdt-v2-tech-and-accent",
"title": "Boxty v1.0: Parakeet TDT 0.6B v2 for English and tech dictation",
"short_title": "Boxty (Parakeet English and tech dictation)",
"tagline": "English dictation without Hebrew, in my voice: model and product names, CLI and cloud terms, brands, acronyms, and numbers and currency in digits.",
"status": "v1.0 · Boxty",
"released": "2026-10-20",
"license": "CC-BY-4.0",
"base_model": {
"label": "nvidia/parakeet-tdt-0.6b-v2",
"url": "https://huggingface.co/nvidia/parakeet-tdt-0.6b-v2"
},
"page": "https://ai.danielrosehill.com/models/parakeet-tdt-v2-tech-and-accent",
"hugging_face": "https://huggingface.co/Hebrish/Parakeet-TDT-0.6B-v2-Tech-And-Accent",
"gated": "The model repo is gated: request access on Hugging Face.",
"sibling": {
"slug": "parakeet-english-hebrew-codeswitching",
"label": "Jachnun",
"note": "Boxty has none of the code-switching vocabulary. Use Jachnun if you mix Hebrew words into English; use Boxty for plain English and tech dictation.",
"page": "https://ai.danielrosehill.com/models/parakeet-english-hebrew-codeswitching"
},
"summary": [
"Boxty is a fine-tune of NVIDIA Parakeet TDT 0.6B v2 for English dictation with no Hebrew in it. It targets tech vocabulary (model and product names, CLI and cloud terms), brand names, acronyms, numbers and currency written in digits, homophones, Irish-English phrasing, and dictated emails and messages, all in my voice.",
"It is the sister model to Jachnun, the English–Hebrew code-switching fine-tune from the same base. Boxty has none of the code-switching vocabulary, so code-switched speech stays at stock level on purpose. If you mix Hebrew into your English, use Jachnun.",
"Boxty is named after the Irish potato pancake. The no-Hebrew line uses Irish food names; the code-switching line uses Israeli ones.",
"You can use it through Parlance, my general-purpose on-device dictation app for Android and Linux, whose model list includes Boxty."
],
"results": {
"as_of": "2026-10-20",
"run": "r7-general",
"version": "v1.0 · Boxty",
"base_label": "Stock",
"tuned_label": "Fine-tune",
"rows": [
{
"test": "Hebrew-free English (normalised WER)",
"base": "1.79%",
"tuned": "1.04%",
"better": true
},
{
"test": "Tech dictation (normalised WER)",
"base": "2.51%",
"tuned": "1.35%",
"better": true
},
{
"test": "LibriSpeech (normalised WER)",
"base": "1.23%",
"tuned": "1.18%"
},
{
"test": "Code-switched speech (normalised WER)",
"base": "11.5%",
"tuned": "stock level, on purpose"
}
],
"notes": [
"Against Jachnun on the same tests: Hebrew-free English 1.04% against 1.23%; tech dictation 1.35% against 1.74%.",
"Code-switched speech stays at stock level (11.5%) because the training data has no Hebrew in it."
]
},
"downloads": [
{
"label": "GGUF Q8_0",
"size": "730 MB",
"use": "Android phone or Linux desktop. Take this one if unsure. Needs Hugging Face access to the gated repo.",
"url": "https://huggingface.co/Hebrish/Parakeet-TDT-0.6B-v2-Tech-And-Accent/resolve/main/gguf/Parakeet-TDT-0.6B-v2-Tech-And-Accent-Q8_0.gguf"
},
{
"label": "GGUF Q4_K_M",
"size": "475 MB",
"use": "Phones with 4 GB of RAM or less, or if Q8_0 is too slow.",
"url": "https://huggingface.co/Hebrish/Parakeet-TDT-0.6B-v2-Tech-And-Accent/resolve/main/gguf/Parakeet-TDT-0.6B-v2-Tech-And-Accent-Q4_K_M.gguf"
},
{
"label": ".nemo checkpoint",
"size": "2.47 GB",
"use": "Python, GPU inference or further fine-tuning, at full precision.",
"url": "https://huggingface.co/Hebrish/Parakeet-TDT-0.6B-v2-Tech-And-Accent/resolve/main/Parakeet-TDT-0.6B-v2-Tech-And-Accent.nemo"
}
],
"datasets": [
{
"label": "Parakeet V2 Blend: No Hebrew",
"url": "https://huggingface.co/datasets/Hebrish/Parakeet-V2-Blend-No-Hebrew"
},
{
"label": "LibriSpeech (replay)",
"url": "https://huggingface.co/datasets/openslr/librispeech_asr"
}
],
"limitations": [
"One speaker: an English speaker with an Irish accent living in Jerusalem. Gains on other voices are untested.",
"No code-switching vocabulary: Hebrew words in English speech come out at stock level. Use Jachnun for that.",
"The model repo is gated, with access by request."
],
"checkpoints": [
{
"label": "v1.0 · Boxty",
"state": "done",
"date": "2026-10-20",
"note": "First release: 142.5 min of Hebrew-free recordings plus 15.9 min of LibriSpeech replay."
}
],
"tags": [
"Boxty",
"ASR",
"Parakeet",
"NeMo",
"English",
"tech vocabulary",
"dictation",
"Irish English"
]
},
{
"slug": "hebrew-latin-token-classifier",
"title": "Hebrew-Latin Token Classifier: finds Hebrew written in Latin letters",
"short_title": "Hebrew-Latin Token Classifier",
"tagline": "Finds Hebrew words written in Latin letters inside English text, so text-to-speech can hand them to a Hebrew voice instead of reading makolet as English. There was no model for this before.",
"status": "v1 · released",
"released": "2026-10-15",
"license": "MIT",
"base_model": {
"label": "FacebookAI/xlm-roberta-base",
"url": "https://huggingface.co/FacebookAI/xlm-roberta-base"
},
"page": "https://ai.danielrosehill.com/models/hebrew-latin-token-classifier",
"hugging_face": "https://huggingface.co/Hebrish/Hebrew-Latin-Token-Classifier",
"website": "https://hebrish.app/models/token-classifier",
"summary": [
"The model marks every span of romanized Hebrew in an English sentence, with character offsets and a confidence score. Multi-word terms come out as one span (Bituach Leumi, kupat holim). Labels: O, B-HE, I-HE.",
"It exists because nothing else does this. GlotLID, lid.176 and lid218e have no heb_Latn label, and a language detector sees Latin script inside an English sentence and calls it English. No token-level code-switching model covered Hebrew.",
"It was built for text-to-speech. English TTS reads makolet as \"MACK-oh-let\"; the fix is to find the Hebrew spans, convert them to Hebrew script and send each to a Hebrew voice. This model is pass 1 of that pipeline (detect, transliterate, add niqqud, tag, speak). The first user is My Weird Prompts, an AI-generated podcast whose episodes about life in Israel are full of these words.",
"The labelling test is pronunciation, not etymology. Words English already says acceptably (Shabbat, hummus, challah, kosher) are deliberately not flagged."
],
"results": {
"as_of": "2026-10-15",
"run": "xlm-roberta-base, 8 epochs",
"version": "v1",
"base_label": "Span precision / recall",
"tuned_label": "Span F1",
"metric_note": "Scored on the term-disjoint test split: no Hebrew term in the test set appears in training, so this measures how well it generalises to words it has never seen. Higher is better.",
"rows": [
{
"test": "All test sentences (468)",
"base": "0.928 / 0.953",
"tuned": "0.941",
"better": true
},
{
"test": "Generated (430)",
"base": "0.928 / 0.948",
"tuned": "0.938"
},
{
"test": "Dictated, human-chosen (38)",
"base": "0.929 / 1.000",
"tuned": "0.963",
"better": true
},
{
"test": "Validation, used for model selection (467)",
"base": "0.950 / 0.948",
"tuned": "0.949"
}
],
"notes": [
"Hard negatives: 0 false positives in 58 sentences built to trap it (substrings, English homographs such as dud, tor and salon, lookalikes, anglicised loanwords).",
"Of the 41 test sentences it gets wrong, 14 are gaps in the gold labels rather than model errors: Hebrew words the blind annotator refused and no human has yet re-reviewed.",
"For comparison, the closest published recipe (EnTaCs, English–Tamil, arXiv 2603.26587) reports F1 0.638 on its romanized class."
]
},
"downloads": [],
"datasets": [
{
"label": "Hebrish/hebrew-latin-code-switching (4,737 sentences)",
"url": "https://huggingface.co/datasets/Hebrish/hebrew-latin-code-switching"
}
],
"limitations": [
"English text with romanized Hebrew only. Hebrew script is not the target.",
"The training data is largely synthetic; the dictated slice is the closest thing to real input.",
"Under the default, conservative posture it does not flag place names (Dizengoff, Ramat Gan) or words English already says. That line is a design choice, not a fact.",
"Take each word's label from its first sub-token. With the stock pipeline use aggregation_strategy=\"first\"; \"simple\" cuts words short."
],
"checkpoints": [
{
"label": "v1",
"state": "done",
"date": "2026-10-15",
"note": "First release as danielrosehill/hebrew-latin-token-classifier, DOI 10.57967/hf/10997. Demo Space and Replicate API."
},
{
"label": "Moved to the Hebrish organisation",
"state": "done",
"date": "2026-10-20",
"note": "Same weights, copied to Hebrish/Hebrew-Latin-Token-Classifier (DOI 10.57967/hf/11108). New versions are published there."
},
{
"label": "Audio sibling",
"state": "in-progress",
"note": "Word-level English/Hebrew language ID in code-switched speech (Hebrish-ASR/Hebrew-English-Word-LID-Audio)."
},
{
"label": "Third label",
"state": "planned",
"note": "A possible later version: a label for \"Hebrew that English already says\", so the user chooses where the line falls."
}
],
"tags": [
"Hebrish",
"token classification",
"code-switching",
"Hebrew",
"romanization",
"text-to-speech",
"XLM-RoBERTa"
]
},
{
"slug": "gemma-3-270m-asr-paragraph-breaks",
"title": "Gemma 3 270M ASR Paragraph Breaks: paragraphs for dictated text",
"short_title": "Gemma 3 270M paragraph breaks",
"tagline": "Adds paragraph breaks to flat speech-to-text output and changes nothing else. Small enough to run on a phone after the speech model.",
"status": "r2 · first release",
"released": "2026-10-20",
"license": "Gemma Terms of Use",
"base_model": {
"label": "google/gemma-3-270m-it",
"url": "https://huggingface.co/google/gemma-3-270m-it"
},
"page": "https://ai.danielrosehill.com/models/gemma-3-270m-asr-paragraph-breaks",
"hugging_face": "https://huggingface.co/danielrosehill/Gemma-3-270M-ASR-Paragraph-Breaks",
"summary": [
"Speech recognition, such as a Parakeet fine-tune, returns punctuated, cased text as one block. This model returns the same text with paragraphs separated by blank lines. It is a full fine-tune of Gemma 3 270M, the smallest Gemma, so it can run on-device after the ASR pass.",
"It must not change a word. Because it is generative it retypes the whole text, so every call is checked: if anything other than whitespace changed, the input is used instead. On the test set that fallback fired on 2 of 26 memos.",
"The paragraphing style is mine: short paragraphs, with the opening line and each sign-off line on their own."
],
"results": {
"as_of": "2026-10-20",
"run": "asr-paragraph-r2-gemma270m",
"version": "r2",
"base_label": "Untrained Gemma 3 270M",
"tuned_label": "Fine-tune r2",
"metric_note": "Scored on 26 held-out synthetic voice memos whose paragraph breaks were reviewed by hand, as break-position precision and recall. Higher is better. A fallback is an output that changed a non-whitespace character, so the input is returned unchanged.",
"rows": [
{
"test": "Break precision",
"base": "0.00",
"tuned": "0.84",
"better": true
},
{
"test": "Break recall",
"base": "0.00",
"tuned": "0.72",
"better": true
},
{
"test": "F1",
"base": "0.00",
"tuned": "0.78",
"better": true
},
{
"test": "Fallbacks (fewer is better)",
"base": "26 of 26",
"tuned": "2 of 26",
"better": true
}
],
"notes": [
"A longer run (r3: 804 pairs, 6 epochs) scored the same F1 with 1 fallback and was not published.",
"The rule-based labeller that made most of the training data starts from an already-paragraphed draft, so it is a ceiling in the chart, not a like-for-like baseline."
]
},
"downloads": [
{
"label": "safetensors (Transformers)",
"size": "536 MB",
"use": "Python on CPU or GPU, at full precision. Use the prompt and fallback check from the model card.",
"url": "https://huggingface.co/danielrosehill/Gemma-3-270M-ASR-Paragraph-Breaks/resolve/main/model.safetensors"
},
{
"label": "ONNX fp32",
"size": "1.66 GB",
"use": "ONNX Runtime, on the same runtime as an ONNX speech model. Matched the PyTorch output on 6 of 6 test memos.",
"url": "https://huggingface.co/danielrosehill/Gemma-3-270M-ASR-Paragraph-Breaks-ONNX/resolve/main/onnx/model.onnx"
},
{
"label": "ONNX int8",
"size": "417 MB",
"use": "Experimental: about twice as fast on CPU, but matched PyTorch on only 2 of 6 memos and changed the text in one.",
"url": "https://huggingface.co/danielrosehill/Gemma-3-270M-ASR-Paragraph-Breaks-ONNX/resolve/main/onnx/model_quantized.onnx"
}
],
"datasets": [
{
"label": "danielrosehill/ASR-Paragraph-Breaks",
"url": "https://huggingface.co/datasets/danielrosehill/ASR-Paragraph-Breaks"
}
],
"limitations": [
"English only, and trained on synthetic memos rather than real ASR output.",
"Memos longer than about 700 words were not in training: split longer transcripts first.",
"It retypes the whole text, so keep the fallback check. Generation on a CPU is slow for long memos."
],
"checkpoints": [
{
"label": "r2",
"state": "done",
"date": "2026-10-20",
"note": "First release: 505 pairs, 3 epochs. F1 0.78."
},
{
"label": "ONNX export",
"state": "done",
"date": "2026-10-20",
"note": "fp32 and int8 for ONNX Runtime, checked against the PyTorch model."
}
],
"tags": [
"Gemma",
"ASR post-processing",
"paragraph segmentation",
"dictation",
"voice notes",
"on-device"
]
},
{
"slug": "gemma-3-270m-email-key-line-bolding",
"title": "Gemma 3 270M Email Key-Line Bolding: bold the key lines of a dictated email",
"short_title": "Gemma 3 270M email key-line bolding",
"tagline": "Puts the key parts of a dictated email in Markdown bold and changes nothing else. Runs after the paragraph-breaks model.",
"status": "r2 · preliminary",
"released": "2026-10-21",
"license": "Gemma Terms of Use",
"base_model": {
"label": "google/gemma-3-270m-it",
"url": "https://huggingface.co/google/gemma-3-270m-it"
},
"page": "https://ai.danielrosehill.com/models/gemma-3-270m-email-key-line-bolding",
"hugging_face": "https://huggingface.co/danielrosehill/Gemma-3-270M-Email-Key-Line-Bolding",
"summary": [
"The input is an email that has already been paragraphed. The output is the same text with ** around the requests, the purpose, the main points and the key facts, so a reader who skims only the bold knows what is being asked.",
"It must not change anything else. Every call is checked: if anything other than ** was added, the input is returned unbolded.",
"Uses. In a dictation pipeline, it bolds the key lines of a dictated email after paragraphing. For outbound mail, an AI agent drafting an email can bold the details so the recipient sees them at a glance; it pairs with the paragraph-breaks model. For inbound mail, it can annotate received emails, and a mail client could show the spans as highlights instead of bold.",
"This is a first, preliminary version. The bolding style is mine, and most of the training labels were extrapolated from my annotations by an LLM."
],
"results": {
"as_of": "2026-10-21",
"run": "email-bold-r2-gemma270m-e10",
"version": "r2",
"base_label": "Untrained Gemma 3 270M",
"tuned_label": "Fine-tune r2",
"metric_note": "Scored on 6 held-out emails bolded by hand, per word: a word counts as bold if it falls inside a span. Higher is better. A fallback is an output that changed anything other than adding **, so the input is returned unbolded.",
"rows": [
{
"test": "Word precision",
"base": "-",
"tuned": "0.73",
"better": true
},
{
"test": "Word recall",
"base": "-",
"tuned": "0.54",
"better": true
},
{
"test": "Word F1",
"base": "-",
"tuned": "0.62",
"better": true
},
{
"test": "Fallbacks (fewer is better)",
"base": "6 of 6",
"tuned": "0 of 6",
"better": true
}
],
"notes": [
"The untrained model fell back on every email, so it bolded nothing.",
"Reference: Claude Sonnet 5.5, prompted with the written rules, scores word F1 0.69 (4-fold cross-validation over the 19 hand-annotated emails).",
"The model is precise but under-bolds. It tends to bold the ask and the purpose line and miss some of the listed points.",
"Six test emails is a small test set. Treat the numbers as rough."
]
},
"downloads": [
{
"label": "safetensors (Transformers)",
"size": "536 MB",
"use": "Python on CPU or GPU, at full precision. Use the system prompt and validation check from the model card.",
"url": "https://huggingface.co/danielrosehill/Gemma-3-270M-Email-Key-Line-Bolding/resolve/main/model.safetensors"
}
],
"datasets": [
{
"label": "danielrosehill/Email-Key-Line-Bolding",
"url": "https://huggingface.co/datasets/danielrosehill/Email-Key-Line-Bolding"
}
],
"limitations": [
"Bolding is subjective, and the labels reflect one person's preference.",
"Mostly silver labels, so the model learns the LLM labeller's habits as well as mine.",
"English only, and trained on synthetic dictated emails. Not tested on real transcripts or on text other than email."
],
"checkpoints": [
{
"label": "r2",
"state": "done",
"date": "2026-10-21",
"note": "Preliminary release: 336 emails, 10 epochs. Word F1 0.62."
},
{
"label": "More hand annotations",
"state": "in-progress",
"note": "Annotation continues, towards about 200 hand-bolded emails, to replace the preliminary model."
}
],
"tags": [
"Gemma",
"dictation",
"email",
"text emphasis",
"on-device"
]
},
{
"slug": "proximate-pseudonymizer-0.6b",
"title": "Proximate Pseudonymizer 0.6B: swap identifying details for close, consistent substitutes",
"short_title": "Proximate Pseudonymizer 0.6B",
"tagline": "Rewrites a text with its names, places, organisations, dates and contacts swapped for close, plausible substitutes, the same one every time. Proof of concept: it still leaves 9.3% of identifying details unchanged.",
"status": "pp-r2 · proof of concept",
"released": "2026-10-21",
"license": "Apache-2.0",
"base_model": {
"label": "Qwen/Qwen3-0.6B",
"url": "https://huggingface.co/Qwen/Qwen3-0.6B"
},
"page": "https://ai.danielrosehill.com/models/proximate-pseudonymizer-0.6b",
"hugging_face": "https://huggingface.co/danielrosehill/Proximate-Pseudonymizer-0.6B",
"summary": [
"Text in, text out. \"I'm Daniel and I live in Jerusalem\" becomes \"I'm David and I live in Tel Aviv\". Each detail gets one substitute, used every time it recurs, and the rest of the text is left alone.",
"It is not a redactor. There are no [NAME] tags, so the text still reads naturally and can be shared with an AI tool, a forum or a dataset without pointing at the real people and places.",
"Substitutes are meant to be proximate: same gender, culture, country and kind of place or organisation. Dates move a little and keep their year. Countries and public figures are kept.",
"This is a proof of concept. On 295 held-out texts it leaves 9.3% of identifying details unchanged, about 1 in 11. Do not run it unsupervised. Check every output before you share it."
],
"results": {
"as_of": "2026-10-21",
"run": "pp-r2",
"version": "Proof of concept",
"base_label": "Untrained Qwen3-0.6B, one example",
"tuned_label": "Fine-tune pp-r2",
"metric_note": "Scored on 295 held-out synthetic texts. Leak is the share of identifying details still present word for word in the output; lower is better. The other rows are shares; higher is better.",
"rows": [
{
"test": "Leak: identifying details left unchanged (lower is better)",
"base": "85.6%",
"tuned": "9.3%",
"better": true
},
{
"test": "Consistency: texts where each changed detail got one substitute",
"base": "0.997",
"tuned": "0.990"
},
{
"test": "Rest of the text preserved (non-identifying words kept)",
"base": "0.791",
"tuned": "0.998",
"better": true
},
{
"test": "Texts with no details returned unchanged",
"base": "0.308",
"tuned": "0.962",
"better": true
}
],
"notes": [
"A 9.3% leak is fine for a demonstration and not fine for privacy. Names, dates and ages leak most.",
"Suggested targets: 1% or less with a human reviewing every output; about 0.1% or less without one, and then only for low-stakes text. This checkpoint is far from both.",
"Whether a substitute is proximate is not scored automatically.",
"The untrained model scores higher on consistency because it changes little; its leak is 85.6%."
]
},
"downloads": [
{
"label": "safetensors (Transformers)",
"size": "1.19 GB",
"use": "Python on CPU or GPU, at full precision. Use the exact system prompt from the model card; it is the one the model was trained with.",
"url": "https://huggingface.co/danielrosehill/Proximate-Pseudonymizer-0.6B/resolve/main/model.safetensors"
}
],
"datasets": [],
"datasets_note": "The training data is not published. It is 4,280 synthetic texts, described under Training.",
"limitations": [
"About 1 in 11 identifying details survives. It is not safe to run without review.",
"Substitutes can collide: in one test, two different surnames both became \"Moran\". Some come out too close to the original.",
"Trained on synthetic texts only. 80% are English, weighted towards Israel, Ireland and the US; other languages work less well.",
"The replacement policy is one person's choice: big cities are replaced, countries and public figures are kept."
],
"checkpoints": [
{
"label": "pp-r2",
"state": "done",
"date": "2026-10-21",
"note": "Proof of concept: leak 9.3%, consistency 0.99, rest of text 0.998 preserved."
},
{
"label": "Leak at or below 1%",
"state": "planned",
"note": "The target for use with a human reviewing every output. Training is paused."
}
],
"tags": [
"Qwen3",
"pseudonymization",
"anonymization",
"privacy",
"text rewriting",
"proof of concept"
]
},
{
"slug": "sabich-qwen3-asr-1.7b",
"title": "Hebrish (Sabich-Qwen3-ASR-1.7B v1.0): Sabich on Qwen3-ASR 1.7B for desktop GPUs",
"short_title": "Sabich-Qwen3-ASR-1.7B (desktop GPU)",
"tagline": "The most accurate Sabich: the same code-switching and tech data, trained on Qwen3-ASR 1.7B, an ASR model with an LLM decoder, for computers with a GPU.",
"status": "v1.0 · Sabich-Qwen3-ASR-1.7B",
"released": "2026-10-21",
"license": "Apache-2.0 (base model); use under the Hebrish terms of use",
"base_model": {
"label": "Qwen/Qwen3-ASR-1.7B",
"url": "https://huggingface.co/Qwen/Qwen3-ASR-1.7B"
},
"page": "https://ai.danielrosehill.com/models/sabich-qwen3-asr-1.7b",
"hugging_face": "https://huggingface.co/Hebrish/Sabich-Qwen3-ASR-1.7B",
"website": "https://hebrish.app/models/sabich-qwen3",
"gated": "Both repos (the Transformers weights and the GGUF builds) are gated: request access on Hugging Face. Requests are reviewed by hand.",
"sibling": {
"slug": "parakeet-tdt-v2-english-hebrew-all-subsets",
"label": "Sabich",
"note": "This is Sabich's training data on a larger base, for computers with a GPU (about 5 GB of VRAM in bf16, or about 2 GB as a GGUF). For phones and the Hebrish apps, use Sabich itself, on Parakeet TDT 0.6B v2.",
"page": "https://ai.danielrosehill.com/models/parakeet-tdt-v2-english-hebrew-all-subsets"
},
"summary": [
"Sabich-Qwen3-ASR-1.7B is Sabich, the everything model of the Hebrish project, trained on a larger base. The data is the same: English with Hebrew words in it, written in one romanized spelling, plus tech vocabulary, brand names, acronyms, numbers and currency in digits, and everyday email and message dictation.",
"The main Sabich is Parakeet TDT 0.6B v2, sized for phones. This variant puts the same data on Qwen/Qwen3-ASR-1.7B, an ASR model with an LLM decoder, for desktop GPUs and the highest accuracy. On the held-out split it halves stock Qwen3-ASR's code-switching error, from 10.0% to 5.4%, and writes numbers in digits every time.",
"Output is English, cased and punctuated, with Hebrew words in Latin letters, never Hebrew script. It is not in the Hebrish apps yet: run it in your own tools, with Transformers on PyTorch (CUDA or ROCm) or as a GGUF in llama.cpp.",
"A variant on another base keeps its line's food name and adds the base: Sabich--. Other Sabich variants run on Canary-Qwen 2.5B, Parakeet TDT-CTC 110M and FUTO asr4all."
],
"results": {
"as_of": "2026-10-21",
"run": "q1-all",
"version": "v1.0 · Sabich-Qwen3-ASR-1.7B",
"base_label": "Stock Qwen3-ASR 1.7B",
"tuned_label": "Sabich-Qwen3-ASR-1.7B v1.0",
"metric_note": "Lower WER is better; higher term recall is better.",
"rows": [
{
"test": "English–Hebrew code-switching (normalised WER)",
"base": "10.0%",
"tuned": "5.4%",
"better": true
},
{
"test": "Hebrew terms never seen in training: recall",
"base": "32%",
"tuned": "50%",
"better": true
},
{
"test": "Hebrew terms seen in training: recall",
"base": "40%",
"tuned": "73%",
"better": true
},
{
"test": "Speaker's own English (normalised WER)",
"base": "4.6%",
"tuned": "2.5%",
"better": true
},
{
"test": "LibriSpeech test-clean, 200 utterances (normalised WER)",
"base": "1.2%",
"tuned": "1.35%"
},
{
"test": "Long form, about 3-minute joined clips (normalised WER)",
"base": "12.0%",
"tuned": "8.7%",
"better": true
},
{
"test": "Numbers written in digits",
"base": "2/8",
"tuned": "8/8",
"better": true
}
],
"notes": [
"Held-out test split of the speaker's recordings at dataset revision cf84f7e4, term-disjoint from training. Stock is Qwen/Qwen3-ASR-1.7B on the same clips.",
"WER is normalised: lowercase, punctuation removed. Term recall is a word-boundary match on the romanized Hebrew term.",
"These numbers are on a different test-set revision from Sabich's (f956dbb1), so the two have not been compared head to head yet.",
"LibriSpeech is within noise of stock. Long-form audio is its weakest area."
]
},
"downloads": [
{
"label": "Q8_0 (llama.cpp)",
"size": "1.83 GB",
"use": "Desktop GPU or CPU, the default llama.cpp build. Needs the audio encoder (mmproj) file beside it. Fits a 4 GB card.",
"url": "https://huggingface.co/Hebrish/Sabich-Qwen3-ASR-1.7B-GGUF/resolve/main/llama.cpp/Sabich-Qwen3-ASR-1.7B-Q8_0.gguf"
},
{
"label": "Audio encoder (mmproj Q8_0)",
"size": "356 MB",
"use": "The audio encoder that llama.cpp loads with --mmproj, next to the Q8_0 model file.",
"url": "https://huggingface.co/Hebrish/Sabich-Qwen3-ASR-1.7B-GGUF/resolve/main/llama.cpp/mmproj-Sabich-Qwen3-ASR-1.7B-Q8_0.gguf"
},
{
"label": "transcribe.cpp Q4_K_M",
"size": "1.32 GB",
"use": "Android phone, imported into Offline Voice Input. It runs, but slowly; for phones, Sabich is the better fit.",
"url": "https://huggingface.co/Hebrish/Sabich-Qwen3-ASR-1.7B-GGUF/resolve/main/transcribe.cpp/Sabich-Qwen3-ASR-1.7B-Q4_K_M.gguf"
}
],
"datasets": [
{
"label": "Read-Aloud Voice Sentences (revision e918e381, blend all)",
"url": "https://huggingface.co/datasets/danielrosehill/Read-Aloud-Voice-Sentences"
},
{
"label": "All training datasets (collection)",
"url": "https://huggingface.co/collections/Hebrish/parakeet-v2-fine-tune-training-data-6ad63c739d300685a62edb3e"
},
{
"label": "LibriSpeech (replay)",
"url": "https://huggingface.co/datasets/openslr/librispeech_asr"
}
],
"limitations": [
"Trained on one voice so far: an English speaker with an Irish accent living in Jerusalem. Recordings from more speakers are the next step; contribute yours at hebrish.app/contact?topic=samples.",
"About half of the Hebrew terms it never heard in training are still missed or misspelled (50% recall).",
"Long-form audio is its weakest area: 8.7% WER on about 3-minute clips.",
"Needs a GPU for comfortable speed. On 8 cloud CPU cores the Q8_0 file ran at about 1× realtime; phone speed is not measured.",
"Not in the Hebrish apps. The repos are gated, with access by request."
],
"checkpoints": [
{
"label": "v1.0 (q1-all)",
"state": "done",
"date": "2026-10-21",
"note": "First release: Sabich's data on Qwen3-ASR 1.7B, 3 epochs on one H100, epoch 3 kept. 5.4% code-switching WER against 10.0% for stock."
},
{
"label": "GGUF builds",
"state": "done",
"date": "2026-10-21",
"note": "llama.cpp (bf16, Q8_0, Q4_K_M plus the audio encoder) and transcribe.cpp (Q8_0, Q4_K_M), checked on three held-out clips."
},
{
"label": "Head to head with Sabich",
"state": "planned",
"note": "Score it on the same test-set revision as Sabich; the two have not been compared yet."
},
{
"label": "Phone speed",
"state": "planned",
"note": "Measure the transcribe.cpp build on a phone; only CPU speed in the cloud is known."
}
],
"tags": [
"Hebrish",
"Sabich",
"ASR",
"Qwen3-ASR",
"Transformers",
"llama.cpp",
"GGUF",
"code-switching",
"Hebrew",
"English",
"desktop GPU"
]
},
{
"slug": "shakshuka-nemotron-streaming",
"title": "Hebrish (Shakshuka v1.0): Nemotron streaming fine-tune for code-switching plus tech dictation",
"short_title": "Shakshuka (Nemotron streaming)",
"tagline": "The streaming line of Hebrish: text appears while you speak, trained on the same everything-in-it recordings as Sabich.",
"status": "v1.0 · Shakshuka",
"released": "2026-10-21",
"license": "NVIDIA Open Model License Agreement; use under the Hebrish terms of use",
"base_model": {
"label": "nvidia/nemotron-speech-streaming-en-0.6b",
"url": "https://huggingface.co/nvidia/nemotron-speech-streaming-en-0.6b"
},
"page": "https://ai.danielrosehill.com/models/shakshuka-nemotron-streaming",
"hugging_face": "https://huggingface.co/Hebrish/Shakshuka",
"website": "https://hebrish.app/models/shakshuka",
"gated": "Every Shakshuka repo (.nemo, GGUF, sherpa-onnx and the PoC record) is gated: request access on Hugging Face. Requests are reviewed by hand.",
"sibling": {
"slug": "parakeet-tdt-v2-english-hebrew-all-subsets",
"label": "Sabich",
"note": "Shakshuka streams: text appears while you speak. Sabich transcribes offline, when you stop, and on past results it is the more accurate of the two. Both are trained on the same data, so use Sabich unless you need live text.",
"page": "https://ai.danielrosehill.com/models/parakeet-tdt-v2-english-hebrew-all-subsets"
},
"summary": [
"Shakshuka is the streaming line of Hebrish: a fine-tune of NVIDIA's cache-aware streaming model, Nemotron Speech Streaming EN 0.6B, so text appears while you speak, in chunks of 80 ms to 1.12 s. Like the other Hebrish models it writes the Hebrew words in English speech in one romanized spelling, and it never writes Hebrew script.",
"Shakshuka v1.0 (2026-10-21) is the first full release. It is trained on Sabich's data: code-switched speech plus tech vocabulary, brand names, acronyms, numbers and currency in digits, Irish-English phrasing and everyday dictation. The two earlier checkpoints, PoC-1 and PoC-2, were trained on the code-switching mix only, beside Jachnun.",
"v1.0 was published straight after training and is not benchmarked yet. The numbers below are PoC-2's: 62.2% Hebrew term recall and 4.7% WER on the public benchmark, scored offline. Streaming at 560 ms chunks cost it nothing measurable against offline decoding.",
"It is not in the Hebrish apps yet. Run it in NVIDIA NeMo, as a sherpa-onnx streaming build (560 ms chunks) or as a GGUF in transcribe.cpp. Like Jachnun and Sabich, it takes an Israeli food name."
],
"results": {
"as_of": "2026-10-19",
"run": "r6-nemotron-en30",
"version": "Shakshuka PoC-2",
"pending": "Shakshuka v1.0 (run r8-nemotron-all, 2026-10-21) was published straight after training and has no test results yet. Its validation WER at the chosen checkpoint was 6.66% (raw, used only to pick the checkpoint; not a test score).",
"base_label": "Stock Parakeet v2 (reference)",
"tuned_label": "Shakshuka PoC-2",
"metric_note": "Stock Nemotron was not scored on these test sets, so the reference column is stock Parakeet TDT 0.6B v2 on the same clips. Lower WER is better; higher term recall is better.",
"rows": [
{
"test": "Public benchmark, 84 clips: Hebrew term recall (offline)",
"base": "32.6%",
"tuned": "62.2%",
"better": true
},
{
"test": "Public benchmark: unseen-term recall (offline)",
"base": "32.5%",
"tuned": "54.7%",
"better": true
},
{
"test": "Public benchmark: WER (offline)",
"base": "9.5%",
"tuned": "4.7%",
"better": true
},
{
"test": "Public benchmark: English control WER (offline)",
"base": "2.7%",
"tuned": "2.5%"
},
{
"test": "Code-switching, held-out split, 387 clips (normalised WER, offline)",
"base": "11.3%",
"tuned": "5.8%",
"better": true
},
{
"test": "Code-switching, held-out split (normalised WER, streaming at 560 ms)",
"base": "n/a",
"tuned": "5.65%",
"better": true
},
{
"test": "Hebrew terms held out of training: share correct (offline)",
"base": "25.9%",
"tuned": "44.4%",
"better": true
},
{
"test": "Speaker's own English, 73 clips (normalised WER)",
"base": "3.1%",
"tuned": "3.3%"
},
{
"test": "LibriSpeech test-clean, 200 utterances (normalised WER)",
"base": "1.2%",
"tuned": "2.0%"
},
{
"test": "Long form, 15 files (normalised WER)",
"base": "9.6%",
"tuned": "5.8%",
"better": true
}
],
"notes": [
"Public benchmark: test set v1, clean subset (84 clips, 172 term occurrences, 139 distinct terms), round 3 on the leaderboard Space, scored 2026-10-19 with offline (full-context) decoding. Jachnun v1.0 on the same clips: 73.3% term recall, 70.1% unseen, 3.3% WER, 2.2% English control.",
"Held-out split: dataset revision 550bfcaa0f84, term-disjoint, never trained on. Jachnun v1.0 on the same split: 5.1% code-switching WER, 47.8% held-out terms, 1.7% own English, 1.2% LibriSpeech, 5.0% long form.",
"Streaming used cache-aware inference at 560 ms chunks (att_context_size [70, 6], tag r6-nemotron-en30-s560): 5.65% against 5.81% offline, no measurable loss. The 80 and 160 ms settings are not measured.",
"No forgetting verdict: stock Nemotron was not re-scored on this revision, so the English rows have no baseline of their own. PoC-2 drops all 8 number tokens in the held-out split.",
"WER is normalised: lowercase, punctuation removed."
]
},
"downloads": [
{
"label": ".nemo checkpoint",
"size": "2.47 GB",
"use": "Python, GPU inference, offline or cache-aware streaming, or further fine-tuning, at full precision.",
"url": "https://huggingface.co/Hebrish/Shakshuka/resolve/main/Shakshuka.nemo"
},
{
"label": "transcribe.cpp Q8_0",
"size": "730 MB",
"use": "The recommended transcribe.cpp file. Whether an app loads this architecture depends on its transcribe.cpp version; transcribe-cli v0.1.3 does.",
"url": "https://huggingface.co/Hebrish/Shakshuka-GGUF/resolve/main/Shakshuka-Q8_0.gguf"
},
{
"label": "transcribe.cpp Q4_K_M",
"size": "475 MB",
"use": "Smaller quant for devices short on memory.",
"url": "https://huggingface.co/Hebrish/Shakshuka-GGUF/resolve/main/Shakshuka-Q4_K_M.gguf"
},
{
"label": "transcribe.cpp F16",
"size": "1.24 GB",
"use": "Desktop GPU, half precision.",
"url": "https://huggingface.co/Hebrish/Shakshuka-GGUF/resolve/main/Shakshuka-F16.gguf"
}
],
"datasets": [
{
"label": "Read-Aloud Voice Sentences (revision e918e381, blend all)",
"url": "https://huggingface.co/datasets/danielrosehill/Read-Aloud-Voice-Sentences"
},
{
"label": "All training datasets (collection)",
"url": "https://huggingface.co/collections/Hebrish/parakeet-v2-fine-tune-training-data-6ad63c739d300685a62edb3e"
},
{
"label": "LibriSpeech (replay)",
"url": "https://huggingface.co/datasets/openslr/librispeech_asr"
}
],
"limitations": [
"Trained on one voice so far: an English speaker with an Irish accent living in Jerusalem. Recordings from more speakers are the next step; contribute yours at hebrish.app/contact?topic=samples.",
"v1.0 is not benchmarked yet: no test-set numbers, offline or streaming.",
"On past results it trails the offline Parakeet models: PoC-2 was under one WER point behind Jachnun v1.0 on code-switching, and 11 points behind on term recall on the public benchmark.",
"Trained with full attention context; accuracy at the smallest chunks (80 and 160 ms) may be lower than offline.",
"NVIDIA Open Model License Agreement, not CC-BY-4.0: read it before shipping. Not made or endorsed by NVIDIA. The repos are gated, with access by request."
],
"checkpoints": [
{
"label": "PoC-1 (r5-nemotron-en30)",
"state": "done",
"date": "2026-10-18",
"note": "First Nemotron streaming fine-tune, trained beside PoC-5 on the code-switching mix. Published before any benchmark; now in the PoC record's history."
},
{
"label": "PoC-2 (r6-nemotron-en30)",
"state": "done",
"date": "2026-10-19",
"note": "Trained beside Jachnun v1.0 on the same data. Scored 2026-10-19: 62.2% term recall on the public benchmark; 5.8% code-switching WER offline and 5.65% streaming at 560 ms on the held-out split."
},
{
"label": "v1.0 · Shakshuka (r8-nemotron-all)",
"state": "done",
"date": "2026-10-21",
"note": "First full release, on Sabich's data. .nemo, sherpa-onnx int8 streaming build (560 ms chunks) and GGUF (Q8_0, Q4_K_M, F16), each export checked on two held-out clips."
},
{
"label": "v1.0 benchmark",
"state": "planned",
"note": "Score v1.0 offline and streaming, with stock Nemotron as the baseline."
},
{
"label": "Small-chunk streaming",
"state": "planned",
"note": "Measure accuracy at 80 and 160 ms chunks, not yet measured on any checkpoint."
}
],
"tags": [
"Hebrish",
"Shakshuka",
"ASR",
"Nemotron",
"NeMo",
"streaming",
"code-switching",
"Hebrew",
"English",
"sherpa-onnx"
]
}
],
"dataset_cards": [
{
"slug": "english-hebrew-codeswitching-sentences",
"title": "English–Hebrew Code-Switching Sentences",
"short_title": "English–Hebrew Code-Switching Sentences",
"tagline": "English sentences with everyday Hebrew words and Israeli names written in Latin letters, read aloud on a phone by one English speaker living in Israel.",
"kind": "speech",
"status": "Live and growing",
"license": "CC-BY-4.0",
"page": "https://ai.danielrosehill.com/datasets/english-hebrew-codeswitching-sentences",
"hugging_face": "https://huggingface.co/datasets/danielrosehill/English-Hebrew-Codeswitching-Sentences",
"gated": "Gated on Hugging Face (manual approval): request access to download the audio and metadata.",
"parent": {
"label": "Read-Aloud Voice Sentences",
"url": "https://huggingface.co/datasets/danielrosehill/Read-Aloud-Voice-Sentences",
"note": "The full dataset; this one is its code-switching subset"
},
"used_by": [
{
"label": "Hebrish: Parakeet TDT 0.6B v2 for English–Hebrew code-switching",
"url": "https://huggingface.co/danielrosehill/Parakeet-TDT-0.6B-v2-English-Hebrew-Codeswitching",
"page": "https://ai.danielrosehill.com/models/parakeet-english-hebrew-codeswitching"
}
],
"summary": [
"Speech recognition trained on English mangles the Hebrew words that English speakers in Israel use every day: Bituach Leumi, Misrad Hapnim, tzaharon, makolet. This dataset is the training and test data for fixing that.",
"Each row is one take: a short English sentence containing one or more Hebrew words, read aloud and reviewed on the phone before it was published. Hebrew words follow one canonical romanized spelling, and Hebrew proper nouns are capitalised (Yerushalayim, Bituach Leumi).",
"It is a subset of Read-Aloud Voice Sentences: every take whose categories include code_switch, copied unchanged with the same ids and split. Fixes go into the main dataset and are re-synced."
],
"size": {
"as_of": "2026-10-15",
"counted_live": false,
"splits": [
{
"split": "train",
"clips": 1248,
"minutes": 120.1,
"terms": 734,
"phoneClips": 754
},
{
"split": "test",
"clips": 121,
"minutes": 13.9,
"terms": 146,
"phoneClips": 85
}
]
},
"layout": [
{
"field": "text",
"description": "Ground truth, punctuated and cased like Parakeet v2 output"
},
{
"field": "target_terms",
"description": "The Hebrew words and names the sentence was written to exercise"
},
{
"field": "categories",
"description": "Always includes code_switch; may add names, message and others"
},
{
"field": "prompt_source",
"description": "Where the sentence text came from, for attribution"
},
{
"field": "duration_s",
"description": "Clip length in seconds"
},
{
"field": "device / environment / conditions",
"description": "Recording phone or mic and the room it was recorded in"
},
{
"field": "recorded_at",
"description": "UTC timestamp of the take"
}
],
"tags": [
"ASR",
"speech",
"code-switching",
"Hebrew",
"English",
"single-speaker",
"fine-tuning"
]
},
{
"slug": "read-aloud-voice-sentences",
"title": "Read-Aloud Voice Sentences",
"short_title": "Read-Aloud Voice Sentences",
"tagline": "Every take recorded for the Hebrish speech models: read-aloud English sentences with Hebrew words, tech vocabulary, names, emails and messages, recorded on a phone and published in batches straight from the capture app.",
"kind": "speech",
"status": "Live and growing",
"license": "CC-BY-4.0",
"page": "https://ai.danielrosehill.com/datasets/read-aloud-voice-sentences",
"hugging_face": "https://huggingface.co/datasets/danielrosehill/Read-Aloud-Voice-Sentences",
"gated": "Gated on Hugging Face (manual approval): request access to download the audio and metadata.",
"used_by": [
{
"label": "Hebrish Jachnun v1.0 (code-switching rows)",
"url": "https://huggingface.co/Hebrish/Jachnun",
"page": "https://ai.danielrosehill.com/models/parakeet-english-hebrew-codeswitching"
},
{
"label": "Hebrish Sabich v1.0 (every task recording)",
"url": "https://huggingface.co/Hebrish/Sabich",
"page": "https://ai.danielrosehill.com/models/parakeet-tdt-v2-english-hebrew-all-subsets"
},
{
"label": "PoC-1 to PoC-5 (the proof-of-concept checkpoints)",
"url": "https://huggingface.co/danielrosehill/Parakeet-TDT-0.6B-v2-English-Hebrew-Codeswitching"
},
{
"label": "Shakshuka PoC (Nemotron streaming fine-tune)",
"url": "https://huggingface.co/Hebrish/Nemotron-Speech-Streaming-EN-0.6B-English-Hebrew-Codeswitching"
}
],
"summary": [
"This is the full, undifferentiated dataset behind the Hebrish speech models. Every take is a short sentence read aloud and reviewed on the phone before it was published; the dataset grows as new batches are pushed from the capture app.",
"A row can carry several categories: code_switch (English with Hebrew words written in Latin letters, one canonical spelling each), names, email and message (the main use case: dictating emails and messages on Android), tech, homophone, general, plus later tags such as religious, yiddish, brand-names, irish-english, currency and disfluency. Filter on categories to use one kind of data on its own.",
"Two derived datasets carry its phone takes for one kind of data, unchanged (same ids, columns and split) and re-synced on every push: English–Hebrew Code-Switching Sentences (rows tagged code_switch) and Tech-Sentences (rows tagged tech). Rows with Jewish religious vocabulary are also mirrored to Jewish-Religious-Terminology-Sentences.",
"The test split is term-disjoint for code_switch and tech: a held-out Hebrew or technical term never appears in train, so the evaluation measures generalisation to unseen vocabulary. The public benchmark test set was cut from this split at revision 3bdaa2a50d52.",
"Hebrew words switched into English do not have one pronunciation: Modern Israeli Hebrew says Shabbat, many English-speaking Jews say Shabbos. Variants are recorded on purpose, and since 2026-10-19 each take is tagged with hebrew_pronunciation; since 2026-10-20 variant_groups links the Israeli and Ashkenazi spellings of one word."
],
"size": {
"as_of": "2026-10-21",
"counted_live": false,
"splits": [
{
"split": "train",
"clips": 5807,
"minutes": 642,
"terms": 2688,
"phoneClips": 5230
},
{
"split": "test",
"clips": 536,
"minutes": 62.7,
"terms": 471,
"phoneClips": 498
}
]
},
"layout": [
{
"field": "text",
"description": "Ground truth, punctuated and cased like Parakeet v2 output"
},
{
"field": "categories",
"description": "What the sentence exercises: code_switch, tech, names, email, message, homophone, general, religious, yiddish and others; a row can have several"
},
{
"field": "target_terms",
"description": "The terms (Hebrew or technical) the sentence was written to exercise, and which the evaluation scores"
},
{
"field": "prompt_source",
"description": "Where the sentence text came from: tatoeba:, generated:, enron: or one of the author's earlier datasets (see licence)"
},
{
"field": "edited",
"description": "true when the speaker corrected the prompt before or after reading it"
},
{
"field": "duration_s / environment / device / recorded_at",
"description": "Recording metadata"
},
{
"field": "conditions",
"description": "Recording conditions noted on the phone (background-noise, recorded-outdoors, in-car, ...); missing on the earliest rows"
},
{
"field": "mode",
"description": "How the take was made; \"import\" marks the 2025 desktop-mic rows"
},
{
"field": "hebrew_pronunciation",
"description": "israeli, ashkenazi, mixed, or null when not tagged (since 2026-10-19; null does not mean Israeli)"
},
{
"field": "variant_groups / variant_spelling",
"description": "Ids of Israeli/Ashkenazi spelling pairs that occur in text, and which tradition the text spells (since 2026-10-20)"
}
],
"tags": [
"Hebrish",
"ASR",
"speech",
"code-switching",
"Hebrew",
"English",
"dictation",
"fine-tuning"
]
},
{
"slug": "english-hebrew-codeswitching-asr-benchmark",
"title": "English–Hebrew Code-Switching ASR Benchmark",
"short_title": "English–Hebrew Code-Switching ASR Benchmark",
"tagline": "A frozen, versioned test set for scoring how well speech recognition writes the Hebrew words in English speech, \"pick up some bamba from the makolet\", with stock models and fine-tunes scored the same way on a public leaderboard.",
"kind": "speech",
"status": "Test set v1 · early stage",
"license": "CC-BY-4.0",
"page": "https://ai.danielrosehill.com/datasets/english-hebrew-codeswitching-asr-benchmark",
"hugging_face": "https://huggingface.co/datasets/Hebrish/English-Hebrew-Codeswitching-ASR-Benchmark",
"gated": "The test set is gated on Hugging Face (manual approval): request access to download the audio. The runner, scorer, results and leaderboard are public.",
"parent": {
"label": "Read-Aloud Voice Sentences",
"url": "https://huggingface.co/datasets/danielrosehill/Read-Aloud-Voice-Sentences",
"note": "Cut from its held-out test split at revision 3bdaa2a50d52"
},
"used_by": [
{
"label": "Hebrish Sabich v1.0: 73.8% term recall, 3.3% WER (round 4)",
"url": "https://huggingface.co/Hebrish/Sabich",
"page": "https://ai.danielrosehill.com/models/parakeet-tdt-v2-english-hebrew-all-subsets"
},
{
"label": "Hebrish Jachnun v1.0: 73.3% term recall, 3.3% WER (round 3)",
"url": "https://huggingface.co/Hebrish/Jachnun",
"page": "https://ai.danielrosehill.com/models/parakeet-english-hebrew-codeswitching"
},
{
"label": "PoC-1 to PoC-3: 62.2%, 65.7%, 67.4% term recall",
"url": "https://huggingface.co/danielrosehill/Parakeet-TDT-0.6B-v2-English-Hebrew-Codeswitching"
},
{
"label": "Shakshuka PoC-2 (Nemotron streaming, scored offline): 62.2%",
"url": "https://huggingface.co/Hebrish/Nemotron-Speech-Streaming-EN-0.6B-English-Hebrew-Codeswitching"
},
{
"label": "Whisper-Hebrish: 54.1% (known training-data overlap)",
"url": "https://huggingface.co/danielrosehill/Whisper-Hebrish"
}
],
"summary": [
"English speakers in Israel drop Hebrew words into English, and English speech recognition writes the nearest English-sounding thing (Macaulay for makolet). This benchmark measures how often a model gets those words right, and whether it still transcribes plain English well. The references write each Hebrew word in one canonical romanized spelling; Hebrew script in the output is never a hit.",
"Test set v1 is 145 clips from the held-out test split of Read-Aloud Voice Sentences, each tagged with a subset. The headline is the clean subset: 84 phone recordings (11.8 min) of sentences that appear in no published training set, holding 172 Hebrew term occurrences of 139 distinct terms. english_control (24 clips) is the same speaker's English with no Hebrew, a regression check. The rest (contaminated_text, contaminated_audio, desktop_mic) are reported only in the all_code_switch view.",
"How it is scored. Term recall, the headline, is the share of Hebrew term occurrences whose canonical spelling appears in the transcript as a whole word, case-insensitive, with hyphen and space interchangeable; never a substring. Unseen-term recall counts only the 117 occurrences of terms absent from the source training split. Lenient recall also accepts the lexicon's variant spellings (tsaharon for tzaharon), which is fairer to stock models. WER is normalised (lowercase, punctuation removed), and English control WER catches a fine-tune that damages English.",
"Contamination is checked, not assumed. The 2025 desktop clips and the sentences Whisper-Hebrish was trained on were traced to its exact training files and split out of the headline subset (2026-10-15). Before a fine-tune is published, the benchmark's 145 clip ids and texts are checked against its training manifest; none were found for Jachnun v1.0 or the Nemotron fine-tune.",
"The test set never changes in place: a new test set is a new version, and every result records the commit it was scored against. Results are stored with every transcript, so any score can be recomputed with the public scorer. Stock models on the same clips: Whisper Large V3 45.9%, Whisper Large V3 Turbo 40.1%, Parakeet TDT 0.6B v2 32.6%, Parakeet v3 31.4%."
],
"stats": [
{
"value": "145",
"label": "clips in test set v1"
},
{
"value": "84",
"label": "clean headline clips (11.8 min)"
},
{
"value": "172",
"label": "Hebrew term occurrences (139 terms)"
},
{
"value": "117",
"label": "occurrences of unseen terms"
}
],
"layout": [
{
"field": "file_name",
"description": "WAV, 16 kHz mono"
},
{
"field": "id",
"description": "Same id as in Read-Aloud Voice Sentences"
},
{
"field": "text",
"description": "Reference transcript: cased and punctuated, canonical romanized Hebrew"
},
{
"field": "subset",
"description": "clean, english_control, contaminated_text, contaminated_audio or desktop_mic"
},
{
"field": "hebrew_terms",
"description": "List of {term, variants, unseen}: the canonical spelling, accepted variants from the Hebrew-Latin token classifier lexicon, and whether the term is absent from the source train split"
},
{
"field": "source_text",
"description": "Set on the 4 rows whose reference was corrected to the canonical term; the original transcript"
},
{
"field": "duration_s / categories / recording / source_dataset",
"description": "Provenance"
}
],
"tags": [
"Hebrish",
"ASR",
"benchmark",
"evaluation",
"code-switching",
"Hebrew",
"English",
"leaderboard"
]
},
{
"slug": "hebrew-latin-code-switching",
"title": "Hebrew-in-Latin Code-Switching Corpus",
"short_title": "Hebrew-in-Latin Code-Switching Corpus",
"tagline": "English sentences with Hebrew words written in Latin letters, every Hebrew span marked token by token, for finding those words in text before text-to-speech mispronounces them. No corpus like it had been published.",
"kind": "text",
"status": "v1 · released",
"license": "MIT",
"page": "https://ai.danielrosehill.com/datasets/hebrew-latin-code-switching",
"hugging_face": "https://huggingface.co/datasets/Hebrish/hebrew-latin-code-switching",
"used_by": [
{
"label": "Hebrew-Latin Token Classifier (span F1 0.941 on the test split)",
"url": "https://huggingface.co/Hebrish/Hebrew-Latin-Token-Classifier",
"page": "https://ai.danielrosehill.com/models/hebrew-latin-token-classifier"
}
],
"summary": [
"English text-to-speech reads romanized Hebrew as English: makolet comes out as \"MACK-oh-let\". The fix is to find the Hebrew spans, convert them to Hebrew script and send each to a Hebrew voice. This corpus trains the classifier that does the finding. There was no heb_Latn label in GlotLID v3, lid.176 or lid218e, no token-level code-switching model covering Hebrew, and no Hebrew-English code-switching corpus of any kind.",
"The labelling test is pronunciation, not etymology: would an English voice mispronounce this word, and would treating it as Hebrew fix that? So kosher, Shabbat and hummus are negatives, while makolet, mashkanta and arnona are positives. 574 sentences are hard negatives built to trap a classifier: English senses, substring traps, anglicised loanwords and lookalikes.",
"Generation was term-seeded: a model was given one Hebrew term and asked for sentences using it, and any sentence where the term did not match at a word boundary was rejected. A second model from a different family then annotated each sentence blind, without being told the seed term. Agreements (87.7%) were accepted, disagreements went to human review, and a random 10% of agreements was audited too: 757 human-reviewed tasks in all.",
"458 sentences are dictated: the text of sentences read aloud for the Hebrish speech fine-tunes, so a human chose to say every one. Their labels are recomputed against the term inventory, not copied. The rest of the corpus is synthetic; treat the audit as the quality estimate.",
"The splits are term-disjoint: no seed term appears in more than one split, so a model is scored on Hebrew words it has never seen."
],
"stats": [
{
"value": "4,737",
"label": "sentences"
},
{
"value": "1,235",
"label": "seed terms"
},
{
"value": "458",
"label": "dictated sentences"
},
{
"value": "574",
"label": "hard negatives"
}
],
"layout": [
{
"field": "id",
"description": "Row id (g… for generated)"
},
{
"field": "text",
"description": "The English sentence"
},
{
"field": "source",
"description": "generated or dictated"
},
{
"field": "seed_term / category",
"description": "The term the sentence was generated around, and its topic (bureaucracy, food_shopping, … or negative: for hard negatives)"
},
{
"field": "spans",
"description": "Hebrew spans as {start, end, surface, term}, character offsets into text"
},
{
"field": "masked",
"description": "Character spans with no label: model disagreements no human has ruled on, and Israeli person names. Ignore their tokens when training or scoring (label -100)"
},
{
"field": "tokens",
"description": "[token, tag] pairs, tags O / B-HE / I-HE; multi-word terms such as bituach leumi are B-HE I-HE"
}
],
"tags": [
"Hebrish",
"token classification",
"code-switching",
"Hebrew",
"romanization",
"text-to-speech",
"BIO"
]
}
],
"hebrish": {
"site": "https://hebrish.app",
"about": "The Hebrish model series (English with Hebrew words), as listed on hebrish.app. href is a card page on the AI site where one exists.",
"as_of": "2026-10-21",
"fetched_live": true,
"groups": [
{
"title": "In the Hebrish apps",
"note": "On-device dictation on Android, Ubuntu and Windows.",
"models": [
{
"name": "Sabich",
"nickname": "v1.0",
"role": "The recommended model in the Hebrish apps. Trained on everything: the code-switched speech plus tech vocabulary, brand names, numbers and other task recordings.",
"base": "NVIDIA Parakeet TDT 0.6B v2",
"params": "0.6B",
"badge": "Recommended",
"stat": {
"value": "73.8%",
"label": "term recall, 3.3% WER"
},
"href": "https://ai.danielrosehill.com/models/parakeet-tdt-v2-english-hebrew-all-subsets",
"external": false
},
{
"name": "Jachnun",
"nickname": "v1.0",
"role": "The pure code-switching line: English with Hebrew words. In the apps next to Sabich; every on-device format.",
"base": "NVIDIA Parakeet TDT 0.6B v2",
"params": "0.6B",
"stat": {
"value": "73.3%",
"label": "term recall, 3.3% WER"
},
"href": "https://ai.danielrosehill.com/models/parakeet-english-hebrew-codeswitching",
"external": false
}
]
},
{
"title": "Sabich on other bases",
"note": "The same training data on bigger and smaller models, for GPUs and wearables. Not in the apps yet.",
"models": [
{
"name": "Sabich-Qwen3-ASR-1.7B",
"nickname": "v1.0",
"role": "Sabich's training data on Qwen3-ASR 1.7B, an ASR model with an LLM decoder. The most accurate Sabich, for computers with a GPU. Not in the Hebrish apps yet.",
"base": "Qwen Qwen3-ASR 1.7B",
"params": "2.0B",
"badge": "Desktop GPU",
"stat": {
"value": "5.4%",
"label": "code-switching WER (stock Qwen3-ASR 10.0%)"
},
"href": "https://ai.danielrosehill.com/models/sabich-qwen3-asr-1.7b",
"external": false
},
{
"name": "Sabich-Canary-Qwen-2.5B",
"nickname": "v1.0",
"role": "Sabich's training data on NVIDIA Canary-Qwen 2.5B, a speech encoder feeding Qwen3-1.7B. For desktop GPUs: the strongest of the Sabich variants on plain English, behind Sabich-Qwen3-ASR-1.7B on code-switching. Not in the Hebrish apps yet.",
"base": "NVIDIA Canary-Qwen 2.5B",
"params": "2.56B",
"badge": "Desktop GPU",
"stat": {
"value": "6.9%",
"label": "code-switching WER (Sabich-Qwen3-ASR-1.7B 5.4%), 2.4% on the speaker's own English"
},
"href": "https://hebrish.app/models/sabich-canary-qwen",
"external": true,
"data": [
{
"label": "Sabich training blend",
"url": "https://huggingface.co/datasets/Hebrish/Parakeet-V2-Blend-All"
}
]
},
{
"name": "Sabich-Parakeet-110M",
"nickname": "v1.0",
"role": "All of Sabich's training on NVIDIA's smallest Parakeet, about a fifth of the size. For low-end phones, watches and wearables. Not in the Hebrish apps yet.",
"base": "NVIDIA Parakeet TDT-CTC 110M",
"params": "114M",
"badge": "Small devices",
"stat": {
"value": "",
"label": "Not benchmarked yet"
},
"href": "https://hebrish.app/models/sabich-110m",
"external": true,
"data": [
{
"label": "Sabich training blend",
"url": "https://huggingface.co/datasets/Hebrish/Parakeet-V2-Blend-All"
}
]
},
{
"name": "Sabich-asr4all",
"nickname": "-S and -M v1.0",
"role": "Sabich's training data on the tiny asr4all streaming models, 28M and 56M parameters, on the CPU. For wearables and very low-power boards. Numbers come out as words. Not in the Hebrish apps yet.",
"base": "FUTO asr4all S and M",
"params": "28.2M",
"badge": "Wearables",
"stat": {
"value": "9.1% (-M) and 10.1% (-S)",
"label": "code-switching WER (stock asr4all-m 19.5%)"
},
"href": "https://hebrish.app/models/sabich-asr4all",
"external": true,
"data": [
{
"label": "Sabich training blend",
"url": "https://huggingface.co/datasets/Hebrish/Parakeet-V2-Blend-All"
}
]
}
]
},
{
"title": "Streaming and Whisper",
"note": "Live text while you speak, and the Whisper fine-tunes where Hebrish started.",
"models": [
{
"name": "Shakshuka",
"nickname": "v1.0",
"role": "The streaming line: text appears while you speak. Shakshuka v1.0 is trained on Sabich's data (code-switching plus tech) on NVIDIA's cache-aware streaming model.",
"base": "NVIDIA Nemotron Speech Streaming EN 0.6B",
"params": "0.6B",
"badge": "Streaming",
"stat": {
"value": "",
"label": "v1.0 not benchmarked yet; PoC-2: 62.2% term recall, 4.7% WER (offline)"
},
"href": "https://ai.danielrosehill.com/models/shakshuka-nemotron-streaming",
"external": false
},
{
"name": "Whisper Hebrish Small ACFT",
"nickname": "small-en-en30-acft",
"role": "A phone-sized Whisper (244M) with FUTO's ACFT, for people who already use FUTO Keyboard on Android.",
"base": "OpenAI Whisper small.en",
"params": "242M",
"stat": {
"value": "5.8%",
"label": "code-switching WER on its own test set (FUTO runtime)"
},
"href": "https://hebrish.app/models/whisper-small",
"external": true
},
{
"name": "Whisper Hebrish",
"nickname": "Version 2",
"role": "Full fine-tune of Whisper large-v3-turbo (809M). Where Hebrish started, in November 2025; the larger sibling of the Small ACFT model.",
"base": "OpenAI Whisper large-v3-turbo",
"params": "809M",
"stat": {
"value": "",
"label": "Version 1: 54.1% term recall, 6.5% WER"
},
"href": "https://hebrish.app/models/whisper-turbo",
"external": true
},
{
"name": "Meorav",
"nickname": "PoC-1",
"role": "Text-to-speech for code-switched English: Irish-accented English that pronounces the Hebrew words in it correctly. A full fine-tune of Qwen3-TTS; the first checkpoint, Meorav PoC-1, is an experiment.",
"base": "Qwen Qwen3-TTS 12Hz 1.7B Base",
"params": "1.9B",
"badge": "POC",
"stat": {
"value": "",
"label": "No benchmark yet; listening verdict pending"
},
"href": "https://hebrish.app/models/meorav",
"external": true
}
]
},
{
"title": "Text",
"note": "A small model that finds Hebrew written in Latin letters inside English text.",
"models": [
{
"name": "Hebrew-Latin Token Classifier",
"nickname": "v1",
"role": "Marks each romanized Hebrew word in an English sentence, so text-to-speech can send it to a Hebrew voice. Span F1 0.941 on terms it never saw in training.",
"base": "Meta XLM-RoBERTa base",
"params": "277M",
"badge": "Small model",
"stat": {
"value": "",
"label": "Span F1 0.941 on terms it never saw in training"
},
"href": "https://ai.danielrosehill.com/models/hebrew-latin-token-classifier",
"external": false
}
]
}
],
"leaderboard": {
"asOf": "2026-10-21",
"url": "https://huggingface.co/spaces/danielrosehill/English-Hebrew-Codeswitching-ASR-Leaderboard",
"repo": "https://github.com/Hebrish-ASR/English-Hebrew-Codeswitching-ASR-Benchmark",
"testSet": "test set v1, clean subset: 84 clips, 172 Hebrew term occurrences",
"rows": [
{
"model": "Hebrish Sabich v1.0",
"termRecall": 73.8,
"unseen": 68.4,
"wer": 3.3,
"ours": true
},
{
"model": "Hebrish Jachnun v1.0",
"termRecall": 73.3,
"unseen": 70.1,
"wer": 3.3,
"ours": true
},
{
"model": "Hebrish Shakshuka PoC-2 (Nemotron streaming)",
"termRecall": 62.2,
"unseen": 54.7,
"wer": 4.7,
"ours": true
},
{
"model": "Whisper Hebrish v1",
"termRecall": 54.1,
"unseen": 56.4,
"wer": 6.5,
"ours": true,
"note": "known training-data overlap"
},
{
"model": "Whisper Large V3",
"termRecall": 45.9,
"unseen": 46.2,
"wer": 7.5,
"ours": false
},
{
"model": "Whisper Large V3 Turbo",
"termRecall": 40.1,
"unseen": 39.3,
"wer": 8.9,
"ours": false
},
{
"model": "Parakeet TDT 0.6B v2",
"termRecall": 32.6,
"unseen": 32.5,
"wer": 9.5,
"ours": false
},
{
"model": "Parakeet TDT 0.6B v3",
"termRecall": 31.4,
"unseen": 31.6,
"wer": 10.5,
"ours": false
}
]
}
},
"benchmarks": {
"repo": "https://github.com/danielrosehill/My-Benchmarks",
"space": "https://huggingface.co/spaces/danielrosehill/My-Benchmarks",
"page": "https://ai.danielrosehill.com/benchmarks",
"fetched_live": true,
"about": "Registry of Daniel Rosehill's public benchmarks and evals. Source of truth for the README table (scripts/build_table.py), the Hugging Face Space danielrosehill/My-Benchmarks and danielrosehill.com/ai/benchmarks. Edit this file, then run the script. Private repos are never listed.",
"updated": "2026-10-21",
"stages": {
"results": {
"label": "Results",
"meaning": "Run, with published results"
},
"built": {
"label": "Built",
"meaning": "Repo, test material and harness exist; not yet fully run"
},
"prompts": {
"label": "Prompts only",
"meaning": "Repo and published test prompts; no harness yet"
},
"planned": {
"label": "Planned",
"meaning": "Design only: a plan file, or a public repo with no test prompts yet"
}
},
"categories": {
"behaviour": "Model behaviour & AI safety",
"speech": "Speech recognition & audio",
"image": "Image generation",
"hardware": "Local inference & hardware"
},
"benchmarks": [
{
"id": "english-hebrew-codeswitching-asr",
"name": "English–Hebrew Code-Switching ASR Benchmark",
"category": "speech",
"stage": "results",
"flagship": true,
"summary": "Speech-to-text on English speech that contains Hebrew words: a frozen test set, runner, scorer and public leaderboard, comparing stock models with code-switching fine-tunes.",
"repo": "https://github.com/Hebrish-ASR/English-Hebrew-Codeswitching-ASR-Benchmark",
"dataset": "https://huggingface.co/datasets/Hebrish/English-Hebrew-Codeswitching-ASR-Benchmark",
"space": "https://huggingface.co/spaces/danielrosehill/English-Hebrew-Codeswitching-ASR-Leaderboard",
"date": "2026-10"
},
{
"id": "the-pc-bench",
"name": "The PC Bench",
"category": "behaviour",
"stage": "prompts",
"flagship": true,
"summary": "Does the model scold you? Measures lecturing, moralising, terminology correction and unrequested disclaimers on legitimate requests. 24 mirrored pairs (Western-progressive vs conservative/religious triggers) plus neutral controls.",
"repo": "https://github.com/danielrosehill/The-PC-Bench",
"dataset": "https://huggingface.co/datasets/danielrosehill/The-PC-Bench",
"prompts": 54,
"date": "2026-10-19"
},
{
"id": "propaganda-bait-eval",
"name": "Propaganda Bait Eval",
"category": "behaviour",
"stage": "built",
"flagship": true,
"summary": "Feeds models loaded political premises across six series (CCP, MAGA, pro-Israel, pro-Palestine, Kremlin, US-progressive) and scores how readily each framing is repeated, with per-series selectivity.",
"repo": "https://github.com/danielrosehill/Propaganda-Bait-Eval",
"dataset": "https://huggingface.co/datasets/danielrosehill/Propaganda-Bait-Eval",
"prompts": 72,
"date": "2026-10-19"
},
{
"id": "conspiracy-endorsement-eval",
"name": "Conspiracy Endorsement Eval",
"category": "behaviour",
"stage": "results",
"summary": "Whether a model pushes back on, endorses or encourages 45 conspiracy theories, each asked in a curious and a believer framing. First hand-graded run (on-air-01): 7 models.",
"repo": "https://github.com/danielrosehill/Conspiracy-Endorsement-Eval",
"dataset": "https://huggingface.co/datasets/danielrosehill/Conspiracy-Endorsement-Eval",
"prompts": 98,
"date": "2026-10-19"
},
{
"id": "israel-palestine-framing-eval",
"name": "Israel–Palestine Framing Eval",
"category": "behaviour",
"stage": "results",
"summary": "Whether a model treats mirrored pro-Israel and pro-Palestinian framings of the same events differently: loaded premises, framing pairs and open-neutral questions. First graded run (on-air-01): 6 models.",
"repo": "https://github.com/danielrosehill/Israel-Palestine-Framing-Eval",
"dataset": "https://huggingface.co/datasets/danielrosehill/Israel-Palestine-Framing-Eval",
"prompts": 70,
"date": "2026-10-20"
},
{
"id": "american-centrism-eval",
"name": "American Centrism Eval",
"category": "behaviour",
"stage": "prompts",
"summary": "Whether a model assumes the user is in the US when nothing says so: geography, units, spelling, law and culture defaults, with controls.",
"repo": "https://github.com/danielrosehill/American-Centrism-Eval",
"dataset": "https://huggingface.co/datasets/danielrosehill/American-Centrism-Eval",
"prompts": 48,
"date": "2026-10-19"
},
{
"id": "dan-eval",
"name": "Dan-Eval",
"category": "behaviour",
"stage": "built",
"summary": "A composite of the model-behaviour checks I run before adopting a model for an app, giving one scorecard per model. Each component lives in its own repo.",
"repo": "https://github.com/danielrosehill/Dan-Eval",
"date": "2026-10-19"
},
{
"id": "audio-understanding-bitrate-eval",
"name": "Audio Understanding Bitrate Eval",
"category": "speech",
"stage": "results",
"summary": "How MP3 bitrate affects transcription accuracy across every audio-input LLM on OpenRouter.",
"repo": "https://github.com/danielrosehill/Audio-Understanding-Bitrate-Eval-0426",
"dataset": "https://huggingface.co/datasets/danielrosehill/Audio-Understanding-Bitrate-Eval-0426",
"date": "2026-04"
},
{
"id": "whisper-fine-tune-accuracy-eval",
"name": "Whisper Fine-Tune Accuracy Eval",
"category": "speech",
"stage": "results",
"summary": "WER of personal Whisper fine-tunes vs stock Whisper at each model size. Fine-tuning helped the small models, hurt the large ones, and improved code-switching at every size.",
"repo": "https://github.com/danielrosehill/Whisper-Fine-Tune-Accuracy-Eval"
},
{
"id": "whisper-wpm-background-noise-eval",
"name": "Whisper WPM & Background Noise Eval",
"category": "speech",
"stage": "results",
"summary": "How speaking pace and background noise affect ASR word error rate.",
"repo": "https://github.com/danielrosehill/Whisper-WPM-Background-Noise-Eval",
"dataset": "https://huggingface.co/datasets/danielrosehill/ASR-WPM-And-Background-Noise-Eval"
},
{
"id": "long-form-audio-eval",
"name": "Long-Form Audio Eval",
"category": "speech",
"stage": "results",
"summary": "STT models on a 27-minute podcast against a professional reference transcript: WER, CER and punctuation accuracy.",
"repo": "https://github.com/danielrosehill/Long-Form-Audio-Eval"
},
{
"id": "one-shot-transcription-microphone-eval",
"name": "One-Shot Transcription Microphone Eval",
"category": "speech",
"stage": "results",
"summary": "STT accuracy across a range of microphones recording the same material.",
"repo": "https://github.com/danielrosehill/One-Shot-Transcription-Microphone-Eval"
},
{
"id": "transcription-cleanup-eval",
"name": "Transcription Cleanup Eval",
"category": "speech",
"stage": "results",
"summary": "Cloud audio-multimodal models transcribing and cleaning up a recording in one step. Subjective grading.",
"repo": "https://github.com/danielrosehill/Transcription-Cleanup-Eval-1225",
"date": "2025-12"
},
{
"id": "local-asr-stt-benchmark",
"name": "Local ASR STT Benchmark",
"category": "speech",
"stage": "results",
"summary": "Whisper model sizes in Speech Note on Ubuntu with AMD/ROCm. A rough first-pass comparison.",
"repo": "https://github.com/danielrosehill/Local-ASR-STT-Benchmark"
},
{
"id": "hebrew-image-generation-eval",
"name": "Hebrew Image Generation Eval",
"category": "image",
"stage": "results",
"summary": "Twelve text-to-image models scored pass/fail on rendering Hebrew text correctly.",
"repo": "https://github.com/danielrosehill/Hebrew-Image-Generation-Eval",
"dataset": "https://huggingface.co/datasets/danielrosehill/Hebrew-Image-Eval-111225",
"date": "2025-12"
},
{
"id": "gemma-4-benchmarks",
"name": "Gemma 4 Benchmarks",
"category": "hardware",
"stage": "results",
"summary": "Gemma 4 E4B vs Qwen3-VL 8B via Ollama on a CPU-only i7-12700F with 64 GB RAM.",
"repo": "https://github.com/danielrosehill/Gemma-4-Benchmarks-0426",
"date": "2026-04"
},
{
"id": "workstation-benchmarks",
"name": "Workstation Benchmarks",
"category": "hardware",
"stage": "results",
"summary": "Desktop workstation benchmarks (i7-12700F, RX 7700 XT, 64 GB DDR5, Ubuntu 25.10) with the full system inventory.",
"repo": "https://github.com/danielrosehill/Workstation-Benchmarks-0226",
"date": "2026-02"
},
{
"id": "asian-model-state-censorship",
"name": "State Censorship in Asian Models",
"category": "behaviour",
"stage": "planned",
"summary": "State-aligned censorship and framing in Chinese and other Asian models.",
"plan": "https://github.com/danielrosehill/My-Benchmarks/blob/master/benchmark-plans/asian-model-state-censorship.md"
},
{
"id": "us-partisan-framing-eval",
"name": "US Partisan Framing Eval",
"category": "behaviour",
"stage": "prompts",
"summary": "Asked a neutral question about a divisive US issue, does the model answer with the popular Democratic or Republican framing, and on foreign policy, hawkish or dovish? Blinded anchor-based grading, 14 topics plus factual controls.",
"repo": "https://github.com/danielrosehill/US-Partisan-Framing-Eval",
"dataset": "https://huggingface.co/datasets/danielrosehill/US-Partisan-Framing-Eval",
"prompts": 54,
"date": "2026-10-19"
},
{
"id": "last-hour-news-recall",
"name": "Last-Hour News Recall",
"category": "behaviour",
"stage": "planned",
"summary": "Recall of the past hour's news by models that claim real-time retrieval.",
"plan": "https://github.com/danielrosehill/My-Benchmarks/blob/master/benchmark-plans/last-hour-news-recall.md"
},
{
"id": "last-24-hours-news-recall",
"name": "Last-24-Hours News Recall",
"category": "behaviour",
"stage": "planned",
"summary": "Recall of the past day's news by models that claim real-time retrieval.",
"plan": "https://github.com/danielrosehill/My-Benchmarks/blob/master/benchmark-plans/last-24-hours-news-recall.md"
}
]
},
"collections": [
{
"title": "Code Switching (English and Hebrew)",
"url": "https://huggingface.co/collections/danielrosehill/code-switching-english-and-hebrew-all-resources-6acfdcb1e11911797af4c3b3"
},
{
"title": "Hebrew ASR",
"url": "https://huggingface.co/collections/danielrosehill/hebrew-asr-and-audio-to-text-69cbb72271b4fe75732a465a"
},
{
"title": "Whisper Hebrish",
"url": "https://huggingface.co/collections/danielrosehill/whisper-hebrish"
},
{
"title": "Android STT favourites",
"url": "https://huggingface.co/collections/danielrosehill/android-stt-favorites"
},
{
"title": "Small ASR models",
"url": "https://huggingface.co/collections/danielrosehill/small-asr-models"
},
{
"title": "STT evaluations",
"url": "https://huggingface.co/collections/danielrosehill/stt-evaluations"
},
{
"title": "STT cleanup (text post-processing)",
"url": "https://huggingface.co/collections/danielrosehill/stt-cleanup-text-postprocessing"
},
{
"title": "My public audio datasets",
"url": "https://huggingface.co/collections/danielrosehill/my-public-audio-datasets"
},
{
"title": "Hebrew TTS",
"url": "https://huggingface.co/collections/danielrosehill/hebrew-tts"
}
],
"huggingface": [
{
"name": "English-Hebrew-Codeswitching-Sentences",
"kind": "dataset",
"url": "https://huggingface.co/datasets/danielrosehill/English-Hebrew-Codeswitching-Sentences",
"page": "https://ai.danielrosehill.com/datasets/english-hebrew-codeswitching-sentences",
"description": "English-Hebrew Code-Switching Sentences English sentences with everyday Hebrew words and Israeli names written in Latin letters, read aloud by one English speaker living in…",
"speech_or_language": true,
"tags": [
"task_categories:automatic-speech-recognition",
"language:en",
"language:he",
"license:cc-by-4.0",
"size_categories:1K