diff --git a/.cspell-wordlist.txt b/.cspell-wordlist.txt index 964521ab13..ea3e2b263a 100644 --- a/.cspell-wordlist.txt +++ b/.cspell-wordlist.txt @@ -314,3 +314,5 @@ Partitioner denoised ttfa TTFA +phonemized +subsentences diff --git a/.github/workflows/clang-tidy.yml b/.github/workflows/clang-tidy.yml index d0ecea74fd..8f4af4a919 100644 --- a/.github/workflows/clang-tidy.yml +++ b/.github/workflows/clang-tidy.yml @@ -24,6 +24,11 @@ jobs: steps: - name: Checkout uses: actions/checkout@v6 + with: + # The phonemis submodule carries headers that cpp/extensions/speech + # includes; without it those sources fail to parse and clang-tidy + # reports misleading findings on the broken AST. + submodules: true - name: Setup uses: ./.github/actions/setup diff --git a/apps/speech/app/_layout.tsx b/apps/speech/app/_layout.tsx index 1cf61aa30b..e98be4cd43 100644 --- a/apps/speech/app/_layout.tsx +++ b/apps/speech/app/_layout.tsx @@ -48,6 +48,13 @@ export default function Layout() { title: 'Text-to-Speech (SuperTonic)', }} /> + ); } diff --git a/apps/speech/app/index.tsx b/apps/speech/app/index.tsx index c0dd5ee7b7..61cd5f6097 100644 --- a/apps/speech/app/index.tsx +++ b/apps/speech/app/index.tsx @@ -29,6 +29,12 @@ export default function Home() { router.navigate('text-to-speech/')}> Text-to-Speech (SuperTonic) + router.navigate('kokoro-text-to-speech/')} + > + Text-to-Speech (Kokoro) + ); diff --git a/apps/speech/app/kokoro-text-to-speech/index.tsx b/apps/speech/app/kokoro-text-to-speech/index.tsx new file mode 100644 index 0000000000..121bfa702d --- /dev/null +++ b/apps/speech/app/kokoro-text-to-speech/index.tsx @@ -0,0 +1,350 @@ +import React, { useCallback, useEffect, useRef, useState } from 'react'; +import { View, Text, StyleSheet, ScrollView, TextInput } from 'react-native'; +import { + useTextToSpeech, + models, + KOKORO_SAMPLE_RATE, + type KokoroTtsModel, +} from 'react-native-executorch'; +import { AudioContext, type AudioBufferQueueSourceNode } from 'react-native-audio-api'; + +import ScreenWrapper from '../../components/ScreenWrapper'; +import { ModelPicker } from '../../components/ModelPicker'; +import { ModelStatus } from '../../components/ModelStatus'; +import { Button } from '../../components/Button'; +import { theme } from '../../theme'; + +const LANGUAGE_OPTIONS = [ + { label: 'English (US)', value: 'EN_US' as const }, + { label: 'English (GB)', value: 'EN_GB' as const }, + { label: 'Spanish', value: 'ES' as const }, + { label: 'French', value: 'FR' as const }, + { label: 'Italian', value: 'IT' as const }, + { label: 'Portuguese', value: 'PT' as const }, + { label: 'Hindi', value: 'HI' as const }, + { label: 'Polish', value: 'PL' as const }, + { label: 'German', value: 'DE' as const }, +]; + +type KokoroLanguage = (typeof LANGUAGE_OPTIONS)[number]['value']; + +// cspell:disable +const SAMPLE_TEXTS: Record = { + EN_US: + 'Kokoro is a compact text-to-speech model that runs entirely on your device. ' + + 'It converts text into phonemes first, then predicts how long each sound should last, ' + + 'and finally synthesizes the waveform. No internet connection is required.', + EN_GB: + 'Kokoro is a compact text-to-speech model that runs entirely on your device, ' + + 'converting text into phonemes before synthesising the waveform.', + ES: 'Kokoro es un modelo de síntesis de voz que funciona completamente en tu dispositivo, sin conexión a internet.', + FR: 'Kokoro est un modèle de synthèse vocale qui fonctionne entièrement sur votre appareil, sans connexion internet.', + IT: 'Kokoro è un modello di sintesi vocale che funziona interamente sul tuo dispositivo, senza connessione a internet.', + PT: 'Kokoro é um modelo de síntese de voz que funciona inteiramente no seu dispositivo, sem ligação à internet.', + HI: 'कोकोरो एक छोटा टेक्स्ट-टू-स्पीच मॉडल है जो पूरी तरह से आपके डिवाइस पर चलता है।', + PL: 'Kokoro to niewielki model syntezy mowy, który działa w całości na twoim urządzeniu, bez połączenia z internetem.', + DE: 'Kokoro ist ein kompaktes Sprachsynthesemodell, das vollständig auf deinem Gerät läuft, ganz ohne Internetverbindung.', +}; +// cspell:enable + +const SPEED_OPTIONS = [ + { label: '0.8x', value: 0.8 }, + { label: '0.9x', value: 0.9 }, + { label: '1.0x', value: 1.0 }, + { label: '1.1x', value: 1.1 }, + { label: '1.25x', value: 1.25 }, +]; + +function KokoroContent() { + const [language, setLanguage] = useState('EN_US'); + const [text, setText] = useState(SAMPLE_TEXTS.EN_US); + const [speed, setSpeed] = useState(1.0); + const [isSynthesizing, setIsSynthesizing] = useState(false); + const [isPlaying, setIsPlaying] = useState(false); + const [chunkProgress, setChunkProgress] = useState(null); + const [runError, setRunError] = useState(null); + const [totalDuration, setTotalDuration] = useState(null); + + const model = models.textToSpeech.KOKORO[language].XNNPACK_FP32 as KokoroTtsModel; + const voiceNames = Object.keys(model.voices); + const [voice, setVoice] = useState(voiceNames[0]!); + + const audioCtxRef = useRef(null); + const queueSourceRef = useRef(null); + + const { isReady, downloadProgress, error, synthesize, synthesizeStop } = useTextToSpeech(model); + + useEffect(() => { + setVoice(Object.keys(models.textToSpeech.KOKORO[language].XNNPACK_FP32.voices)[0]!); + setText(SAMPLE_TEXTS[language]); + }, [language]); + + const getAudioContext = useCallback(async () => { + if (!audioCtxRef.current || audioCtxRef.current.state === 'closed') { + audioCtxRef.current = new AudioContext({ sampleRate: KOKORO_SAMPLE_RATE }); + } + if (audioCtxRef.current.state === 'suspended') { + await audioCtxRef.current.resume(); + } + return audioCtxRef.current; + }, []); + + const stopAudioQueue = useCallback(() => { + if (queueSourceRef.current) { + queueSourceRef.current.clearBuffers(); + queueSourceRef.current.stop(); + queueSourceRef.current = null; + } + setIsPlaying(false); + }, []); + + useEffect(() => { + return () => { + synthesizeStop?.(); + stopAudioQueue(); + if (audioCtxRef.current) { + audioCtxRef.current.close().catch(() => {}); + audioCtxRef.current = null; + } + }; + }, [stopAudioQueue, synthesizeStop]); + + const preparePlaybackSource = useCallback(async () => { + stopAudioQueue(); + const ctx = await getAudioContext(); + + const source = ctx.createBufferQueueSource(); + source.connect(ctx.destination); + source.onBufferEnded = (event) => { + if (event.isLastBufferInQueue) setIsPlaying(false); + }; + queueSourceRef.current = source; + return { ctx, source }; + }, [getAudioContext, stopAudioQueue]); + + const handleSynthesize = async () => { + if (!synthesize || isSynthesizing || !text.trim()) return; + + setRunError(null); + setChunkProgress(null); + setTotalDuration(null); + setIsSynthesizing(true); + + try { + const { ctx, source } = await preparePlaybackSource(); + let durationSum = 0; + let started = false; + + for await (const chunk of synthesize(text, { voice, speed })) { + setChunkProgress( + `Chunk ${chunk.chunkIndex + 1}/${chunk.totalChunks} (${chunk.duration.toFixed(1)}s)` + ); + durationSum += chunk.duration; + + const buffer = ctx.createBuffer(1, chunk.audio.length, KOKORO_SAMPLE_RATE); + buffer.copyToChannel(chunk.audio as Float32Array, 0); + source.enqueueBuffer(buffer); + + if (!started) { + started = true; + setIsPlaying(true); + source.start(0, 0); + } + } + + setTotalDuration(durationSum); + setChunkProgress(null); + } catch (err) { + setRunError(err instanceof Error ? err.message : String(err)); + } finally { + setIsSynthesizing(false); + } + }; + + const handleStopPlayback = () => { + synthesizeStop?.(); + stopAudioQueue(); + setIsSynthesizing(false); + }; + + const isBusy = isSynthesizing || isPlaying; + + return ( + + + Kokoro Text-to-Speech + + Phoneme-driven on-device speech synthesis. Pick a language to load its model, phonemizer + assets and voices. + + + + + {runError && ( + + {runError} + + )} + + + ({ ...l, disabled: isBusy }))} + selectedValue={language} + onValueChange={setLanguage} + /> + ({ label: name, value: name, disabled: isBusy }))} + selectedValue={voice} + onValueChange={setVoice} + /> + ({ ...s, disabled: isBusy }))} + selectedValue={speed} + onValueChange={setSpeed} + /> + + + + Input Text + + + + + + {!isPlaying ? ( +