{ "metadata": { "benchmark": "VoiceCodeBench", "slug": "voice-code-bench", "description": "Can speech recognition preserve exact structured values in workplace audio?", "benchmark_id": "voice_code_bench", "family": "voice_code_bench", "version": "1", "updated": "2026-09-24", "dataset_type": "public", "industry": "voice", "tasks": { "overall": "Overall", "ctem": "CTEM", "wer": "WER" }, "models": [ "assemblyai/universal-3-5-pro", "azure_speech/universal-language-model", "cartesia/ink-2", "cohere/cohere-transcribe-03-2026", "deepgram/flux-general-en", "deepgram/nova-3-streaming", "elevenlabs/scribe_v2_realtime", "google_cloud/chirp_3", "grok/grok-voice-transcribe-1.0", "grok/grok-voice-transcribe-2.0", "groq/whisper-large-v3", "groq/whisper-large-v3-turbo", "inworld/inworld-stt-1", "meta/muse_voice_transcribe", "mistral/voxtral-mini-2602", "mistral/voxtral-mini-transcribe-realtime-2602", "openai/gpt-4o-mini-transcribe-streaming-response", "openai/gpt-4o-transcribe", "openai/gpt-live-transcribe", "reson8/resonant-1" ], "partners": [], "showBadge": false, "visible": true, "use_cost_per_test": true, "accuracy_label": "TSR (%)", "cost_label": "STT Cost / Task", "cost_decimals": 4, "show_stderr": false, "runner": "external", "mode": "one-shot", "archived": false, "partner": true, "total_models": 20 }, "tasks": { "overall": { "openai/gpt-live-transcribe": { "accuracy": 67.667, "latency": 70.342, "stderr": 0.0, "cost_per_test": 0.018995, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-voice-transcribe-2.0": { "accuracy": 65.333, "latency": 67.479, "stderr": 0.0, "cost_per_test": 0.003724, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "cartesia/ink-2": { "accuracy": 62.0, "latency": 69.197, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cartesia", "harness": null }, "openai/gpt-4o-transcribe": { "accuracy": 59.667, "latency": 2.661, "stderr": 0.0, "cost_per_test": 0.003121, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepgram/nova-3-streaming": { "accuracy": 59.333, "latency": 67.543, "stderr": 0.0, "cost_per_test": 0.005363, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Deepgram", "harness": null }, "reson8/resonant-1": { "accuracy": 59.0, "latency": 1.446, "stderr": 0.0, "cost_per_test": 0.007685, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reson8", "harness": null }, "inworld/inworld-stt-1": { "accuracy": 57.333, "latency": 70.47, "stderr": 0.0, "cost_per_test": 0.002705, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Inworld", "harness": null }, "groq/whisper-large-v3-turbo": { "accuracy": 57.0, "latency": 2.946, "stderr": 0.0, "cost_per_test": 0.000745, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Groq", "harness": null }, "meta/muse_voice_transcribe": { "accuracy": 57.0, "latency": 67.168, "stderr": 0.0, "cost_per_test": 0.003354, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "deepgram/flux-general-en": { "accuracy": 55.667, "latency": 68.8, "stderr": 0.0, "cost_per_test": 0.007172, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Deepgram", "harness": null }, "google_cloud/chirp_3": { "accuracy": 55.333, "latency": 68.097, "stderr": 0.0, "cost_per_test": 0.018008, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google Cloud", "harness": null }, "elevenlabs/scribe_v2_realtime": { "accuracy": 55.333, "latency": 77.696, "stderr": 0.0, "cost_per_test": 0.007263, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Elevenlabs", "harness": null }, "groq/whisper-large-v3": { "accuracy": 54.333, "latency": 2.197, "stderr": 0.0, "cost_per_test": 0.002067, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Groq", "harness": null }, "mistral/voxtral-mini-2602": { "accuracy": 53.0, "latency": 1.885, "stderr": 0.0, "cost_per_test": 0.003352, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral", "harness": null }, "grok/grok-voice-transcribe-1.0": { "accuracy": 52.0, "latency": 67.789, "stderr": 0.0, "cost_per_test": 0.003724, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "openai/gpt-4o-mini-transcribe-streaming-response": { "accuracy": 51.667, "latency": 2.222, "stderr": 0.0, "cost_per_test": 0.001565, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "mistral/voxtral-mini-transcribe-realtime-2602": { "accuracy": 46.0, "latency": 15.376, "stderr": 0.0, "cost_per_test": 0.006704, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral", "harness": null }, "cohere/cohere-transcribe-03-2026": { "accuracy": 44.667, "latency": 1.945, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "azure_speech/universal-language-model": { "accuracy": 37.333, "latency": 30.615, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Azure_Speech", "harness": null }, "assemblyai/universal-3-5-pro": { "accuracy": 30.333, "latency": 67.954, "stderr": 0.0, "cost_per_test": 0.008451, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Assemblyai", "harness": null } }, "ctem": { "openai/gpt-live-transcribe": { "accuracy": 91.363, "latency": 70.342, "stderr": 0.0, "cost_per_test": 0.018995, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "grok/grok-voice-transcribe-2.0": { "accuracy": 90.621, "latency": 67.479, "stderr": 0.0, "cost_per_test": 0.003724, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "cartesia/ink-2": { "accuracy": 89.541, "latency": 69.197, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cartesia", "harness": null }, "openai/gpt-4o-transcribe": { "accuracy": 88.731, "latency": 2.661, "stderr": 0.0, "cost_per_test": 0.003121, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "reson8/resonant-1": { "accuracy": 88.664, "latency": 1.446, "stderr": 0.0, "cost_per_test": 0.007685, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reson8", "harness": null }, "deepgram/nova-3-streaming": { "accuracy": 87.989, "latency": 67.543, "stderr": 0.0, "cost_per_test": 0.005363, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Deepgram", "harness": null }, "inworld/inworld-stt-1": { "accuracy": 87.989, "latency": 70.47, "stderr": 0.0, "cost_per_test": 0.002705, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Inworld", "harness": null }, "meta/muse_voice_transcribe": { "accuracy": 87.854, "latency": 67.168, "stderr": 0.0, "cost_per_test": 0.003354, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "groq/whisper-large-v3-turbo": { "accuracy": 87.584, "latency": 2.946, "stderr": 0.0, "cost_per_test": 0.000745, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Groq", "harness": null }, "groq/whisper-large-v3": { "accuracy": 87.247, "latency": 2.197, "stderr": 0.0, "cost_per_test": 0.002067, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Groq", "harness": null }, "mistral/voxtral-mini-2602": { "accuracy": 86.977, "latency": 1.885, "stderr": 0.0, "cost_per_test": 0.003352, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral", "harness": null }, "google_cloud/chirp_3": { "accuracy": 86.842, "latency": 68.097, "stderr": 0.0, "cost_per_test": 0.018008, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google Cloud", "harness": null }, "elevenlabs/scribe_v2_realtime": { "accuracy": 86.842, "latency": 77.696, "stderr": 0.0, "cost_per_test": 0.007263, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Elevenlabs", "harness": null }, "openai/gpt-4o-mini-transcribe-streaming-response": { "accuracy": 86.64, "latency": 2.222, "stderr": 0.0, "cost_per_test": 0.001565, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "deepgram/flux-general-en": { "accuracy": 86.572, "latency": 68.8, "stderr": 0.0, "cost_per_test": 0.007172, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Deepgram", "harness": null }, "grok/grok-voice-transcribe-1.0": { "accuracy": 84.683, "latency": 67.789, "stderr": 0.0, "cost_per_test": 0.003724, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "cohere/cohere-transcribe-03-2026": { "accuracy": 83.873, "latency": 1.945, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "mistral/voxtral-mini-transcribe-realtime-2602": { "accuracy": 83.333, "latency": 15.376, "stderr": 0.0, "cost_per_test": 0.006704, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral", "harness": null }, "azure_speech/universal-language-model": { "accuracy": 80.702, "latency": 30.615, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Azure_Speech", "harness": null }, "assemblyai/universal-3-5-pro": { "accuracy": 77.126, "latency": 67.954, "stderr": 0.0, "cost_per_test": 0.008451, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Assemblyai", "harness": null } }, "wer": { "grok/grok-voice-transcribe-2.0": { "accuracy": 10.53, "latency": 67.479, "stderr": 0.0, "cost_per_test": 0.003724, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "azure_speech/universal-language-model": { "accuracy": 10.397, "latency": 30.615, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Azure_Speech", "harness": null }, "deepgram/nova-3-streaming": { "accuracy": 9.547, "latency": 67.543, "stderr": 0.0, "cost_per_test": 0.005363, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Deepgram", "harness": null }, "deepgram/flux-general-en": { "accuracy": 9.384, "latency": 68.8, "stderr": 0.0, "cost_per_test": 0.007172, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Deepgram", "harness": null }, "cartesia/ink-2": { "accuracy": 9.249, "latency": 69.197, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cartesia", "harness": null }, "grok/grok-voice-transcribe-1.0": { "accuracy": 8.965, "latency": 67.789, "stderr": 0.0, "cost_per_test": 0.003724, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "SpaceXAI", "harness": null }, "inworld/inworld-stt-1": { "accuracy": 8.108, "latency": 70.47, "stderr": 0.0, "cost_per_test": 0.002705, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Inworld", "harness": null }, "meta/muse_voice_transcribe": { "accuracy": 7.637, "latency": 67.168, "stderr": 0.0, "cost_per_test": 0.003354, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Meta", "harness": null }, "cohere/cohere-transcribe-03-2026": { "accuracy": 6.986, "latency": 1.945, "stderr": 0.0, "cost_per_test": null, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Cohere", "harness": null }, "groq/whisper-large-v3-turbo": { "accuracy": 6.711, "latency": 2.946, "stderr": 0.0, "cost_per_test": 0.000745, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Groq", "harness": null }, "google_cloud/chirp_3": { "accuracy": 6.628, "latency": 68.097, "stderr": 0.0, "cost_per_test": 0.018008, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Google Cloud", "harness": null }, "mistral/voxtral-mini-transcribe-realtime-2602": { "accuracy": 6.602, "latency": 15.376, "stderr": 0.0, "cost_per_test": 0.006704, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral", "harness": null }, "reson8/resonant-1": { "accuracy": 6.399, "latency": 1.446, "stderr": 0.0, "cost_per_test": 0.007685, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Reson8", "harness": null }, "openai/gpt-live-transcribe": { "accuracy": 5.916, "latency": 70.342, "stderr": 0.0, "cost_per_test": 0.018995, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "groq/whisper-large-v3": { "accuracy": 5.753, "latency": 2.197, "stderr": 0.0, "cost_per_test": 0.002067, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Groq", "harness": null }, "mistral/voxtral-mini-2602": { "accuracy": 5.542, "latency": 1.885, "stderr": 0.0, "cost_per_test": 0.003352, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Mistral", "harness": null }, "assemblyai/universal-3-5-pro": { "accuracy": 5.518, "latency": 67.954, "stderr": 0.0, "cost_per_test": 0.008451, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Assemblyai", "harness": null }, "elevenlabs/scribe_v2_realtime": { "accuracy": 5.104, "latency": 77.696, "stderr": 0.0, "cost_per_test": 0.007263, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "Elevenlabs", "harness": null }, "openai/gpt-4o-mini-transcribe-streaming-response": { "accuracy": 4.839, "latency": 2.222, "stderr": 0.0, "cost_per_test": 0.001565, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null }, "openai/gpt-4o-transcribe": { "accuracy": 4.041, "latency": 2.661, "stderr": 0.0, "cost_per_test": 0.003121, "temperature": null, "top_p": null, "max_output_tokens": null, "reasoning": null, "reasoning_effort": null, "verbosity": null, "compute_effort": null, "provider": "OpenAI", "harness": null } } } }