WhisperLiveKit/benchmarks/h100_scatter/bench_scatter_en_aware.json at main · QuentinFuxa/WhisperLiveKit · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
{
  "system_info": {
    "platform": "Linux-5.15.0-177-generic-x86_64-with-glibc2.35",
    "machine": "x86_64",
    "cpu": "H100 80GB HBM3",
    "gpu": "NVIDIA H100 80GB HBM3"
  },
  "lang": "en",
  "mode": "aware",
  "speed": 1.0,
  "n_samples": 4,
  "sample_names": [
    "en_librivox_continuous",
    "en_30s",
    "en_60s",
    "en_120s"
  ],
  "total_audio_s": 390.0,
  "total_benchmark_time_s": 4747.0,
  "results": [
    {
      "label": "fw LA base",
      "backend": "faster-whisper",
      "model_size": "base",
      "policy": "localagreement",
      "color": "#4a9eff",
      "marker": "o",
      "size": 100,
      "rtf": 0.3673,
      "wer_pct": 24.6,
      "n_samples": 4
    },
    {
      "label": "fw LA small",
      "backend": "faster-whisper",
      "model_size": "small",
      "policy": "localagreement",
      "color": "#4a9eff",
      "marker": "o",
      "size": 220,
      "rtf": 0.4804,
      "wer_pct": 8.2,
      "n_samples": 4
    },
    {
      "label": "fw LA turbo",
      "backend": "faster-whisper",
      "model_size": "large-v3-turbo",
      "policy": "localagreement",
      "color": "#4a9eff",
      "marker": "o",
      "size": 340,
      "rtf": 0.3032,
      "wer_pct": 5.3,
      "n_samples": 4
    },
    {
      "label": "fw SS base",
      "backend": "faster-whisper",
      "model_size": "base",
      "policy": "simulstreaming",
      "color": "#4a9eff",
      "marker": "s",
      "size": 100,
      "rtf": 0.0968,
      "wer_pct": 9.5,
      "n_samples": 4
    },
    {
      "label": "fw SS small",
      "backend": "faster-whisper",
      "model_size": "small",
      "policy": "simulstreaming",
      "color": "#4a9eff",
      "marker": "s",
      "size": 220,
      "rtf": 0.1363,
      "wer_pct": 7.0,
      "n_samples": 4
    },
    {
      "label": "fw SS turbo",
      "backend": "faster-whisper",
      "model_size": "large-v3-turbo",
      "policy": "simulstreaming",
      "color": "#4a9eff",
      "marker": "s",
      "size": 340,
      "rtf": 0.1442,
      "wer_pct": 18.1,
      "n_samples": 4
    },
    {
      "label": "qwen3 causal",
      "backend": "qwen3-streaming",
      "model_size": "0.6b",
      "policy": "",
      "color": "#b06ee8",
      "marker": "D",
      "size": 220,
      "rtf": 0.0647,
      "wer_pct": 7.3,
      "n_samples": 4
    },
    {
      "label": "qwen3 windowed",
      "backend": "qwen3-streaming",
      "model_size": "0.6b",
      "policy": "",
      "color": "#8e44ad",
      "marker": "D",
      "size": 220,
      "rtf": 0.0567,
      "wer_pct": 5.2,
      "n_samples": 4
    }
  ]
}