{
  "source": "https://ananasdda.github.io/pocket-ai-lab/benchmarks/",
  "generated": "2026-10-27",
  "results": [
    {
      "model": "Gemma 2 2B",
      "modelId": "mlx-community/gemma-2-2b-it-4bit",
      "backend": "MLX",
      "quantization": "4bit",
      "parameters": "2B",
      "device": "iPhone Air",
      "chip": "A19",
      "iOS": "26.6.1",
      "appVersion": "1.0.1",
      "date": "2026-09-03T22:10:23Z",
      "genTokPerSec": 29.66,
      "genStdDev": 1.37,
      "prefillTokPerSecEst": 286.9,
      "pp512": 272.4,
      "tg128": 24.73,
      "loadSeconds": 3.7,
      "peakMemoryMB": 2130,
      "throttled": false,
      "throttleDropPercent": 7.8,
      "suiteVersion": 2,
      "tasksPassed": 26,
      "tasksTotal": 46,
      "score": 56.5,
      "mathPassed": 3,
      "instructionsPassed": 10,
      "knowledgePassed": 4,
      "retrievalPassed": 4,
      "multilingualPassed": 5
    },
    {
      "model": "Phi-3.5 Mini",
      "modelId": "mlx-community/Phi-3.5-mini-instruct-4bit",
      "backend": "MLX",
      "quantization": "4bit",
      "parameters": "3.8B",
      "device": "iPhone Air",
      "chip": "A19",
      "iOS": "26.6.1",
      "appVersion": "1.0.1",
      "date": "2026-09-03T22:21:23Z",
      "genTokPerSec": 19.91,
      "genStdDev": 2.63,
      "prefillTokPerSecEst": 201.3,
      "pp512": 133.5,
      "tg128": 14.17,
      "loadSeconds": 0.74,
      "peakMemoryMB": 3192,
      "throttled": true,
      "throttleDropPercent": 23.4,
      "suiteVersion": 2,
      "tasksPassed": 33,
      "tasksTotal": 46,
      "score": 71.7,
      "mathPassed": 4,
      "instructionsPassed": 14,
      "knowledgePassed": 6,
      "retrievalPassed": 4,
      "multilingualPassed": 5
    },
    {
      "model": "Llama 3.2 3B",
      "modelId": "mlx-community/Llama-3.2-3B-Instruct-4bit",
      "backend": "MLX",
      "quantization": "4bit",
      "parameters": "3B",
      "device": "iPhone Air",
      "chip": "A19",
      "iOS": "26.6.1",
      "appVersion": "1.0.1",
      "date": "2026-09-03T22:30:55Z",
      "genTokPerSec": 24.32,
      "genStdDev": 3.7,
      "prefillTokPerSecEst": 245.4,
      "pp512": 188.5,
      "tg128": 19.48,
      "loadSeconds": 2.12,
      "peakMemoryMB": 2544,
      "throttled": true,
      "throttleDropPercent": 24.4,
      "suiteVersion": 2,
      "tasksPassed": 36,
      "tasksTotal": 46,
      "score": 78.3,
      "mathPassed": 2,
      "instructionsPassed": 20,
      "knowledgePassed": 6,
      "retrievalPassed": 4,
      "multilingualPassed": 4
    },
    {
      "model": "Qwen 2.5 3B",
      "modelId": "mlx-community/Qwen2.5-3B-Instruct-4bit",
      "backend": "MLX",
      "quantization": "4bit",
      "parameters": "3B",
      "device": "iPhone Air",
      "chip": "A19",
      "iOS": "26.6.1",
      "appVersion": "1.0.1",
      "date": "2026-09-03T22:39:06Z",
      "genTokPerSec": 24.53,
      "genStdDev": 1.75,
      "prefillTokPerSecEst": 280.8,
      "pp512": 241.1,
      "tg128": 21.68,
      "loadSeconds": 1.23,
      "peakMemoryMB": 2345,
      "throttled": false,
      "throttleDropPercent": 12.9,
      "suiteVersion": 2,
      "tasksPassed": 32,
      "tasksTotal": 46,
      "score": 69.6,
      "mathPassed": 2,
      "instructionsPassed": 15,
      "knowledgePassed": 6,
      "retrievalPassed": 4,
      "multilingualPassed": 5
    },
    {
      "model": "Llama 3.1 8B",
      "modelId": "mlx-community/Llama-3.1-8B-Instruct-4bit",
      "backend": "MLX",
      "quantization": "4bit",
      "parameters": "8B",
      "device": "iPhone Air",
      "chip": "A19",
      "iOS": "26.6.1",
      "appVersion": "1.0.1",
      "date": "2026-09-03T22:55:09Z",
      "genTokPerSec": 8.18,
      "genStdDev": 3.81,
      "prefillTokPerSecEst": 106.4,
      "pp512": 43.5,
      "tg128": 4.99,
      "loadSeconds": 3.1,
      "peakMemoryMB": 5137,
      "throttled": true,
      "throttleDropPercent": 65.1,
      "suiteVersion": 2,
      "tasksPassed": 37,
      "tasksTotal": 46,
      "score": 80.4,
      "mathPassed": 4,
      "instructionsPassed": 19,
      "knowledgePassed": 5,
      "retrievalPassed": 4,
      "multilingualPassed": 5
    },
    {
      "model": "Mistral 7B v0.3",
      "modelId": "mlx-community/Mistral-7B-Instruct-v0.3-4bit",
      "backend": "MLX",
      "quantization": "4bit",
      "parameters": "7B",
      "device": "iPhone Air",
      "chip": "A19",
      "iOS": "26.6.1",
      "appVersion": "1.0.1",
      "date": "2026-09-03T23:11:30Z",
      "genTokPerSec": 8.59,
      "genStdDev": 3.63,
      "prefillTokPerSecEst": 122.6,
      "pp512": 44.8,
      "tg128": 5.08,
      "loadSeconds": 1.94,
      "peakMemoryMB": 4641,
      "throttled": true,
      "throttleDropPercent": 60.4,
      "suiteVersion": 2,
      "tasksPassed": 30,
      "tasksTotal": 46,
      "score": 65.2,
      "mathPassed": 2,
      "instructionsPassed": 13,
      "knowledgePassed": 6,
      "retrievalPassed": 4,
      "multilingualPassed": 5
    },
    {
      "model": "Gemma 3 4B (Vision)",
      "modelId": "mlx-community/gemma-3-4b-it-4bit",
      "backend": "MLX",
      "quantization": "4bit",
      "parameters": "4B",
      "device": "iPhone Air",
      "chip": "A19",
      "iOS": "26.6.1",
      "appVersion": "1.0.1",
      "date": "2026-09-03T23:22:28Z",
      "genTokPerSec": 22.16,
      "genStdDev": 2.95,
      "prefillTokPerSecEst": 161.4,
      "pp512": 121.9,
      "tg128": 12.82,
      "loadSeconds": 5.05,
      "peakMemoryMB": 4559,
      "throttled": true,
      "throttleDropPercent": 20.9,
      "suiteVersion": 2,
      "tasksPassed": 35,
      "tasksTotal": 46,
      "score": 76.1,
      "mathPassed": 1,
      "instructionsPassed": 18,
      "knowledgePassed": 7,
      "retrievalPassed": 4,
      "multilingualPassed": 5
    },
    {
      "model": "Qwen2-VL 7B (Vision)",
      "modelId": "mlx-community/Qwen2-VL-7B-Instruct-4bit",
      "backend": "MLX",
      "quantization": "4bit",
      "parameters": "7B",
      "device": "iPhone Air",
      "chip": "A19",
      "iOS": "26.6.1",
      "appVersion": "1.0.1",
      "date": "2026-09-03T23:34:00Z",
      "genTokPerSec": 11.48,
      "genStdDev": 3.91,
      "prefillTokPerSecEst": 122.7,
      "pp512": 46.1,
      "tg128": 5.35,
      "loadSeconds": 3.5,
      "peakMemoryMB": 5464,
      "throttled": true,
      "throttleDropPercent": 50.9,
      "suiteVersion": 2,
      "tasksPassed": 36,
      "tasksTotal": 46,
      "score": 78.3,
      "mathPassed": 5,
      "instructionsPassed": 14,
      "knowledgePassed": 7,
      "retrievalPassed": 4,
      "multilingualPassed": 6
    },
    {
      "model": "Llama 3.2 3B (GGUF)",
      "modelId": "bartowski/Llama-3.2-3B-Instruct-GGUF",
      "backend": "llama.cpp",
      "quantization": "Q4_K_M",
      "parameters": "3B",
      "device": "iPhone Air",
      "chip": "A19",
      "iOS": "26.6.1",
      "appVersion": "1.0.1",
      "date": "2026-09-03T23:41:42Z",
      "genTokPerSec": 27.21,
      "genStdDev": 0.93,
      "prefillTokPerSecEst": 546.4,
      "pp512": 458.6,
      "tg128": 24.74,
      "loadSeconds": 8.87,
      "peakMemoryMB": 696,
      "throttled": false,
      "throttleDropPercent": 6.5,
      "suiteVersion": 2,
      "tasksPassed": 36,
      "tasksTotal": 46,
      "score": 78.3,
      "mathPassed": 2,
      "instructionsPassed": 20,
      "knowledgePassed": 6,
      "retrievalPassed": 4,
      "multilingualPassed": 4
    },
    {
      "model": "Phi-3.5 Mini (GGUF)",
      "modelId": "bartowski/Phi-3.5-mini-instruct-GGUF",
      "backend": "llama.cpp",
      "quantization": "Q4_K_M",
      "parameters": "3.8B",
      "device": "iPhone Air",
      "chip": "A19",
      "iOS": "26.6.1",
      "appVersion": "1.0.1",
      "date": "2026-09-03T23:51:41Z",
      "genTokPerSec": 23.54,
      "genStdDev": 1.22,
      "prefillTokPerSecEst": 404.2,
      "pp512": 291.2,
      "tg128": 19.95,
      "loadSeconds": 0.26,
      "peakMemoryMB": 1758,
      "throttled": false,
      "throttleDropPercent": 9.7,
      "suiteVersion": 2,
      "tasksPassed": 35,
      "tasksTotal": 46,
      "score": 76.1,
      "mathPassed": 5,
      "instructionsPassed": 15,
      "knowledgePassed": 7,
      "retrievalPassed": 4,
      "multilingualPassed": 4
    },
    {
      "model": "Gemma 3 4B (GGUF, Vision)",
      "modelId": "unsloth/gemma-3-4b-it-GGUF",
      "backend": "llama.cpp",
      "quantization": "Q4_K_M",
      "parameters": "4B",
      "device": "iPhone Air",
      "chip": "A19",
      "iOS": "26.6.1",
      "appVersion": "1.0.1",
      "date": "2026-09-04T00:00:26Z",
      "genTokPerSec": 21.51,
      "genStdDev": 0.94,
      "prefillTokPerSecEst": 485,
      "pp512": 390.6,
      "tg128": 17.24,
      "loadSeconds": 2.3,
      "peakMemoryMB": 2149,
      "throttled": false,
      "throttleDropPercent": 7.5,
      "suiteVersion": 2,
      "tasksPassed": 38,
      "tasksTotal": 46,
      "score": 82.6,
      "mathPassed": 3,
      "instructionsPassed": 19,
      "knowledgePassed": 7,
      "retrievalPassed": 4,
      "multilingualPassed": 5
    },
    {
      "model": "Gemma 2 2B",
      "modelId": "mlx-community/gemma-2-2b-it-4bit",
      "backend": "MLX",
      "quantization": "4bit",
      "parameters": "2B",
      "device": "iPhone Air",
      "chip": "A19",
      "iOS": "26.6.1",
      "appVersion": "1.1.0",
      "date": "2026-10-22T20:20:03Z",
      "genTokPerSec": 30.28,
      "genStdDev": 0.28,
      "prefillTokPerSecEst": 288,
      "pp512": 277.1,
      "tg128": 27.38,
      "loadSeconds": 3.7,
      "peakMemoryMB": 2148,
      "throttled": false,
      "throttleDropPercent": 1.7,
      "suiteVersion": 3,
      "tasksPassed": 48,
      "tasksTotal": 118,
      "score": 40.7,
      "mathPassed": 3,
      "instructionsPassed": 19,
      "knowledgePassed": 19,
      "retrievalPassed": 4,
      "multilingualPassed": 3
    },
    {
      "model": "Phi-3.5 Mini",
      "modelId": "mlx-community/Phi-3.5-mini-instruct-4bit",
      "backend": "MLX",
      "quantization": "4bit",
      "parameters": "3.8B",
      "device": "iPhone Air",
      "chip": "A19",
      "iOS": "26.6.1",
      "appVersion": "1.1.0",
      "date": "2026-10-22T20:43:06Z",
      "genTokPerSec": 20.71,
      "genStdDev": 2.84,
      "prefillTokPerSecEst": 209.5,
      "pp512": 123.1,
      "tg128": 11.84,
      "loadSeconds": 0.98,
      "peakMemoryMB": 3189,
      "throttled": true,
      "throttleDropPercent": 24.2,
      "suiteVersion": 3,
      "tasksPassed": 63,
      "tasksTotal": 118,
      "score": 53.4,
      "mathPassed": 4,
      "instructionsPassed": 23,
      "knowledgePassed": 29,
      "retrievalPassed": 4,
      "multilingualPassed": 3
    },
    {
      "model": "Llama 3.2 3B",
      "modelId": "mlx-community/Llama-3.2-3B-Instruct-4bit",
      "backend": "MLX",
      "quantization": "4bit",
      "parameters": "3B",
      "device": "iPhone Air",
      "chip": "A19",
      "iOS": "26.6.1",
      "appVersion": "1.1.0",
      "date": "2026-10-22T21:02:30Z",
      "genTokPerSec": 23.92,
      "genStdDev": 3.3,
      "prefillTokPerSecEst": 245.7,
      "pp512": 183.3,
      "tg128": 19.87,
      "loadSeconds": 2.13,
      "peakMemoryMB": 2568,
      "throttled": true,
      "throttleDropPercent": 22.8,
      "suiteVersion": 3,
      "tasksPassed": 70,
      "tasksTotal": 118,
      "score": 59.3,
      "mathPassed": 2,
      "instructionsPassed": 36,
      "knowledgePassed": 26,
      "retrievalPassed": 4,
      "multilingualPassed": 2
    },
    {
      "model": "Qwen 2.5 3B",
      "modelId": "mlx-community/Qwen2.5-3B-Instruct-4bit",
      "backend": "MLX",
      "quantization": "4bit",
      "parameters": "3B",
      "device": "iPhone Air",
      "chip": "A19",
      "iOS": "26.6.1",
      "appVersion": "1.1.0",
      "date": "2026-10-22T21:19:32Z",
      "genTokPerSec": 26.25,
      "genStdDev": 2.76,
      "prefillTokPerSecEst": 279.1,
      "pp512": 249.2,
      "tg128": 22.44,
      "loadSeconds": 1.68,
      "peakMemoryMB": 2360,
      "throttled": false,
      "throttleDropPercent": 19,
      "suiteVersion": 3,
      "tasksPassed": 65,
      "tasksTotal": 118,
      "score": 55.1,
      "mathPassed": 2,
      "instructionsPassed": 32,
      "knowledgePassed": 24,
      "retrievalPassed": 4,
      "multilingualPassed": 3
    },
    {
      "model": "Llama 3.1 8B",
      "modelId": "mlx-community/Llama-3.1-8B-Instruct-4bit",
      "backend": "MLX",
      "quantization": "4bit",
      "parameters": "8B",
      "device": "iPhone Air",
      "chip": "A19",
      "iOS": "26.6.1",
      "appVersion": "1.1.0",
      "date": "2026-10-22T21:55:26Z",
      "genTokPerSec": 7.61,
      "genStdDev": 3.76,
      "prefillTokPerSecEst": 106.5,
      "pp512": 42,
      "tg128": 4.48,
      "loadSeconds": 2.75,
      "peakMemoryMB": 5121,
      "throttled": true,
      "throttleDropPercent": 65.9,
      "suiteVersion": 3,
      "tasksPassed": 68,
      "tasksTotal": 118,
      "score": 57.6,
      "mathPassed": 4,
      "instructionsPassed": 33,
      "knowledgePassed": 24,
      "retrievalPassed": 4,
      "multilingualPassed": 3
    },
    {
      "model": "Mistral 7B v0.3",
      "modelId": "mlx-community/Mistral-7B-Instruct-v0.3-4bit",
      "backend": "MLX",
      "quantization": "4bit",
      "parameters": "7B",
      "device": "iPhone Air",
      "chip": "A19",
      "iOS": "26.6.1",
      "appVersion": "1.1.0",
      "date": "2026-10-22T22:33:42Z",
      "genTokPerSec": 9.07,
      "genStdDev": 4.21,
      "prefillTokPerSecEst": 123.6,
      "pp512": 44,
      "tg128": 4.44,
      "loadSeconds": 1.1,
      "peakMemoryMB": 4671,
      "throttled": true,
      "throttleDropPercent": 65,
      "suiteVersion": 3,
      "tasksPassed": 61,
      "tasksTotal": 118,
      "score": 51.7,
      "mathPassed": 2,
      "instructionsPassed": 27,
      "knowledgePassed": 25,
      "retrievalPassed": 4,
      "multilingualPassed": 3
    },
    {
      "model": "Gemma 3 4B (Vision)",
      "modelId": "mlx-community/gemma-3-4b-it-4bit",
      "backend": "MLX",
      "quantization": "4bit",
      "parameters": "4B",
      "device": "iPhone Air",
      "chip": "A19",
      "iOS": "26.6.1",
      "appVersion": "1.1.0",
      "date": "2026-10-22T22:58:54Z",
      "genTokPerSec": 21.77,
      "genStdDev": 4.21,
      "prefillTokPerSecEst": 161.2,
      "pp512": 106.3,
      "tg128": 8.07,
      "loadSeconds": 4.99,
      "peakMemoryMB": 4577,
      "throttled": true,
      "throttleDropPercent": 31.6,
      "suiteVersion": 3,
      "tasksPassed": 68,
      "tasksTotal": 118,
      "score": 57.6,
      "mathPassed": 1,
      "instructionsPassed": 30,
      "knowledgePassed": 30,
      "retrievalPassed": 4,
      "multilingualPassed": 3
    },
    {
      "model": "Qwen2-VL 7B (Vision)",
      "modelId": "mlx-community/Qwen2-VL-7B-Instruct-4bit",
      "backend": "MLX",
      "quantization": "4bit",
      "parameters": "7B",
      "device": "iPhone Air",
      "chip": "A19",
      "iOS": "26.6.1",
      "appVersion": "1.1.0",
      "date": "2026-10-22T23:26:52Z",
      "genTokPerSec": 11.65,
      "genStdDev": 4.84,
      "prefillTokPerSecEst": 124.7,
      "pp512": 46.2,
      "tg128": 4.81,
      "loadSeconds": 2.46,
      "peakMemoryMB": 5461,
      "throttled": true,
      "throttleDropPercent": 59.2,
      "suiteVersion": 3,
      "tasksPassed": 75,
      "tasksTotal": 118,
      "score": 63.6,
      "mathPassed": 5,
      "instructionsPassed": 26,
      "knowledgePassed": 36,
      "retrievalPassed": 4,
      "multilingualPassed": 4
    },
    {
      "model": "Llama 3.2 3B (GGUF)",
      "modelId": "bartowski/Llama-3.2-3B-Instruct-GGUF",
      "backend": "llama.cpp",
      "quantization": "Q4_K_M",
      "parameters": "3B",
      "device": "iPhone Air",
      "chip": "A19",
      "iOS": "26.6.1",
      "appVersion": "1.1.0",
      "date": "2026-10-22T23:45:33Z",
      "genTokPerSec": 26.89,
      "genStdDev": 0.64,
      "prefillTokPerSecEst": 593.4,
      "pp512": 459,
      "tg128": 21.89,
      "loadSeconds": 0.24,
      "peakMemoryMB": 721,
      "throttled": false,
      "throttleDropPercent": 4.7,
      "suiteVersion": 3,
      "tasksPassed": 70,
      "tasksTotal": 118,
      "score": 59.3,
      "mathPassed": 1,
      "instructionsPassed": 35,
      "knowledgePassed": 27,
      "retrievalPassed": 4,
      "multilingualPassed": 3
    },
    {
      "model": "Phi-3.5 Mini (GGUF)",
      "modelId": "bartowski/Phi-3.5-mini-instruct-GGUF",
      "backend": "llama.cpp",
      "quantization": "Q4_K_M",
      "parameters": "3.8B",
      "device": "iPhone Air",
      "chip": "A19",
      "iOS": "26.6.1",
      "appVersion": "1.1.0",
      "date": "2026-10-23T00:09:42Z",
      "genTokPerSec": 23.21,
      "genStdDev": 1.15,
      "prefillTokPerSecEst": 404,
      "pp512": 279.5,
      "tg128": 14.53,
      "loadSeconds": 0.25,
      "peakMemoryMB": 1739,
      "throttled": true,
      "throttleDropPercent": 9.3,
      "suiteVersion": 3,
      "tasksPassed": 78,
      "tasksTotal": 118,
      "score": 66.1,
      "mathPassed": 5,
      "instructionsPassed": 28,
      "knowledgePassed": 39,
      "retrievalPassed": 4,
      "multilingualPassed": 2
    },
    {
      "model": "Gemma 3 4B (GGUF, Vision)",
      "modelId": "unsloth/gemma-3-4b-it-GGUF",
      "backend": "llama.cpp",
      "quantization": "Q4_K_M",
      "parameters": "4B",
      "device": "iPhone Air",
      "chip": "A19",
      "iOS": "26.6.1",
      "appVersion": "1.1.0",
      "date": "2026-10-23T00:29:47Z",
      "genTokPerSec": 20.72,
      "genStdDev": 1.58,
      "prefillTokPerSecEst": 525.1,
      "pp512": 313.7,
      "tg128": 10.87,
      "loadSeconds": 2.21,
      "peakMemoryMB": 2175,
      "throttled": true,
      "throttleDropPercent": 14.1,
      "suiteVersion": 3,
      "tasksPassed": 71,
      "tasksTotal": 118,
      "score": 60.2,
      "mathPassed": 3,
      "instructionsPassed": 32,
      "knowledgePassed": 29,
      "retrievalPassed": 4,
      "multilingualPassed": 3
    },
    {
      "model": "Gemma 4 E2B (GGUF, Vision)",
      "modelId": "unsloth/gemma-4-E2B-it-GGUF",
      "backend": "llama.cpp",
      "quantization": "Q4_K_M",
      "parameters": "E2B",
      "device": "iPhone Air",
      "chip": "A19",
      "iOS": "26.6.1",
      "appVersion": "1.1.0",
      "date": "2026-10-23T00:45:38Z",
      "genTokPerSec": 35.02,
      "genStdDev": 2.26,
      "prefillTokPerSecEst": 847.6,
      "pp512": 707.5,
      "tg128": 32.84,
      "loadSeconds": 2.65,
      "peakMemoryMB": 1490,
      "throttled": false,
      "throttleDropPercent": 12.1,
      "suiteVersion": 3,
      "tasksPassed": 75,
      "tasksTotal": 118,
      "score": 63.6,
      "mathPassed": 3,
      "instructionsPassed": 32,
      "knowledgePassed": 33,
      "retrievalPassed": 4,
      "multilingualPassed": 3
    },
    {
      "model": "Qwen 3.5 0.8B (GGUF, Vision)",
      "modelId": "unsloth/Qwen3.5-0.8B-GGUF",
      "backend": "llama.cpp",
      "quantization": "Q4_K_M",
      "parameters": "0.8B",
      "device": "iPhone Air",
      "chip": "A19",
      "iOS": "26.6.1",
      "appVersion": "1.1.0",
      "date": "2026-10-23T00:53:42Z",
      "genTokPerSec": 73.67,
      "genStdDev": 0.25,
      "prefillTokPerSecEst": 2172.2,
      "pp512": 2241.1,
      "tg128": 72.76,
      "loadSeconds": 0.28,
      "peakMemoryMB": 598,
      "throttled": false,
      "throttleDropPercent": 0.6,
      "suiteVersion": 3,
      "tasksPassed": 55,
      "tasksTotal": 118,
      "score": 46.6,
      "mathPassed": 2,
      "instructionsPassed": 26,
      "knowledgePassed": 20,
      "retrievalPassed": 4,
      "multilingualPassed": 3
    },
    {
      "model": "Qwen 3.5 2B (GGUF, Vision)",
      "modelId": "unsloth/Qwen3.5-2B-GGUF",
      "backend": "llama.cpp",
      "quantization": "Q4_K_M",
      "parameters": "2B",
      "device": "iPhone Air",
      "chip": "A19",
      "iOS": "26.6.1",
      "appVersion": "1.1.0",
      "date": "2026-10-23T01:06:31Z",
      "genTokPerSec": 36.65,
      "genStdDev": 1.32,
      "prefillTokPerSecEst": 1024.9,
      "pp512": 854.3,
      "tg128": 33.09,
      "loadSeconds": 0.39,
      "peakMemoryMB": 1054,
      "throttled": false,
      "throttleDropPercent": 6.6,
      "suiteVersion": 3,
      "tasksPassed": 76,
      "tasksTotal": 118,
      "score": 64.4,
      "mathPassed": 2,
      "instructionsPassed": 32,
      "knowledgePassed": 35,
      "retrievalPassed": 4,
      "multilingualPassed": 3
    },
    {
      "model": "Qwen 3.5 4B (GGUF, Vision)",
      "modelId": "unsloth/Qwen3.5-4B-GGUF",
      "backend": "llama.cpp",
      "quantization": "Q4_K_M",
      "parameters": "4B",
      "device": "iPhone Air",
      "chip": "A19",
      "iOS": "26.6.1",
      "appVersion": "1.1.0",
      "date": "2026-10-23T01:28:20Z",
      "genTokPerSec": 14.65,
      "genStdDev": 2.7,
      "prefillTokPerSecEst": 406.1,
      "pp512": 157.5,
      "tg128": 5.06,
      "loadSeconds": 0.44,
      "peakMemoryMB": 1252,
      "throttled": true,
      "throttleDropPercent": 31.2,
      "suiteVersion": 3,
      "tasksPassed": 81,
      "tasksTotal": 118,
      "score": 68.6,
      "mathPassed": 1,
      "instructionsPassed": 36,
      "knowledgePassed": 38,
      "retrievalPassed": 4,
      "multilingualPassed": 2
    },
    {
      "model": "LFM2 1.2B (GGUF)",
      "modelId": "LiquidAI/LFM2-1.2B-GGUF",
      "backend": "llama.cpp",
      "quantization": "Q4_K_M",
      "parameters": "1.2B",
      "device": "iPhone Air",
      "chip": "A19",
      "iOS": "26.6.1",
      "appVersion": "1.1.0",
      "date": "2026-10-23T01:38:21Z",
      "genTokPerSec": 77.33,
      "genStdDev": 0.44,
      "prefillTokPerSecEst": 1613.8,
      "pp512": 1533.7,
      "tg128": 75.91,
      "loadSeconds": 0.11,
      "peakMemoryMB": 249,
      "throttled": false,
      "throttleDropPercent": 1.1,
      "suiteVersion": 3,
      "tasksPassed": 69,
      "tasksTotal": 118,
      "score": 58.5,
      "mathPassed": 3,
      "instructionsPassed": 36,
      "knowledgePassed": 24,
      "retrievalPassed": 4,
      "multilingualPassed": 2
    },
    {
      "model": "LFM2 2.6B (GGUF)",
      "modelId": "LiquidAI/LFM2-2.6B-GGUF",
      "backend": "llama.cpp",
      "quantization": "Q4_K_M",
      "parameters": "2.6B",
      "device": "iPhone Air",
      "chip": "A19",
      "iOS": "26.6.1",
      "appVersion": "1.1.0",
      "date": "2026-10-23T01:52:11Z",
      "genTokPerSec": 35.88,
      "genStdDev": 0.71,
      "prefillTokPerSecEst": 685.2,
      "pp512": 559.6,
      "tg128": 33.12,
      "loadSeconds": 0.14,
      "peakMemoryMB": 266,
      "throttled": false,
      "throttleDropPercent": 3.9,
      "suiteVersion": 3,
      "tasksPassed": 71,
      "tasksTotal": 118,
      "score": 60.2,
      "mathPassed": 2,
      "instructionsPassed": 36,
      "knowledgePassed": 27,
      "retrievalPassed": 4,
      "multilingualPassed": 2
    },
    {
      "model": "Qwen2.5 1.5B (Core ML)",
      "modelId": "anziank/grio-qwen2.5-1.5b-instruct-coreml-stateful-int8",
      "backend": "Core ML",
      "quantization": "int8-W8A16",
      "parameters": "1.5B",
      "device": "iPhone Air",
      "chip": "A19",
      "iOS": "26.6.1",
      "appVersion": "1.1.0",
      "date": "2026-10-23T02:51:41Z",
      "genTokPerSec": 3.03,
      "genStdDev": 0.26,
      "prefillTokPerSecEst": 653.5,
      "pp512": 646.5,
      "tg128": 2.94,
      "loadSeconds": 39.41,
      "peakMemoryMB": 4354,
      "throttled": false,
      "throttleDropPercent": 15.4,
      "suiteVersion": 3,
      "tasksPassed": 29,
      "tasksTotal": 118,
      "score": 24.6,
      "mathPassed": 2,
      "instructionsPassed": 9,
      "knowledgePassed": 17,
      "retrievalPassed": 1,
      "multilingualPassed": 0
    },
    {
      "model": "Qwen2.5 1.5B (Core ML)",
      "modelId": "anziank/grio-qwen2.5-1.5b-instruct-coreml-stateful-int8",
      "backend": "Core ML",
      "quantization": "int8-W8A16",
      "parameters": "1.5B",
      "device": "iPhone Air",
      "chip": "A19",
      "iOS": "26.6.1",
      "appVersion": "1.0.1",
      "date": "2026-09-06T08:32:05Z",
      "genTokPerSec": 3.4,
      "genStdDev": 0.2,
      "prefillTokPerSecEst": 645.1,
      "pp512": 703.4,
      "tg128": 3.4,
      "loadSeconds": 40.86,
      "peakMemoryMB": 4130,
      "throttled": false,
      "throttleDropPercent": 10.8,
      "suiteVersion": 2,
      "tasksPassed": 12,
      "tasksTotal": 46,
      "score": 26.1,
      "mathPassed": 2,
      "instructionsPassed": 5,
      "knowledgePassed": 4,
      "retrievalPassed": 1,
      "multilingualPassed": 0
    },
    {
      "model": "Gemma 4 E2B (GGUF, Vision)",
      "modelId": "unsloth/gemma-4-E2B-it-GGUF",
      "backend": "llama.cpp",
      "quantization": "Q4_K_M",
      "parameters": "E2B",
      "device": "iPhone Air",
      "chip": "A19",
      "iOS": "26.6.1",
      "appVersion": "1.0.1",
      "date": "2026-10-21T22:31:36Z",
      "genTokPerSec": 34.92,
      "genStdDev": 1.21,
      "prefillTokPerSecEst": 823.7,
      "pp512": 740.2,
      "tg128": 32.22,
      "loadSeconds": 2.85,
      "peakMemoryMB": 1420,
      "throttled": false,
      "throttleDropPercent": 6.5,
      "suiteVersion": 3,
      "tasksPassed": 75,
      "tasksTotal": 118,
      "score": 63.6,
      "mathPassed": 3,
      "instructionsPassed": 32,
      "knowledgePassed": 33,
      "retrievalPassed": 4,
      "multilingualPassed": 3
    },
    {
      "model": "LFM2 1.2B (GGUF)",
      "modelId": "LiquidAI/LFM2-1.2B-GGUF",
      "backend": "llama.cpp",
      "quantization": "Q4_K_M",
      "parameters": "1.2B",
      "device": "iPhone Air",
      "chip": "A19",
      "iOS": "26.6.1",
      "appVersion": "1.0.1",
      "date": "2026-10-21T23:13:21Z",
      "genTokPerSec": 77.85,
      "genStdDev": 0.3,
      "prefillTokPerSecEst": 1618.1,
      "pp512": 1623.8,
      "tg128": 76.43,
      "loadSeconds": 0.49,
      "peakMemoryMB": 203,
      "throttled": false,
      "throttleDropPercent": 0.7,
      "suiteVersion": 3,
      "tasksPassed": 69,
      "tasksTotal": 118,
      "score": 58.5,
      "mathPassed": 3,
      "instructionsPassed": 36,
      "knowledgePassed": 24,
      "retrievalPassed": 4,
      "multilingualPassed": 2
    },
    {
      "model": "LFM2 2.6B (GGUF)",
      "modelId": "LiquidAI/LFM2-2.6B-GGUF",
      "backend": "llama.cpp",
      "quantization": "Q4_K_M",
      "parameters": "2.6B",
      "device": "iPhone Air",
      "chip": "A19",
      "iOS": "26.6.1",
      "appVersion": "1.0.1",
      "date": "2026-10-21T23:24:47Z",
      "genTokPerSec": 34.94,
      "genStdDev": 1.48,
      "prefillTokPerSecEst": 685.6,
      "pp512": 528.3,
      "tg128": 32.86,
      "loadSeconds": 0.14,
      "peakMemoryMB": 220,
      "throttled": false,
      "throttleDropPercent": 7.6,
      "suiteVersion": 3,
      "tasksPassed": 71,
      "tasksTotal": 118,
      "score": 60.2,
      "mathPassed": 2,
      "instructionsPassed": 36,
      "knowledgePassed": 27,
      "retrievalPassed": 4,
      "multilingualPassed": 2
    },
    {
      "model": "Qwen 3.5 0.8B (GGUF, Vision)",
      "modelId": "unsloth/Qwen3.5-0.8B-GGUF",
      "backend": "llama.cpp",
      "quantization": "Q4_K_M",
      "parameters": "0.8B",
      "device": "iPhone Air",
      "chip": "A19",
      "iOS": "26.6.1",
      "appVersion": "1.0.1",
      "date": "2026-10-21T22:36:30Z",
      "genTokPerSec": 73.78,
      "genStdDev": 0.18,
      "prefillTokPerSecEst": 2155.5,
      "pp512": 2271.8,
      "tg128": 73.15,
      "loadSeconds": 0.28,
      "peakMemoryMB": 550,
      "throttled": false,
      "throttleDropPercent": 0.5,
      "suiteVersion": 3,
      "tasksPassed": 55,
      "tasksTotal": 118,
      "score": 46.6,
      "mathPassed": 2,
      "instructionsPassed": 26,
      "knowledgePassed": 20,
      "retrievalPassed": 4,
      "multilingualPassed": 3
    },
    {
      "model": "Qwen 3.5 2B (GGUF, Vision)",
      "modelId": "unsloth/Qwen3.5-2B-GGUF",
      "backend": "llama.cpp",
      "quantization": "Q4_K_M",
      "parameters": "2B",
      "device": "iPhone Air",
      "chip": "A19",
      "iOS": "26.6.1",
      "appVersion": "1.0.1",
      "date": "2026-10-21T22:46:45Z",
      "genTokPerSec": 36.79,
      "genStdDev": 1.26,
      "prefillTokPerSecEst": 1022.5,
      "pp512": 803.5,
      "tg128": 33.98,
      "loadSeconds": 0.39,
      "peakMemoryMB": 1002,
      "throttled": false,
      "throttleDropPercent": 6.1,
      "suiteVersion": 3,
      "tasksPassed": 76,
      "tasksTotal": 118,
      "score": 64.4,
      "mathPassed": 2,
      "instructionsPassed": 32,
      "knowledgePassed": 35,
      "retrievalPassed": 4,
      "multilingualPassed": 3
    },
    {
      "model": "Qwen 3.5 4B (GGUF, Vision)",
      "modelId": "unsloth/Qwen3.5-4B-GGUF",
      "backend": "llama.cpp",
      "quantization": "Q4_K_M",
      "parameters": "4B",
      "device": "iPhone Air",
      "chip": "A19",
      "iOS": "26.6.1",
      "appVersion": "1.0.1",
      "date": "2026-10-21T23:05:35Z",
      "genTokPerSec": 15.15,
      "genStdDev": 1.37,
      "prefillTokPerSecEst": 402.2,
      "pp512": 236.3,
      "tg128": 6.41,
      "loadSeconds": 0.63,
      "peakMemoryMB": 1207,
      "throttled": true,
      "throttleDropPercent": 16.1,
      "suiteVersion": 3,
      "tasksPassed": 81,
      "tasksTotal": 118,
      "score": 68.6,
      "mathPassed": 1,
      "instructionsPassed": 36,
      "knowledgePassed": 38,
      "retrievalPassed": 4,
      "multilingualPassed": 2
    },
    {
      "model": "Gemma 2 2B",
      "modelId": "mlx-community/gemma-2-2b-it-4bit",
      "backend": "MLX",
      "quantization": "4bit",
      "parameters": "2B",
      "device": "iPhone Air",
      "chip": "A19",
      "iOS": "26.6.1",
      "appVersion": "1.0.1",
      "date": "2026-09-04T11:26:44Z",
      "genTokPerSec": 37.93,
      "genStdDev": 0.54,
      "prefillTokPerSecEst": 316.2,
      "pp512": 268.8,
      "tg128": 35.8,
      "loadSeconds": 2.03,
      "peakMemoryMB": 2089,
      "throttled": false,
      "throttleDropPercent": 2.6,
      "suiteVersion": 2,
      "tasksPassed": 26,
      "tasksTotal": 46,
      "score": 56.5,
      "mathPassed": 3,
      "instructionsPassed": 10,
      "knowledgePassed": 4,
      "retrievalPassed": 4,
      "multilingualPassed": 5
    },
    {
      "model": "Phi-3.5 Mini",
      "modelId": "mlx-community/Phi-3.5-mini-instruct-4bit",
      "backend": "MLX",
      "quantization": "4bit",
      "parameters": "3.8B",
      "device": "iPhone Air",
      "chip": "A19",
      "iOS": "26.6.1",
      "appVersion": "1.0.1",
      "date": "2026-09-04T11:34:17Z",
      "genTokPerSec": 23.3,
      "genStdDev": 2.73,
      "prefillTokPerSecEst": 224.8,
      "pp512": 94,
      "tg128": 14.7,
      "loadSeconds": 0.96,
      "peakMemoryMB": 3167,
      "throttled": true,
      "throttleDropPercent": 20.9,
      "suiteVersion": 2,
      "tasksPassed": 33,
      "tasksTotal": 46,
      "score": 71.7,
      "mathPassed": 4,
      "instructionsPassed": 14,
      "knowledgePassed": 6,
      "retrievalPassed": 4,
      "multilingualPassed": 5
    },
    {
      "model": "Llama 3.2 3B",
      "modelId": "mlx-community/Llama-3.2-3B-Instruct-4bit",
      "backend": "MLX",
      "quantization": "4bit",
      "parameters": "3B",
      "device": "iPhone Air",
      "chip": "A19",
      "iOS": "26.6.1",
      "appVersion": "1.0.1",
      "date": "2026-09-04T11:38:59Z",
      "genTokPerSec": 29.99,
      "genStdDev": 1.68,
      "prefillTokPerSecEst": 247.3,
      "pp512": 181.2,
      "tg128": 17.77,
      "loadSeconds": 1.2,
      "peakMemoryMB": 2508,
      "throttled": true,
      "throttleDropPercent": 10.6,
      "suiteVersion": 2,
      "tasksPassed": 36,
      "tasksTotal": 46,
      "score": 78.3,
      "mathPassed": 2,
      "instructionsPassed": 20,
      "knowledgePassed": 6,
      "retrievalPassed": 4,
      "multilingualPassed": 4
    },
    {
      "model": "Qwen 2.5 3B",
      "modelId": "mlx-community/Qwen2.5-3B-Instruct-4bit",
      "backend": "MLX",
      "quantization": "4bit",
      "parameters": "3B",
      "device": "iPhone Air",
      "chip": "A19",
      "iOS": "26.6.1",
      "appVersion": "1.0.1",
      "date": "2026-09-04T11:44:03Z",
      "genTokPerSec": 30.77,
      "genStdDev": 2.28,
      "prefillTokPerSecEst": 282.7,
      "pp512": 238.5,
      "tg128": 29.4,
      "loadSeconds": 0.72,
      "peakMemoryMB": 2332,
      "throttled": false,
      "throttleDropPercent": 13.4,
      "suiteVersion": 2,
      "tasksPassed": 32,
      "tasksTotal": 46,
      "score": 69.6,
      "mathPassed": 2,
      "instructionsPassed": 15,
      "knowledgePassed": 6,
      "retrievalPassed": 4,
      "multilingualPassed": 5
    }
  ]
}