{
  "meta": {
    "timestamp": "2026-09-19 11:57:16",
    "device": "cpu",
    "n_per_task": 400,
    "seed": 13,
    "laya": "0.2.1"
  },
  "jev_published": {
    "ag_news": {
      "accuracy": 0.91,
      "coverage_at_5pct_error": 0.83,
      "latency_p50_ms": 236,
      "n": 100,
      "source": "AbdelStark/jev-benchmarks v0.1.0"
    },
    "banking77": {
      "accuracy": 0.87,
      "coverage_at_5pct_error": 0.86,
      "latency_p50_ms": 246,
      "n": 100,
      "labels": 72,
      "source": "AbdelStark/jev-benchmarks v0.1.0"
    },
    "emotion": {
      "accuracy": 0.48,
      "brier": 0.846,
      "nll": 5.588,
      "coverage_at_5pct_error": 0.0,
      "zero_prob_failures": 0.16,
      "n": 100,
      "source": "AbdelStark/jev-benchmarks v0.1.0"
    },
    "typed_decisions": {
      "accuracy": 0.727,
      "soft_accuracy": 0.58,
      "brier": 0.148,
      "ece": 0.144,
      "score_mae": 0.391,
      "ms_per_case": 710,
      "source": "laya repo comparison table"
    },
    "_independent_": {
      "banking77_accuracy": 0.763,
      "sms_spam_accuracy": 0.93,
      "permuted_accuracy": 0.767,
      "ece": 0.246,
      "latency_p50_ms": "264-276",
      "option_order_flip_rate": 0.13,
      "source": "nibzard/decision-model-benchmark"
    }
  },
  "suites": {
    "jev.ag_news": {
      "english": {
        "n": 400,
        "accuracy": 0.95,
        "macro_f1": 0.9439,
        "ece": 0.0316,
        "brier": 0.0808,
        "nll": 0.1637,
        "mean_confidence": 0.922,
        "acc_at_50_coverage": 1.0,
        "seconds": 25.9,
        "ms_per_case": 64.8,
        "dropped": 0,
        "jev": "ag_news",
        "in_training": true
      },
      "multilingual": {
        "n": 400,
        "accuracy": 0.93,
        "macro_f1": 0.925,
        "ece": 0.042,
        "brier": 0.1092,
        "nll": 0.2329,
        "mean_confidence": 0.967,
        "acc_at_50_coverage": 0.995,
        "seconds": 14.3,
        "ms_per_case": 35.8,
        "dropped": 0,
        "jev": "ag_news",
        "in_training": true
      },
      "typed-decisions": {
        "n": 400,
        "accuracy": 0.9525,
        "macro_f1": 0.9468,
        "ece": 0.155,
        "brier": 0.1137,
        "nll": 0.2801,
        "mean_confidence": 0.7975,
        "acc_at_50_coverage": 1.0,
        "seconds": 41.4,
        "ms_per_case": 103.6,
        "dropped": 0,
        "jev": "ag_news",
        "in_training": true
      }
    },
    "jev.emotion": {
      "english": {
        "n": 400,
        "accuracy": 0.595,
        "macro_f1": 0.4752,
        "ece": 0.3063,
        "brier": 0.6963,
        "nll": 2.0194,
        "mean_confidence": 0.9013,
        "acc_at_50_coverage": 0.715,
        "seconds": 13.2,
        "ms_per_case": 33.0,
        "dropped": 0,
        "jev": "emotion",
        "in_training": false
      },
      "multilingual": {
        "n": 400,
        "accuracy": 0.53,
        "macro_f1": 0.4441,
        "ece": 0.3412,
        "brier": 0.7659,
        "nll": 2.2797,
        "mean_confidence": 0.8712,
        "acc_at_50_coverage": 0.635,
        "seconds": 7.0,
        "ms_per_case": 17.6,
        "dropped": 0,
        "jev": "emotion",
        "in_training": false
      },
      "typed-decisions": {
        "n": 400,
        "accuracy": 0.6,
        "macro_f1": 0.4741,
        "ece": 0.2005,
        "brier": 0.6113,
        "nll": 1.3242,
        "mean_confidence": 0.7928,
        "acc_at_50_coverage": 0.71,
        "seconds": 19.9,
        "ms_per_case": 49.7,
        "dropped": 0,
        "jev": "emotion",
        "in_training": false
      }
    },
    "jev.banking77_full": {
      "english": {
        "n": 400,
        "accuracy": 0.425,
        "macro_f1": 0.1121,
        "ece": 0.5397,
        "brier": 1.0968,
        "nll": 12.0952,
        "mean_confidence": 0.9616,
        "acc_at_50_coverage": 0.59,
        "seconds": 93.9,
        "ms_per_case": 234.8,
        "dropped": 0,
        "jev": "banking77",
        "in_training": false,
        "n_labels": 77
      },
      "multilingual": {
        "n": 400,
        "accuracy": 0.425,
        "macro_f1": 0.1125,
        "ece": 0.4037,
        "brier": 0.9455,
        "nll": 4.5134,
        "mean_confidence": 0.8229,
        "acc_at_50_coverage": 0.61,
        "seconds": 52.8,
        "ms_per_case": 132.0,
        "dropped": 0,
        "jev": "banking77",
        "in_training": false,
        "n_labels": 77
      },
      "typed-decisions": {
        "n": 400,
        "accuracy": 0.4925,
        "macro_f1": 0.1307,
        "ece": 0.4173,
        "brier": 0.8894,
        "nll": 5.4993,
        "mean_confidence": 0.9061,
        "acc_at_50_coverage": 0.685,
        "seconds": 117.4,
        "ms_per_case": 293.6,
        "dropped": 0,
        "jev": "banking77",
        "in_training": false,
        "n_labels": 77
      }
    },
    "app.support_triage": {
      "english": {
        "n": 400,
        "accuracy": 0.5025,
        "macro_f1": 0.4802,
        "ece": 0.0907,
        "brier": 0.6438,
        "nll": 1.4237,
        "mean_confidence": 0.5065,
        "acc_at_50_coverage": 0.595,
        "seconds": 90.9,
        "ms_per_case": 227.2,
        "dropped": 0,
        "in_training": true
      },
      "multilingual": {
        "n": 400,
        "accuracy": 0.5225,
        "macro_f1": 0.5159,
        "ece": 0.0724,
        "brier": 0.6034,
        "nll": 1.2902,
        "mean_confidence": 0.5949,
        "acc_at_50_coverage": 0.685,
        "seconds": 35.7,
        "ms_per_case": 89.3,
        "dropped": 0,
        "in_training": true
      },
      "typed-decisions": {
        "n": 400,
        "accuracy": 0.505,
        "macro_f1": 0.4699,
        "ece": 0.1149,
        "brier": 0.6603,
        "nll": 1.4739,
        "mean_confidence": 0.4165,
        "acc_at_50_coverage": 0.595,
        "seconds": 90.3,
        "ms_per_case": 225.7,
        "dropped": 0,
        "in_training": true
      }
    },
    "app.email_spam": {
      "english": {
        "n": 400,
        "accuracy": 0.9925,
        "macro_f1": 0.9925,
        "ece": 0.0133,
        "brier": 0.0119,
        "nll": 0.023,
        "mean_confidence": 0.9831,
        "acc_at_50_coverage": 1.0,
        "seconds": 108.8,
        "ms_per_case": 272.0,
        "dropped": 0,
        "in_training": true
      },
      "multilingual": {
        "n": 400,
        "accuracy": 0.9925,
        "macro_f1": 0.9925,
        "ece": 0.0079,
        "brier": 0.0148,
        "nll": 0.0351,
        "mean_confidence": 0.9976,
        "acc_at_50_coverage": 1.0,
        "seconds": 63.4,
        "ms_per_case": 158.6,
        "dropped": 0,
        "in_training": true
      },
      "typed-decisions": {
        "n": 400,
        "accuracy": 0.9575,
        "macro_f1": 0.9575,
        "ece": 0.1085,
        "brier": 0.0979,
        "nll": 0.1927,
        "mean_confidence": 0.849,
        "acc_at_50_coverage": 1.0,
        "seconds": 142.4,
        "ms_per_case": 355.9,
        "dropped": 0,
        "in_training": true
      }
    },
    "app.phishing": {
      "english": {
        "n": 400,
        "accuracy": 0.98,
        "macro_f1": 0.9789,
        "ece": 0.0121,
        "brier": 0.0292,
        "nll": 0.048,
        "mean_confidence": 0.9738,
        "acc_at_50_coverage": 1.0,
        "seconds": 118.8,
        "ms_per_case": 297.0,
        "dropped": 0,
        "in_training": true
      },
      "multilingual": {
        "n": 400,
        "accuracy": 0.9925,
        "macro_f1": 0.9921,
        "ece": 0.0032,
        "brier": 0.0121,
        "nll": 0.0188,
        "mean_confidence": 0.9949,
        "acc_at_50_coverage": 1.0,
        "seconds": 68.2,
        "ms_per_case": 170.6,
        "dropped": 0,
        "in_training": true
      },
      "typed-decisions": {
        "n": 400,
        "accuracy": 0.94,
        "macro_f1": 0.9353,
        "ece": 0.0762,
        "brier": 0.1062,
        "nll": 0.2039,
        "mean_confidence": 0.865,
        "acc_at_50_coverage": 0.99,
        "seconds": 144.0,
        "ms_per_case": 360.1,
        "dropped": 0,
        "in_training": true
      }
    },
    "app.guardrails_jailbreak": {
      "english": {
        "n": 400,
        "accuracy": 0.7075,
        "macro_f1": 0.684,
        "ece": 0.2591,
        "brier": 0.5389,
        "nll": 1.8576,
        "mean_confidence": 0.9657,
        "acc_at_50_coverage": 0.83,
        "seconds": 50.7,
        "ms_per_case": 126.7,
        "dropped": 0,
        "in_training": false
      },
      "multilingual": {
        "n": 400,
        "accuracy": 0.755,
        "macro_f1": 0.7208,
        "ece": 0.1974,
        "brier": 0.425,
        "nll": 1.22,
        "mean_confidence": 0.9454,
        "acc_at_50_coverage": 0.84,
        "seconds": 23.1,
        "ms_per_case": 57.8,
        "dropped": 0,
        "in_training": false
      },
      "typed-decisions": {
        "n": 400,
        "accuracy": 0.7625,
        "macro_f1": 0.733,
        "ece": 0.0869,
        "brier": 0.3365,
        "nll": 0.5061,
        "mean_confidence": 0.841,
        "acc_at_50_coverage": 0.885,
        "seconds": 49.5,
        "ms_per_case": 123.8,
        "dropped": 0,
        "in_training": false
      }
    },
    "app.moderation_toxicity": {
      "english": {
        "n": 400,
        "accuracy": 0.53,
        "macro_f1": 0.4003,
        "ece": 0.2961,
        "brier": 0.6015,
        "nll": 0.8862,
        "mean_confidence": 0.8242,
        "acc_at_50_coverage": 0.735,
        "seconds": 45.5,
        "ms_per_case": 113.8,
        "dropped": 0,
        "in_training": false
      },
      "multilingual": {
        "n": 400,
        "accuracy": 0.525,
        "macro_f1": 0.394,
        "ece": 0.3809,
        "brier": 0.7348,
        "nll": 1.1981,
        "mean_confidence": 0.9059,
        "acc_at_50_coverage": 0.725,
        "seconds": 19.1,
        "ms_per_case": 47.8,
        "dropped": 0,
        "in_training": false
      },
      "typed-decisions": {
        "n": 400,
        "accuracy": 0.53,
        "macro_f1": 0.3967,
        "ece": 0.244,
        "brier": 0.5315,
        "nll": 0.7305,
        "mean_confidence": 0.7553,
        "acc_at_50_coverage": 0.72,
        "seconds": 43.6,
        "ms_per_case": 109.1,
        "dropped": 0,
        "in_training": false
      }
    },
    "app.rag_relevance": {
      "english": {
        "n": 400,
        "accuracy": 0.625,
        "macro_f1": 0.6234,
        "ece": 0.1175,
        "brier": 0.4656,
        "nll": 0.671,
        "mean_confidence": 0.7425,
        "acc_at_50_coverage": 0.72,
        "seconds": 52.0,
        "ms_per_case": 130.0,
        "dropped": 0,
        "in_training": true
      },
      "multilingual": {
        "n": 400,
        "accuracy": 0.6575,
        "macro_f1": 0.6567,
        "ece": 0.1718,
        "brier": 0.489,
        "nll": 0.7659,
        "mean_confidence": 0.8293,
        "acc_at_50_coverage": 0.73,
        "seconds": 22.3,
        "ms_per_case": 55.7,
        "dropped": 0,
        "in_training": true
      },
      "typed-decisions": {
        "n": 400,
        "accuracy": 0.625,
        "macro_f1": 0.6239,
        "ece": 0.0419,
        "brier": 0.4458,
        "nll": 0.6367,
        "mean_confidence": 0.6433,
        "acc_at_50_coverage": 0.73,
        "seconds": 52.8,
        "ms_per_case": 132.0,
        "dropped": 0,
        "in_training": true
      }
    },
    "app.model_routing_domain": {
      "english": {
        "n": 399,
        "accuracy": 0.6391,
        "macro_f1": 0.3199,
        "ece": 0.0889,
        "brier": 0.4938,
        "nll": 1.089,
        "mean_confidence": 0.7121,
        "acc_at_50_coverage": 0.8744,
        "seconds": 50.8,
        "ms_per_case": 127.4,
        "dropped": 0,
        "in_training": false
      },
      "multilingual": {
        "n": 399,
        "accuracy": 0.1228,
        "macro_f1": 0.0912,
        "ece": 0.4898,
        "brier": 1.1396,
        "nll": 2.3047,
        "mean_confidence": 0.6126,
        "acc_at_50_coverage": 0.0452,
        "seconds": 21.9,
        "ms_per_case": 54.9,
        "dropped": 0,
        "in_training": false
      },
      "typed-decisions": {
        "n": 399,
        "accuracy": 0.6591,
        "macro_f1": 0.3426,
        "ece": 0.1058,
        "brier": 0.503,
        "nll": 1.0623,
        "mean_confidence": 0.5909,
        "acc_at_50_coverage": 0.8342,
        "seconds": 53.3,
        "ms_per_case": 133.6,
        "dropped": 0,
        "in_training": false
      }
    }
  }
}