metrics.json

{
  "model": "answerdotai/ModernBERT-base",
  "train_n": 28500,
  "mix": {
    "ragtruth": 10000,
    "vitaminc": 16000,
    "hard_neg": 2500
  },
  "q_drop": 0.5,
  "maxlen": 512,
  "epochs": 3,
  "results": {
    "ragtruth": {
      "accuracy": 0.7468,
      "f1": 0.6823883592574008,
      "n": 2500,
      "f1_ci95": [
        0.6574761399787911,
        0.7058823529411765
      ]
    },
    "vitaminc": {
      "accuracy": 0.8495,
      "f1": 0.8446050593701601,
      "n": 2000
    },
    "flipped_neg": {
      "accuracy": 0.804,
      "f1": 0.8913525498891353,
      "n": 500
    },
    "flipped_pos": {
      "accuracy": 0.762,
      "f1": 0,
      "n": 500
    },
    "manual5": [
      {
        "answer": "Most of the Amazon rainforest, around 60%, lies within Brazi",
        "expected": "grounded",
        "got": "grounded",
        "p_hall": 0.004,
        "pass": true
      },
      {
        "answer": "The Amazon rainforest produces 20% of the world's oxygen and",
        "expected": "hallucinated",
        "got": "hallucinated",
        "p_hall": 0.596,
        "pass": true
      },
      {
        "answer": "Revenue grew 12% in the third quarter, reaching $2.1 billion",
        "expected": "grounded",
        "got": "grounded",
        "p_hall": 0.001,
        "pass": true
      },
      {
        "answer": "The company's revenue fell 12% in Q3.",
        "expected": "hallucinated",
        "got": "hallucinated",
        "p_hall": 0.989,
        "pass": true
      },
      {
        "answer": "The vaccine showed 49% effectiveness in preventing symptomat",
        "expected": "hallucinated",
        "got": "hallucinated",
        "p_hall": 0.989,
        "pass": true
      }
    ]
  }
}