{
  "pack": "jev-as-a-judge",
  "title": "Jev as a judge prompts",
  "testedModel": "typesafe/jev-1.14-20261014",
  "testedAt": "2026-10-25",
  "prompts": [
    {
      "id": "pick-better-answer",
      "title": "Pick the better of two answers",
      "purpose": "This Jev as a judge prompt compares two replies to the same request and says which one a careful human judge would prefer.",
      "request": {
        "state": {
          "user_request": "{{user_request}}",
          "response_a": "{{response_a}}",
          "response_b": "{{response_b}}"
        },
        "questions": {
          "better_response": {
            "type": "choice",
            "instructions": "Which response would a careful expert prefer for `user_request`? First look for mistakes in each response: wrong facts, wrong answers, broken code, or instructions it ignored. Then compare how fully and directly each one does what was asked. Length and the order of the responses do not matter.",
            "criteria": {
              "response_a": "`response_a` handles the request better.",
              "response_b": "`response_b` handles the request better."
            }
          }
        }
      },
      "notes": "Put the user's message in `user_request` and the two replies in `response_a` and `response_b`. `choice` names the winner and `probabilities` shows how close the call was; treat a winner under about 0.7 as a toss-up. Jev agreed with the human majority on 37 of 40 pairs. For comparison, the MT-Bench authors report that two human experts agreed on 81% of first-turn calls like these. Only 6 of the 40 pairs were math or coding questions, so for those, run the code or check the numbers as well.",
      "result": {
        "dataset": "MT-Bench human judgments (LMSYS)",
        "datasetUrl": "https://huggingface.co/datasets/lmsys/mt_bench_human_judgments",
        "license": "CC BY 4.0",
        "n": 40,
        "agreement": 0.925,
        "metric": "pairs where Jev as a judge matched preferred replies, ties left out"
      }
    },
    {
      "id": "rate-helpfulness",
      "title": "Score how helpful an answer is",
      "purpose": "Run this Jev as a judge prompt to rate a reply from 0 to 4 for how well it does what the user asked.",
      "request": {
        "state": {
          "prompt": "{{prompt}}",
          "response": "{{response}}"
        },
        "questions": {
          "helpfulness": {
            "type": "score",
            "instructions": "How helpful is `response` as a reply to `prompt`? Rate the response as a whole: does it do what the prompt asks, is it correct, and would it be useful to the person who asked?",
            "criteria": [
              "Not helpful: ignores the request, is wrong, or cannot be used.",
              "Slightly helpful: touches on the request but misses most of it or has major errors.",
              "Partly helpful: does some of what was asked but leaves out important parts or has clear errors.",
              "Mostly helpful: does what was asked, with small gaps or minor errors.",
              "Fully helpful: does everything the prompt asks, correctly and clearly."
            ]
          }
        }
      },
      "notes": "Put the request in `prompt` and the reply in `response`. `score` runs from 0 to 4 and can land between levels, so round it or pick your own cut-off, such as 3 and up for good enough. Jev was within one point of the human score on 33 of 40 replies but on the exact level for only 15, and it was a little kind to the worst replies and a little harsh on the best. It is good for sorting replies into bad, middling and good; a precise grade needs a person. You can swap in your own five levels, as long as they run from worst to best.",
      "result": {
        "dataset": "HelpSteer2 (NVIDIA)",
        "datasetUrl": "https://huggingface.co/datasets/nvidia/HelpSteer2",
        "license": "CC BY 4.0",
        "n": 40,
        "agreement": 0.825,
        "metric": "rounded helpfulness within 1 point, Jev as a judge, 0 to 4"
      }
    },
    {
      "id": "check-answer-is-true",
      "title": "Check whether an answer is true",
      "purpose": "Use this Jev as a judge prompt to say whether an answer to a factual question is correct or repeats a falsehood.",
      "request": {
        "state": {
          "question": "{{question}}",
          "answer": "{{answer}}"
        },
        "questions": {
          "answer_is_true": {
            "type": "noul",
            "instructions": "Is `answer` a true answer to `question`?",
            "criteria": {
              "true": "Everything the answer states is factually correct. Saying that a premise is false, or that something is not known, counts as true when that is the case.",
              "false": "The answer states something false, repeats a myth or a common misconception, or gets the facts wrong."
            }
          }
        }
      },
      "notes": "Put the question in `question` and the answer to check in `answer`. `noul` is the probability that the answer is true; we tested a cut-off of 0.5, and a higher one, such as 0.7, lets fewer false answers through. Jev agreed with the dataset on 37 of 40 answers, all short answers to questions built around common myths. It judges from what it already knows, so it can be wrong about recent events, niche facts and exact numbers. To check an answer against your own documents, use the passage prompt instead.",
      "result": {
        "dataset": "TruthfulQA",
        "datasetUrl": "https://huggingface.co/datasets/truthfulqa/truthful_qa",
        "license": "Apache 2.0",
        "n": 40,
        "agreement": 0.925,
        "metric": "true or false, Jev as a judge yes at 0.5 up"
      }
    },
    {
      "id": "check-passage-has-answer",
      "title": "Check whether a passage answers the question",
      "purpose": "Use Jev as a judge to say whether a source passage contains the answer, so a RAG app knows when to answer or say it does not know.",
      "request": {
        "state": {
          "passage": "{{passage}}",
          "question": "{{question}}"
        },
        "questions": {
          "passage_answers": {
            "type": "noul",
            "instructions": "Does `passage` state the answer to `question`?",
            "criteria": {
              "true": "A reader could answer the question from the passage alone.",
              "false": "The passage does not give the answer, even if it talks about the same subject."
            }
          }
        }
      },
      "notes": "Put the source text in `passage` and the user's question in `question`. A yes means the passage states the answer, so your app can answer from it; a no means it should say it does not know or search again. Jev passed all 20 answerable questions but also passed 5 of the 20 unanswerable ones, which were written to look answerable: they are on the right topic but ask for a detail the passage never gives. If a wrong yes is costly, raise the cut-off above 0.5. Keep the passage short and on topic, because unrelated text in the state lowers accuracy.",
      "result": {
        "dataset": "SQuAD 2.0",
        "datasetUrl": "https://huggingface.co/datasets/rajpurkar/squad_v2",
        "license": "CC BY-SA 4.0",
        "n": 40,
        "agreement": 0.875,
        "metric": "answerable question passage pairs, Jev as a judge yes from 0.5"
      }
    }
  ]
}
