{
  "pack": "essay-grading",
  "title": "Jev essay grading prompts",
  "testedModel": "typesafe/jev-1.14-20261014",
  "testedAt": "2026-10-25",
  "prompts": [
    {
      "id": "flag-wrong-answer",
      "title": "Flag a wrong short answer",
      "purpose": "This Jev essay grading prompt says whether a student answer is wrong or misses the point, compared with the answer key.",
      "request": {
        "state": {
          "question": "{{question}}",
          "reference_answer": "{{reference_answer}}",
          "student_answer": "{{student_answer}}"
        },
        "questions": {
          "wrong": {
            "type": "noul",
            "instructions": "Is `student_answer` wrong, judged against `reference_answer` as an answer to `question`? Wrong means it gets the main point wrong, misses all the key points, or answers something else. An answer that is incomplete but partly right is not wrong."
          }
        }
      },
      "notes": "Fill in `question`, `reference_answer` (your answer key or marking notes) and `student_answer`. A yes means the answer is wrong or misses the point; an answer that is incomplete but partly right should come back no. On tutor-graded university networking answers Jev agreed 34 times out of 40. Use it to put the likely wrong answers at the top of a teacher's pile; the marks stay with the teacher. Telling full marks from partial credit did not work well enough for us to include: a three-way grade we tested matched the tutors only 51 times out of 90 across two versions.",
      "result": {
        "dataset": "Short Answer Feedback (SAF), communication networks, English",
        "datasetUrl": "https://huggingface.co/datasets/Short-Answer-Feedback/saf_communication_networks_english",
        "license": "CC BY 4.0",
        "n": 40,
        "agreement": 0.85,
        "metric": "yes means incorrect, partial counts correct, Jev essay grading at 0.5"
      }
    },
    {
      "id": "check-math-solution",
      "title": "Check a worked math solution",
      "purpose": "Use this Jev essay grading prompt to say whether every step of a written solution to a word problem is correct.",
      "request": {
        "state": {
          "problem": "{{problem}}",
          "solution": "{{solution}}"
        },
        "questions": {
          "solution_correct": {
            "type": "noul",
            "instructions": "Is every step of `solution` correct for `problem`, with no calculation mistake, wrong reasoning or misread fact anywhere?",
            "criteria": {
              "true": "Each step follows from the problem and the steps before it, and every number is right.",
              "false": "At least one step has a calculation error, a logical error or uses a wrong fact from the problem."
            }
          }
        }
      },
      "notes": "Put the problem in `problem` and the worked solution in `solution`, numbered \"Step 1:\", \"Step 2:\" and so on. On grade-school word problems Jev passed all 20 correct solutions but also passed 5 of 20 with a mistake. On harder competition problems the same request agreed with the experts only 30 times out of 40 and passed 7 of 20 wrong solutions. Jev does not calculate, so recheck the numbers in code and read a yes as a sign the solution looks right, short of proof.",
      "result": {
        "dataset": "ProcessBench (Qwen)",
        "datasetUrl": "https://huggingface.co/datasets/Qwen/ProcessBench",
        "license": "Apache 2.0",
        "n": 40,
        "agreement": 0.875,
        "metric": "jev essay grading matches error step or none, yes from 0.5"
      }
    },
    {
      "id": "check-answer-fully-correct",
      "title": "Pass or fail a short answer",
      "purpose": "For pass or fail marking, this Jev essay grading prompt says whether a student answer is fully correct against the answer key.",
      "request": {
        "state": {
          "question": "{{question}}",
          "reference_answer": "{{reference_answer}}",
          "student_answer": "{{student_answer}}"
        },
        "questions": {
          "full_marks": {
            "type": "noul",
            "instructions": "Would a teacher marking with `reference_answer` give `student_answer` full marks for `question`? Mark the ideas, not the wording: different words, extra detail or other valid examples are fine.",
            "criteria": {
              "true": "Every key point the question asks for is there and right.",
              "false": "A key point is missing or wrong."
            }
          }
        }
      },
      "notes": "Same fields as the wrong-answer check; a yes means full marks. Jev was stricter than the tutors: it refused full marks to 8 of 20 answers they gave full marks, and gave them to 5 of 20 they did not. With 27 of 40 right, use it only as a first pass that sends every no to a teacher; the final mark stays with the teacher. A revised wording is what you see here; the first version scored 25 of 40 on other answers.",
      "result": {
        "dataset": "Short Answer Feedback (SAF), communication networks, English",
        "datasetUrl": "https://huggingface.co/datasets/Short-Answer-Feedback/saf_communication_networks_english",
        "license": "CC BY 4.0",
        "n": 40,
        "agreement": 0.675,
        "metric": "correct or not, Jev essay grading yes at 0.5 or more"
      }
    }
  ]
}
