{
  "pack": "ticket-triage",
  "title": "Jev ticket triage prompts",
  "testedModel": "typesafe/jev-1.14-20261014",
  "testedAt": "2026-10-25",
  "prompts": [
    {
      "id": "route-request-type",
      "title": "Route a message to one of your request types",
      "purpose": "This Jev ticket triage prompt picks which of your request types a customer message belongs to, from a list you supply.",
      "request": {
        "state": {
          "message": "{{message}}"
        },
        "questions": {
          "request_type": {
            "type": "choice",
            "instructions": "Which request type best matches what the customer wants in `message`?",
            "criteria": {
              "card_arrival": "An ordered card has not arrived yet.",
              "lost_or_stolen_card": "The card was lost or stolen.",
              "card_payment_not_recognised": "Does not recognise a card payment.",
              "transaction_charged_twice": "Was charged twice for the same transaction.",
              "request_refund": "Wants a refund.",
              "transfer_not_received_by_recipient": "The recipient has not received a transfer.",
              "exchange_rate": "Asks what exchange rate is used.",
              "top_up_failed": "A top-up failed.",
              "terminate_account": "Wants to close the account.",
              "passcode_forgotten": "Forgot the app passcode."
            }
          }
        }
      },
      "notes": "Put the customer's message in `message`, then replace the ten options with your own request types: a short id as the key and a one-line description as the value. Jev routed all 40 test messages correctly, but these ten types are clearly different from each other. Types that overlap, such as a refund request and a double charge, need descriptions that spell out the difference. Add an `other` option if some messages fit none of them.",
      "result": {
        "dataset": "Banking77 (PolyAI)",
        "datasetUrl": "https://github.com/PolyAI-LDN/task-specific-datasets",
        "license": "CC BY 4.0",
        "n": 40,
        "agreement": 1,
        "metric": "messages where Jev ticket triage picks the type annotators labeled"
      }
    },
    {
      "id": "detect-angry-customer",
      "title": "Spot an angry or annoyed customer",
      "purpose": "Use this Jev ticket triage prompt to say whether the writer of a message sounds angry or annoyed, so the message can jump the queue.",
      "request": {
        "state": {
          "message": "{{message}}"
        },
        "questions": {
          "angry": {
            "type": "noul",
            "instructions": "Does the writer of `message` sound angry or annoyed?"
          }
        }
      },
      "notes": "Put the message in `message`. Tested on Reddit comments labeled by people, not on support tickets. Jev caught 17 of 20 angry or annoyed comments and flagged 6 of 20 that raters did not mark, often blunt or sarcastic ones. Use it to move a message up the queue, not to label the customer.",
      "result": {
        "dataset": "GoEmotions (Google Research)",
        "datasetUrl": "https://huggingface.co/datasets/google-research-datasets/go_emotions",
        "license": "Apache 2.0",
        "n": 40,
        "agreement": 0.775,
        "metric": "anger or annoyance, agreed labels, Jev ticket triage yes from 0.5"
      }
    },
    {
      "id": "detect-out-of-scope",
      "title": "Catch requests you do not handle",
      "purpose": "A Jev ticket triage prompt says whether a message asks for something outside the list of request types your service supports.",
      "request": {
        "state": {
          "message": "{{message}}",
          "supported_requests": "{{supported_requests}}"
        },
        "questions": {
          "out_of_scope": {
            "type": "noul",
            "instructions": "Is `message` asking for something that none of the request types in `supported_requests` covers?",
            "criteria": {
              "true": "No request type on the list covers what the user wants.",
              "false": "At least one request type on the list covers what the user wants."
            }
          }
        }
      },
      "notes": "Put the message in `message` and the list of things you handle in `supported_requests`, as an array of short descriptions. The test list held the 150 request types of the CLINC150 assistant, from banking and travel to cooking and small talk. Jev agreed with the human labels on 38 of 40. Keep the list complete: anything missing from it is treated as out of scope.",
      "result": {
        "dataset": "CLINC150 (oos-eval)",
        "datasetUrl": "https://github.com/clinc/oos-eval",
        "license": "CC BY 3.0",
        "n": 40,
        "agreement": 0.95,
        "metric": "jev ticket triage matches out of scope or one of 150 intents"
      }
    }
  ]
}
