{
 "public_id": "AIR-2024-0036",
 "incident_id": "2024-08-copilot-bunbury-prison-harassment-victim-name",
 "date_disclosed": "2024-08-21",
 "date_occurred": "2024-08",
 "title": "Copilot-generated training scenario named a real sexual-harassment complainant, shown on screen at her former workplace",
 "class": "safety_failure",
 "attack_surface": "enterprise_assistant",
 "initial_vector": "none_autonomous_action",
 "agent_capability_abused": [
  "content_generation",
  "training_data_memorisation"
 ],
 "target_product": "Microsoft Copilot",
 "target_vendor": "Microsoft / Psychosocial Leadership (deployer)",
 "autonomy_level": "assistant",
 "impact_realized": true,
 "impact_type": [
  "privacy_violation",
  "reputational_harm"
 ],
 "blast_radius": "one named individual, re-identified in front of colleagues at Bunbury Regional Prison",
 "cve": [],
 "taxonomy": {
  "owasp_asi": [
   "ASI09"
  ],
  "mitre_atlas": [],
  "labels": [
   "no_adversary",
   "memorisation_leak",
   "synthetic_data_not_synthetic",
   "workplace_deployment"
  ]
 },
 "defense_present": "none; the trainer believed the scenario was fictional",
 "defense_bypassed": false,
 "fix_shipped": "unknown",
 "primary_sources": [
  "https://www.abc.net.au/news/2024-08-21/ai-chatbot-psychosocial-training-bunbury-regional-prison/104230980",
  "https://incidentdatabase.ai/cite/773"
 ],
 "verified_quote": "One scenario included a character called Bronwyn Hendry, the name of a real former employee.",
 "confidence": "high",
 "notes": "The operator's assumption — that a generated 'fictional' example contains no real people — is the failure. Asking a model for synthetic examples of a scenario it has seen reported can return the reported instance. Trainer Charlotte Ingham said she walked in thinking she had a fictional scenario. AI Incident Database incident 773.",
 "risk_category": "privacy",
 "risk_secondary": [
  "safety_harm"
 ],
 "risk_subcategory": "Membership Inference",
 "risk_sub_secondary": [],
 "ai_entity": "generative",
 "agency": "human_acted_on_output",
 "agency_debatable": true,
 "agency_reason": "the model generated the scenario; an operator displayed it on screen",
 "vector_family": "none_autonomous",
 "vector_detail": "none_autonomous_action",
 "guardrail_kind": [
  "none"
 ],
 "guardrail_outcome": "not_applicable",
 "xrefs": {
  "aiid": [
   773
  ],
  "atlas": [],
  "cve": [],
  "ghsa": []
 },
 "primary_asi": "ASI09",
 "agent_framework": "unknown",
 "model_provider": "microsoft",
 "model_name": "",
 "detection_mechanism": "unknown",
 "tool_access": "unknown",
 "financial_loss_band": "unknown",
 "eai_category": "sensitive_data_exposure",
 "eai_domain": "privacy",
 "eai_secondary": [
  "human_manipulation"
 ],
 "eai_reason": "subcategory: Membership Inference",
 "asi_curated": [],
 "asi_scope": "in_scope",
 "asi_provenance": "rule",
 "asi_reason": "ASI09 (agency=human_acted_on_output + reliance signal)",
 "submitted_by": "EAI Internal"
}