{
 "public_id": "AIR-2023-0029",
 "incident_id": "2023-12-malla-underground-llm-crime-market",
 "date_disclosed": "2023-12-01",
 "date_occurred": "2023-12-01",
 "title": "Indiana University \"Malla\" study mapped an underground market of jailbroken LLM services (WormGPT, FraudGPT, DarkGPT) selling malware and phishing generation",
 "class": "in_the_wild",
 "attack_surface": "other",
 "initial_vector": "jailbreak_guardrail_bypass",
 "agent_capability_abused": [
  "code_generation",
  "content_generation",
  "api_relay"
 ],
 "target_product": "OpenAI GPT-3.5 / GPT-4 (relayed by resold services)",
 "target_vendor": "OpenAI (upstream), underground vendors",
 "autonomy_level": "assistant",
 "impact_realized": true,
 "impact_type": [
  "ecosystem_abuse",
  "financial_loss"
 ],
 "blast_radius": "a commercial market of resold jailbroken LLM access serving malware, phishing and scam-site generation at scale",
 "cve": [],
 "taxonomy": {
  "owasp_asi": [
   "ASI01",
   "ASI03"
  ],
  "mitre_atlas": [],
  "labels": [
   "jailbreak",
   "criminal_service_economy",
   "guardrail_bypass",
   "upstream_provider_abuse"
  ]
 },
 "defense_present": "provider safety training and usage policies",
 "defense_bypassed": true,
 "fix_shipped": "ongoing provider account enforcement",
 "primary_sources": [
  "https://theconversation.com/fraudgpt-and-other-malicious-ais-are-the-new-frontier-of-online-threats-what-can-we-do-234820",
  "https://incidentdatabase.ai/cite/736"
 ],
 "verified_quote": "Named malicious LLMs are BadGPT, XXXGPT, Evil-GPT, WormGPT, FraudGPT, BLACKHATGPT, EscapeGPT, DarkGPT, and WolfGPT.",
 "confidence": "high",
 "notes": "First systematic measurement that jailbroken LLM access had become a priced product rather than a hobbyist trick, and that most such services were thin wrappers relaying to mainstream provider APIs. impact_realized true: real malware and phishing were produced and sold. AI Incident Database incident 736.",
 "risk_category": "security_fraud",
 "risk_secondary": [
  "product_misuse"
 ],
 "risk_subcategory": "Multi-agent contagion",
 "risk_sub_secondary": [],
 "ai_entity": "generative",
 "agency": "human_acted_on_output",
 "agency_debatable": false,
 "agency_reason": "jailbroken LLM services generated malware/phishing that human buyers deployed",
 "vector_family": "direct_prompt_injection",
 "vector_detail": "jailbreak_guardrail_bypass",
 "guardrail_kind": [
  "abuse_detection"
 ],
 "guardrail_outcome": "silent",
 "xrefs": {
  "aiid": [
   736
  ],
  "atlas": [],
  "cve": [],
  "ghsa": []
 },
 "primary_asi": "ASI01",
 "agent_framework": "unknown",
 "model_provider": "openai",
 "model_name": "",
 "detection_mechanism": "unknown",
 "tool_access": "unknown",
 "financial_loss_band": "unknown",
 "eai_category": "confused_deputy",
 "eai_domain": "access_control_and_permissions",
 "eai_secondary": [
  "unsafe_actuation"
 ],
 "eai_reason": "subcategory: Multi-agent contagion",
 "asi_curated": [],
 "asi_scope": "in_scope",
 "asi_provenance": "rule",
 "asi_reason": "ASI01 (vector_family=direct_prompt_injection); ASI03 (eai_category=confused_deputy)",
 "submitted_by": "EAI Internal"
}