AI分类准确率怎么看：混淆矩阵比总分更能找漏分

{
  "tickets": [
    {
      "id": "T01",
      "text": "退款还没到账"
    },
    {
      "id": "T02",
      "text": "想退回未开封商品"
    },
    {
      "id": "T03",
      "text": "退款金额需要核对"
    },
    {
      "id": "T04",
      "text": "取消订单后退款"
    },
    {
      "id": "T05",
      "text": "快递到哪了"
    },
    {
      "id": "T06",
      "text": "物流显示异常"
    },
    {
      "id": "T07",
      "text": "包裹迟到了"
    },
    {
      "id": "T08",
      "text": "需要改快递地址"
    },
    {
      "id": "T09",
      "text": "请开电子发票"
    },
    {
      "id": "T10",
      "text": "发票抬头写错"
    },
    {
      "id": "T11",
      "text": "发票什么时候发"
    },
    {
      "id": "T12",
      "text": "重开发票"
    }
  ]
}

python3 -m venv .venv
source .venv/bin/activate
python -m pip install -r requirements.txt
python run.py

{
  "model": "gemini-2.5-flash-lite",
  "messages": [
    {
      "role": "system",
      "content": "严格只输出单个JSON对象，不要Markdown代码围栏，不要解释。只输出JSON：labels数组逐项id、label，label只能refund、delivery、invoice。恰好返回每条工单一次，不自报准确率。"
    },
    {
      "role": "user",
      "content": "任务约束（只输出JSON对象，不用代码围栏）：\n只输出JSON：labels数组逐项id、label，label只能refund、delivery、invoice。恰好返回每条工单一次，不自报准确率。\n待处理合成材料：\n{\"tickets\": [{\"id\": \"T01\", \"text\": \"退款还没到账\"}, {\"id\": \"T02\", \"text\": \"想退回未开封商品\"}, {\"id\": \"T03\", \"text\": \"退款金额需要核对\"}, {\"id\": \"T04\", \"text\": \"取消订单后退款\"}, {\"id\": \"T05\", \"text\": \"快递到哪了\"}, {\"id\": \"T06\", \"text\": \"物流显示异常\"}, {\"id\": \"T07\", \"text\": \"包裹迟到了\"}, {\"id\": \"T08\", \"text\": \"需要改快递地址\"}, {\"id\": \"T09\", \"text\": \"请开电子发票\"}, {\"id\": \"T10\", \"text\": \"发票抬头写错\"}, {\"id\": \"T11\", \"text\": \"发票什么时候发\"}, {\"id\": \"T12\", \"text\": \"重开发票\"}]}"
    }
  ],
  "temperature": 0,
  "max_tokens": 1536,
  "provider": {
    "sort": "success_rate"
  },
  "response_format": {
    "type": "json_object"
  }
}

labels = {r.get('id'): r.get('label') for r in output.get('labels', [])}
expected = {r['id']: 'refund' if n < 4 else 'delivery' if n < 8 else 'invoice' for n, r in enumerate(original['tickets'])}
classes = ['refund', 'delivery', 'invoice']
check('12工单各分类一次且类别受控', set(labels) == set(expected) and len(output.get('labels', [])) == 12 and all((v in classes for v in labels.values())))
matrix = [[0] * 3 for _ in classes]
predictions = []
for key, gold in expected.items():
    guess = labels.get(key)
    predictions.append({'id': key, 'gold': gold, 'predicted': guess, 'correct': gold == guess})
    if guess in classes:
        matrix[classes.index(gold)][classes.index(guess)] += 1
metrics = []
for j, cls in enumerate(classes):
    tp = matrix[j][j]
    fp = sum((matrix[k][j] for k in range(3) if k != j))
    fn = sum((matrix[j][k] for k in range(3) if k != j))
    metrics.append({'class': cls, 'tp': tp, 'fp': fp, 'fn': fn, 'precision': tp / (tp + fp) if tp + fp else None, 'recall': tp / (tp + fn) if tp + fn else None})
check('混淆矩阵总数等于有效分类数', sum(map(sum, matrix)) == sum((v in classes for v in labels.values())))
check('宏平均与全局准确率分别计算', len(metrics) == 3)
write_csv(artifact('class-metrics.csv'), metrics)
write_csv(artifact('ticket-predictions.csv'), predictions)
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
fig, ax = plt.subplots(figsize=(7, 5))
ax.imshow(matrix, cmap='Purples')
ax.set_xticks(range(3), classes)
ax.set_yticks(range(3), classes)
ax.set_xlabel('Predicted')
ax.set_ylabel('Human fixture label')
for y in range(3):
    for x in range(3):
        ax.text(x, y, str(matrix[y][x]), ha='center', va='center', color='white' if matrix[y][x] >= 3 else '#291343')
fig.tight_layout()
fig.savefig(artifact('confusion-matrix.png'), dpi=170)
plt.close(fig)
result = {'matrix': matrix, 'classes': classes, 'metrics': metrics, 'accuracy': sum((r['correct'] for r in predictions)) / len(expected), 'errors': [r for r in predictions if not r['correct']], 'evaluationScope': '12 synthetic examples, not general benchmark'}

{
  "matrix": [
    [
      4,
      0,
      0
    ],
    [
      0,
      4,
      0
    ],
    [
      0,
      0,
      4
    ]
  ],
  "classes": [
    "refund",
    "delivery",
    "invoice"
  ],
  "metrics": [
    {
      "class": "refund",
      "tp": 4,
      "fp": 0,
      "fn": 0,
      "precision": 1.0,
      "recall": 1.0
    },
    {
      "class": "delivery",
      "tp": 4,
      "fp": 0,
      "fn": 0,
      "precision": 1.0,
      "recall": 1.0
    },
    {
      "class": "invoice",
      "tp": 4,
      "fp": 0,
      "fn": 0,
      "precision": 1.0,
      "recall": 1.0
    }
  ],
  "accuracy": 1.0,
  "errors": [],
  "evaluationScope": "12 synthetic examples, not general benchmark"
}