AI测试集泄漏检查：范例和验收题近似重复也要分开

{
  "examples": [
    {
      "id": "E1",
      "text": "常规品签收7天内可退。"
    },
    {
      "id": "E2",
      "text": "发票抬头可以在开票前修改。"
    },
    {
      "id": "E3",
      "text": "北京仓周末暂停发货。"
    },
    {
      "id": "E4",
      "text": "轻杯容量为300毫升。"
    },
    {
      "id": "E5",
      "text": "试用模板共3份。"
    },
    {
      "id": "E6",
      "text": "资料可离线查看。"
    }
  ],
  "holdout": [
    {
      "id": "H1",
      "text": "常规品签收7天内可退。"
    },
    {
      "id": "H2",
      "text": "开出发票以前，能够更换抬头。"
    },
    {
      "id": "H3",
      "text": "上海仓周一开始配送。"
    },
    {
      "id": "H4",
      "text": "大杯容量是500毫升。"
    },
    {
      "id": "H5",
      "text": "模板是否有音频？"
    },
    {
      "id": "H6",
      "text": "离线包如何更新？"
    }
  ]
}

python3 -m venv .venv
source .venv/bin/activate
python -m pip install -r requirements.txt
python run.py

{
  "model": "gemini-2.5-flash-lite",
  "messages": [
    {
      "role": "system",
      "content": "严格只输出单个JSON对象，不要Markdown代码围栏，不要解释。只输出JSON：candidates数组，每项example_id、holdout_id、reason。找完全重复或同意改写的候选，不能因为同一主题就判定重复。候选留人工复核，不自行删数据。"
    },
    {
      "role": "user",
      "content": "任务约束（只输出JSON对象，不用代码围栏）：\n只输出JSON：candidates数组，每项example_id、holdout_id、reason。找完全重复或同意改写的候选，不能因为同一主题就判定重复。候选留人工复核，不自行删数据。\n待处理合成材料：\n{\"examples\": [{\"id\": \"E1\", \"text\": \"常规品签收7天内可退。\"}, {\"id\": \"E2\", \"text\": \"发票抬头可以在开票前修改。\"}, {\"id\": \"E3\", \"text\": \"北京仓周末暂停发货。\"}, {\"id\": \"E4\", \"text\": \"轻杯容量为300毫升。\"}, {\"id\": \"E5\", \"text\": \"试用模板共3份。\"}, {\"id\": \"E6\", \"text\": \"资料可离线查看。\"}], \"holdout\": [{\"id\": \"H1\", \"text\": \"常规品签收7天内可退。\"}, {\"id\": \"H2\", \"text\": \"开出发票以前，能够更换抬头。\"}, {\"id\": \"H3\", \"text\": \"上海仓周一开始配送。\"}, {\"id\": \"H4\", \"text\": \"大杯容量是500毫升。\"}, {\"id\": \"H5\", \"text\": \"模板是否有音频？\"}, {\"id\": \"H6\", \"text\": \"离线包如何更新？\"}]}"
    }
  ],
  "temperature": 0,
  "max_tokens": 1536,
  "provider": {
    "sort": "success_rate"
  },
  "response_format": {
    "type": "json_object"
  }
}

ex = {r['id']: r['text'] for r in original['examples']}
hold = {r['id']: r['text'] for r in original['holdout']}
modelpairs = {(r.get('example_id'), r.get('holdout_id')) for r in output.get('candidates', [])}
check('模型候选只引用既有样例与验收ID', all((a in ex and z in hold for a, z in modelpairs)))
rows = []
exact_hold = set()
quarantine = set()
for eid, etext in ex.items():
    for hid, htext in hold.items():
        same = hashlib.sha256(norm(etext).encode()).hexdigest() == hashlib.sha256(norm(htext).encode()).hexdigest()
        sim = SequenceMatcher(None, norm(etext), norm(htext)).ratio()
        candidate = (eid, hid) in modelpairs
        if same or sim >= 0.65 or candidate:
            action = 'remove exact duplicate' if same else 'quarantine for human semantic review'
            rows.append({'example_id': eid, 'holdout_id': hid, 'same_normalized_sha': same, 'character_similarity': round(sim, 4), 'model_candidate': candidate, 'action': action})
            if same:
                exact_hold.add(hid)
            else:
                quarantine.add(hid)
clean = [r for r in original['holdout'] if r['id'] not in exact_hold | quarantine]
check('完全重复H1由独立哈希检查发现', 'H1' in exact_hold)
check('模型候选不直接当成确证删除', all((r['action'] != 'remove exact duplicate' or r['same_normalized_sha'] for r in rows)))
check('清洁验收集不含自动删除或待人工项', not {r['id'] for r in clean} & (exact_hold | quarantine))
write_csv(artifact('leakage-review.csv'), rows, ['example_id', 'holdout_id', 'same_normalized_sha', 'character_similarity', 'model_candidate', 'action'])
dump(artifact('holdout-clean.json'), clean)
result = {'exactRemoved': sorted(exact_hold), 'quarantined': sorted(quarantine), 'cleanIds': [r['id'] for r in clean], 'rows': rows, 'similarityIsNotDuplicateProof': True}

{
  "exactRemoved": [
    "H1"
  ],
  "quarantined": [
    "H2",
    "H4"
  ],
  "cleanIds": [
    "H3",
    "H5",
    "H6"
  ],
  "rows": [
    {
      "example_id": "E1",
      "holdout_id": "H1",
      "same_normalized_sha": true,
      "character_similarity": 1.0,
      "model_candidate": true,
      "action": "remove exact duplicate"
    },
    {
      "example_id": "E2",
      "holdout_id": "H2",
      "same_normalized_sha": false,
      "character_similarity": 0.3704,
      "model_candidate": true,
      "action": "quarantine for human semantic review"
    },
    {
      "example_id": "E4",
      "holdout_id": "H4",
      "same_normalized_sha": false,
      "character_similarity": 0.7273,
      "model_candidate": false,
      "action": "quarantine for human semantic review"
    }
  ],
  "similarityIsNotDuplicateProof": true
}