# ルール解説メーカー 精度監査:統計と図の再現スクリプト(rule016) # 使い方:分析ページから CSV を2つダウンロードし、下のファイル名を合わせて source() する。 # 必要パッケージ:ggplot2(図)。irr があれば κ の検定も出す。なければ base R だけで動く部分が走る。 summary_file <- "rule_audit_summary.csv" # ゲーム別サマリ rules_file <- "rule_audit_rules.csv" # ルール単位(ロング形式) S <- read.csv(summary_file, fileEncoding = "UTF-8-BOM", stringsAsFactors = FALSE) R <- read.csv(rules_file, fileEncoding = "UTF-8-BOM", stringsAsFactors = FALSE) S <- S[S$include == 1, ]; R <- R[R$include == 1, ] ai <- R[R$origin == "ai" & R$human_verdict != "", ] ai$correct <- ai$human_verdict %in% c("ok", "minor") # 寛容基準の正誤 ai$strict <- ai$human_verdict == "ok" ai$confidence <- factor(ai$confidence, levels = c("high", "mid", "low")) ai$stratum <- factor(ai$stratum, levels = c("light", "mid", "heavy")) ## 1. 全体の割合と Wilson 95%CI wilson <- function(k, n, z = qnorm(0.975)) { p <- k/n; d <- 1 + z^2/n; c <- p + z^2/(2*n); r <- z*sqrt(p*(1-p)/n + z^2/(4*n^2)); c(p, (c-r)/d, (c+r)/d) } n <- nrow(ai) cat("寛容精度:", round(wilson(sum(ai$correct), n), 3), "\n") cat("厳格精度:", round(wilson(sum(ai$strict), n), 3), "\n") cat("幻覚率:", round(wilson(sum(ai$human_verdict == "halluc"), n), 3), "\n") tp <- sum(ai$correct); missed <- sum(S$missed_human) cat("再現率(人基準):", round(wilson(tp, tp + missed), 3), "\n") ## 2. 層・条件・確信度・種別ごとの誤り率と検定 for (f in c("stratum", "condition", "confidence", "category")) { tab <- table(ai[[f]], ai$correct) cat("\n==", f, "==\n"); print(tab); print(round(prop.table(tab, 1), 3)) if (nrow(tab) >= 2 && ncol(tab) == 2) { print(chisq.test(tab)) if (any(chisq.test(tab)$expected < 5)) print(fisher.test(tab)) } } ## 3. 人 vs AI(McNemar と κ) both <- ai[ai$ai_verdict != "", ] if (nrow(both) > 0) { both$ai_correct <- both$ai_verdict %in% c("ok", "minor") print(table(human = both$correct, ai = both$ai_correct)) print(mcnemar.test(table(both$correct, both$ai_correct))) if (requireNamespace("irr", quietly = TRUE)) { print(irr::kappa2(both[, c("human_verdict", "ai_verdict")])) } else { tab <- table(both$human_verdict, both$ai_verdict); po <- sum(diag(tab))/sum(tab) pe <- sum(rowSums(tab) * colSums(tab))/sum(tab)^2; cat("Cohen's kappa:", round((po-pe)/(1-pe), 3), "\n") } } ## 4. ゲーム単位の要約(層別の平均±SD) agg <- aggregate(cbind(p_lenient, halluc_rate, recall_human, elapsed_sec, judge_sec_per_rule, cost_jpy) ~ stratum, data = S, FUN = function(x) c(mean = mean(x, na.rm = TRUE), sd = sd(x, na.rm = TRUE))) print(agg) ## 5. ページ数との相関(Spearman) if ("pdf_pages" %in% names(S)) { for (y in c("n_ai", "missed_human", "elapsed_sec")) print(cor.test(S$pdf_pages, S[[y]], method = "spearman", exact = FALSE)) } ## 6. 図:確信度の校正図、ゲーム別の精度 if (requireNamespace("ggplot2", quietly = TRUE)) { library(ggplot2) cal <- do.call(rbind, lapply(split(ai, ai$confidence), function(d) { w <- wilson(sum(!d$correct), nrow(d)); data.frame(confidence = d$confidence[1], err = w[1], lo = w[2], hi = w[3], n = nrow(d)) })) p1 <- ggplot(cal, aes(confidence, err)) + geom_col(fill = "#d9534f", width = .6) + geom_errorbar(aes(ymin = lo, ymax = hi), width = .2) + scale_y_continuous(labels = scales::percent) + labs(x = "AI self-reported confidence", y = "Error rate (major + hallucination)") + theme_minimal(base_size = 13) ggsave("fig_calibration.png", p1, width = 5, height = 4, dpi = 300) p2 <- ggplot(S, aes(reorder(game, p_lenient), p_lenient, fill = stratum)) + geom_col() + coord_flip() + scale_y_continuous(labels = scales::percent) + labs(x = NULL, y = "Lenient precision") + theme_minimal(base_size = 13) ggsave("fig_precision_by_game.png", p2, width = 6, height = 4, dpi = 300) }