From 68cc90e1ed176c32acba15b43a53accf3bef9ce5 Mon Sep 17 00:00:00 2001 From: Devis Wawan Saputra Date: Thu, 22 Oct 2026 01:28:37 +0700 Subject: [PATCH 01/48] research: add real-data robustness, richer PFA and paired learner bootstrap --- scripts/robustness_extension.py | 121 ++++++++++++++++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 scripts/robustness_extension.py diff --git a/scripts/robustness_extension.py b/scripts/robustness_extension.py new file mode 100644 index 0000000..4682ae1 --- /dev/null +++ b/scripts/robustness_extension.py @@ -0,0 +1,121 @@ +"""Independent real-data robustness extension for the ASSISTments knowledge tracing benchmark. + +Execution: + PYTHONPATH=src python scripts/robustness_extension.py --epochs 25 --patience 4 --boots 2000 + +This reuses the pinned dataset adapter in src/run_experiment.py. The original test +partition has been inspected already; this analysis is EXPLORATORY, not independent +confirmation. No synthetic observations or fabricated experiment outcomes. +""" +from __future__ import annotations +import argparse +import json +import sys +from pathlib import Path +from collections import defaultdict +import numpy as np +import pandas as pd +from sklearn.compose import ColumnTransformer +from sklearn.linear_model import LogisticRegression +from sklearn.pipeline import Pipeline +from sklearn.preprocessing import OneHotEncoder, StandardScaler + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) +import run_experiment as kt + +def paired_cluster_bootstrap(reference, candidate, repetitions=2000, seed=20261022): + """Resample learners, preserve all responses, estimate pooled interaction-weighted Brier difference.""" + if repetitions < 200: + raise ValueError("Use at least 200 bootstrap repetitions") + for col in ("learner", "y", "skill"): + if not np.array_equal(reference[col].to_numpy(), candidate[col].to_numpy()): + raise ValueError("Misaligned prediction frames: " + col) + diff = (candidate["p"].to_numpy()-candidate["y"].to_numpy())**2 - (reference["p"].to_numpy()-reference["y"].to_numpy())**2 + records = pd.DataFrame({"learner":reference["learner"].astype(str).to_numpy(), "loss_diff":diff}) + grp = records.groupby("learner", sort=False)["loss_diff"].agg(["sum","count"]) + sums = grp["sum"].to_numpy(float) + counts = grp["count"].to_numpy(float) + rng = np.random.default_rng(seed) + sampled = np.empty(repetitions) + for i in range(repetitions): + index = rng.integers(0,len(sums),size=len(sums)) + sampled[i] = sums[index].sum()/counts[index].sum() + return {"difference_candidate_minus_reference":float(sums.sum()/counts.sum()), + "bootstrap_95_interval":[float(x) for x in np.quantile(sampled,[.025,.975])], + "resampling_unit":"learner", "estimand_unit":"interaction", + "n_learners":len(sums), "n_interactions":int(counts.sum()), "n_bootstrap":repetitions} + +def pfa_plus_all_skill_history_frame(rows): + features=[]; y=[]; users=[]; skills=[]; first=[] + for row in rows: + skill_success=defaultdict(int);skill_failure=defaultdict(int) + total_success=0;total_failure=0;seen=set() + sequence, outcomes=kt.aligned(row) + for s,response in zip(sequence,outcomes): + features.append({"skill":s,"skill_success":skill_success[s],"skill_failure":skill_failure[s], + "all_success":total_success,"all_failure":total_failure}) + y.append(response);users.append(str(row["user_id"]));skills.append(s);first.append(s not in seen) + if response: + skill_success[s]+=1;total_success+=1 + else: + skill_failure[s]+=1;total_failure+=1 + seen.add(s) + return pd.DataFrame(features),np.asarray(y,dtype=int),users,skills,first + +def stronger_pfa(train, test): + train_x,train_y,*_=pfa_plus_all_skill_history_frame(train) + test_x,test_y,users,skills,first=pfa_plus_all_skill_history_frame(test) + model=Pipeline([ + ("prep",ColumnTransformer([ + ("skill",OneHotEncoder(handle_unknown="ignore"),["skill"]), + ("history",StandardScaler(),["skill_success","skill_failure","all_success","all_failure"])])), + ("clf",LogisticRegression(solver="saga",max_iter=1000,C=1.0,random_state=42))]) + model.fit(train_x,train_y) + predictions=model.predict_proba(test_x)[:,1] + return kt.prediction_frame(test_y,predictions,users,skills,first) + +def run(args): + rows, data_record=kt.load_real_rows() + train, validation, test=kt.split_rows(rows,kt.SEED) + global_rate,skill_prior,_=kt.train_priors(train) + pfa=kt.predict_pfa(kt.fit_pfa(train,kt.SEED),test) + enhanced=stronger_pfa(train,test) + vocabulary=kt.skill_vocabulary(train) + kt.GRU_PATIENCE=args.patience + output={} + for seed in args.seeds: + model,history,best_epoch=kt.train_gru(train,validation,vocabulary,seed,epochs=args.epochs) + prediction=kt.predict_gru(model,test,vocabulary,global_rate,skill_prior) + output[str(seed)]={"best_epoch":best_epoch, "epochs_completed":len(history), + "best_epoch_at_training_cap":best_epoch==args.epochs, + "training_history":history, + "metrics":kt.metric_bundle(prediction["y"],prediction["p"]), + "first_and_repeated":kt.sliced_metrics(prediction), + "paired_vs_pfa":paired_cluster_bootstrap(pfa,prediction,args.boots,seed+100), + "paired_vs_stronger_pfa":paired_cluster_bootstrap(enhanced,prediction,args.boots,seed+200)} + record={"status":"complete_exploratory_real_data","dataset":data_record, + "qualification":"Prior final test data were already inspected; this is not an untouched holdout confirmation.", + "source_code":"scripts/robustness_extension.py", + "split_seed":kt.SEED, "train_learners":len(train),"validation_learners":len(validation),"test_learners":len(test), + "max_epochs":args.epochs,"patience":args.patience,"seeds":args.seeds, + "pfa":kt.metric_bundle(pfa["y"],pfa["p"]), + "stronger_pfa":kt.metric_bundle(enhanced["y"],enhanced["p"]), + "stronger_minus_original_pfa":paired_cluster_bootstrap(pfa,enhanced,args.boots), + "extended_gru":output} + path=Path(args.output) + path.parent.mkdir(parents=True,exist_ok=True) + path.write_text(json.dumps(record,indent=2),encoding="utf-8") + print(json.dumps({"output":str(path),"seed_results":{s:{"auc":v["metrics"]["roc_auc"],"brier":v["metrics"]["brier"], + "best_epoch":v["best_epoch"]} for s,v in output.items()}},indent=2)) + +if __name__=="__main__": + parser=argparse.ArgumentParser() + parser.add_argument("--epochs",type=int,default=25) + parser.add_argument("--patience",type=int,default=4) + parser.add_argument("--boots",type=int,default=2000) + parser.add_argument("--seeds",type=int,nargs="+",default=[13,42,73]) + parser.add_argument("--output",default="results/robustness_extension.json") + args=parser.parse_args() + if args.epochs<=8 or args.patience<2: parser.error("Set epochs > 8 and patience >= 2") + run(args) From 716f0fb31fad73f62a4452d779ddde7b52792756 Mon Sep 17 00:00:00 2001 From: Devis Wawan Saputra Date: Thu, 22 Oct 2026 01:29:07 +0700 Subject: [PATCH 02/48] ci: run full empirical reproducibility and robustness on GitHub Actions --- .github/workflows/arxiv-verification.yml | 67 ++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 .github/workflows/arxiv-verification.yml diff --git a/.github/workflows/arxiv-verification.yml b/.github/workflows/arxiv-verification.yml new file mode 100644 index 0000000..978ad43 --- /dev/null +++ b/.github/workflows/arxiv-verification.yml @@ -0,0 +1,67 @@ +name: arXiv verification: real-data robustness +on: + push: + branches: + - research/kt-arxiv-verification-20261022 + workflow_dispatch: +permissions: + contents: write +jobs: + verify: + if: github.actor != 'github-actions[bot]' + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install pinned scientific packages + run: | + python -m pip install --upgrade pip + pip install -r requirements-repro.txt + pip install pytest + - name: Run repository unit tests + run: PYTHONPATH=src python -m pytest -q + - name: Reproduce original real-data experiment + run: PYTHONPATH=src python src/run_experiment.py + - name: Run enhanced baselines and extended GRU on actual data + run: PYTHONPATH=src python scripts/robustness_extension.py --epochs 25 --patience 4 --boots 2000 --seeds 13 42 73 + - name: Validate empirical outputs + run: | + python - <<'PY' + import json + from pathlib import Path + original=json.loads(Path("results/metrics.json").read_text()) + extension=json.loads(Path("results/robustness_extension.json").read_text()) + assert original["dataset"]["n_learners"]==extension["dataset"]["n_learners"] + assert original["dataset"]["n_interactions"]==extension["dataset"]["n_interactions"] + assert extension["status"]=="complete_exploratory_real_data" + for run in extension["extended_gru"].values(): + assert len(run["training_history"])>=1 + assert run["paired_vs_pfa"]["n_learners"]==623 + assert run["paired_vs_stronger_pfa"]["n_interactions"]==41228 + print("REAL-DATA VERIFICATION PASSED: numeric artifacts match sample identifiers") + PY + - name: Upload verification results + if: always() + uses: actions/upload-artifact@v4 + with: + name: kt-arxiv-verification + path: | + results/metrics.json + results/summary.md + results/robustness_extension.json + results/figures/ + if-no-files-found: warn + - name: Commit empirical records to verification branch + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add results/metrics.json results/summary.md results/figures/ results/robustness_extension.json + if git diff --cached --quiet; then + echo "No changes to empirical artifacts" + else + git commit -m "research: record rerun original and extended real-data results" + git push origin HEAD:research/kt-arxiv-verification-20261022 + fi From 13cab4779a1e5c34c7fed9ec3279f41ae618ffed Mon Sep 17 00:00:00 2001 From: Devis Wawan Saputra Date: Thu, 22 Oct 2026 01:32:21 +0700 Subject: [PATCH 03/48] test: verify pooled learner bootstrap and leakage-free enriched PFA features --- tests/test_arxiv_robustness.py | 35 ++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 tests/test_arxiv_robustness.py diff --git a/tests/test_arxiv_robustness.py b/tests/test_arxiv_robustness.py new file mode 100644 index 0000000..9082967 --- /dev/null +++ b/tests/test_arxiv_robustness.py @@ -0,0 +1,35 @@ +"""Unit tests for manuscript robustness extension (no empirical datasets or synthetic study claims).""" +import importlib.util +import unittest +from pathlib import Path +import pandas as pd + +ROOT=Path(__file__).resolve().parents[1] +SPEC=importlib.util.spec_from_file_location("arxiv_robustness",ROOT/"scripts"/"robustness_extension.py") +MODULE=importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + +class RobustnessTests(unittest.TestCase): + def test_cluster_bootstrap_preserves_interaction_weighted_target(self): + reference=pd.DataFrame({"learner":["a","a","a","b"],"y":[0,0,0,0], + "skill":["x","x","x","x"],"p":[0.,0.,0.,0.]}) + candidate=reference.copy() + candidate["p"]=[1.,1.,1.,0.] + result=MODULE.paired_cluster_bootstrap(reference,candidate,repetitions=300,seed=2) + self.assertAlmostEqual(result["difference_candidate_minus_reference"],.75) + self.assertEqual(result["n_learners"],2) + self.assertEqual(result["n_interactions"],4) + candidate.loc[0,"learner"]="different" + with self.assertRaises(ValueError): + MODULE.paired_cluster_bootstrap(reference,candidate,repetitions=300) + def test_cross_skill_history_uses_only_previous_outcomes(self): + rows=[{"user_id":"one","skill_ids":["x","x","y"],"grades":[1,0,1]}] + x,y,users,skills,first=MODULE.pfa_plus_all_skill_history_frame(rows) + self.assertEqual(y.tolist(),[1,0,1]) + self.assertEqual(x["all_success"].tolist(),[0,1,1]) + self.assertEqual(x["all_failure"].tolist(),[0,0,1]) + self.assertEqual(x["skill_success"].tolist(),[0,1,0]) + self.assertEqual(first,[True,False,True]) + +if __name__=="__main__": + unittest.main() From cab0ca88b821a7077e2976811fcc55b8952ec562 Mon Sep 17 00:00:00 2001 From: Devis Wawan Saputra Date: Thu, 22 Oct 2026 01:47:24 +0700 Subject: [PATCH 04/48] ci: execute full real-data reproduction and robustness in PR with downloadable evidence --- .github/workflows/ci.yml | 47 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d202d70..b510abb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,3 +22,50 @@ jobs: pip install -r requirements.txt pip install pytest - run: PYTHONPATH=src python -m pytest -q + + empirical-arxiv-check: + if: github.event_name == 'pull_request' && github.head_ref == 'research/kt-arxiv-verification-20261022' + runs-on: ubuntu-latest + timeout-minutes: 180 + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: '3.11' + - name: Install reproducibility environment + run: | + python -m pip install --upgrade pip + pip install -r requirements-repro.txt + pip install pytest + - name: Rerun original experiment with real ASSISTments mirror + run: PYTHONPATH=src python src/run_experiment.py + - name: Execute extended GRU and richer PFA robustness + run: PYTHONPATH=src python scripts/robustness_extension.py --epochs 25 --patience 4 --boots 2000 --seeds 13 42 73 + - name: Validate outputs against real-data identifiers + run: | + python - <<'PY' + import json + from pathlib import Path + original=json.loads(Path("results/metrics.json").read_text()) + extension=json.loads(Path("results/robustness_extension.json").read_text()) + assert original["dataset"]["n_learners"]==extension["dataset"]["n_learners"]==4148 + assert original["dataset"]["n_interactions"]==extension["dataset"]["n_interactions"]==274331 + assert original["dataset"]["parquet_sha256"]==extension["dataset"]["parquet_sha256"] + assert extension["status"]=="complete_exploratory_real_data" + assert len(extension["extended_gru"])==3 + for run in extension["extended_gru"].values(): + assert run["paired_vs_pfa"]["n_learners"]==623 + assert run["paired_vs_stronger_pfa"]["n_interactions"]==41228 + print("Original + robustness real-data results: numerical validation passed") + PY + - name: Upload numerical evidence for manuscript update + if: always() + uses: actions/upload-artifact@v4 + with: + name: assistments-arxiv-reproduction + path: | + results/metrics.json + results/summary.md + results/robustness_extension.json + results/figures/ + if-no-files-found: warn From 1005f60065d9ee7af2ca5649c8d280bbf7507d6c Mon Sep 17 00:00:00 2001 From: Devis Wawan Saputra Date: Thu, 22 Oct 2026 16:15:36 +0700 Subject: [PATCH 05/48] =?UTF-8?q?research:=20freeze=20independent=202015?= =?UTF-8?q?=20test=20=E2=80=94=20CONFIRMATORY=5FPROTOCOL=5F2015.md?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CONFIRMATORY_PROTOCOL_2015.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 CONFIRMATORY_PROTOCOL_2015.md diff --git a/CONFIRMATORY_PROTOCOL_2015.md b/CONFIRMATORY_PROTOCOL_2015.md new file mode 100644 index 0000000..087a076 --- /dev/null +++ b/CONFIRMATORY_PROTOCOL_2015.md @@ -0,0 +1,13 @@ +# Frozen external-cohort replication protocol (2026-10-22) + +**Research question.** On a previously unanalysed, different-year learner sample from ASSISTments 2015, does the pre-existing extended GRU pipeline achieve lower held-out pooled Brier loss than the pre-existing enriched PFA pipeline? The models are retrained on 2015 training learners. This tests cross-cohort *replication of the comparison*, not direct transport of the 2009 fitted model or causal effects of architecture. + +- Source: official ASSISTments 2015 Skill Builder Data, `2015_100_skill_builders_main_problems.csv`, Google Drive file ID `0B_hO8cnpcIMgUGZzRnh3bHJrSjQ` linked from https://sites.google.com/site/assistmentsdata/datasets/2015-assistments-skill-builder-data. +- Require `user_id`, `log_id` (event order), `sequence_id` (100 skill-builder concepts), and binary `correct`. Require non-null, unambiguous records and unique learner-event pairs. Sort each learner chronologically using `log_id`. Never use later responses as model inputs for earlier outcomes. +- Learner sampling: uniformly sample without replacement **5,000 distinct learners**, deterministically with seed `20261022` after sorting IDs lexicographically. No filtering based on outcomes or model performance, except invalid records verified before learner sampling; log exclusions. +- Split: shuffle selected learners with seed `20261022`, then 70/15/15, disjoint between training/validation/test. A new dataset, independent of the 2009 train/validation/test learners. All model selection on training/validation; test inspected once. +- Fixed models: pre-existing stronger PFA `stronger_pfa()` (skill success/failure and cross-skill prior counts, logistic regression C=1, SAGA max_iter=1000) and `GRUKnowledgeTracer` model (32-token,16-query,64-hidden, Adam lr=0.002, validation early stopping, max 25 epochs, patience 4, random seed 42). No test-driven tuning. +- Primary estimand: interaction-weighted paired Brier-score difference `GRU - enriched PFA`. Resample **test learners** with replacement (2,000 replicates, seed 20261022), retaining all responses per selected learner. 95% percentile confidence interval. Claim directional external-cohort confirmation **only if** upper CI bound < 0 and computation/data gates pass; if the upper bound >= 0, record as not confirmed. Sign criterion is not proof of architecture causation. +- Secondary descriptive quantities: ROC-AUC, log loss, calibration/ECE, learner and interaction counts, outcome prevalence, checksums and provenance. No choosing between datasets/seeds after seeing test outcomes. +- Scope caveats: This is replication within ASSISTments in another year, on a random 5,000-learner subsample, not independent educational-system validity; 2015 skill-builder `sequence_id` concepts may not be the same units as 2009 mirror skill IDs, and enriched PFA and GRU have unequal information/capacity. +- No outcome claims before executed logs and machine-readable artifacts have been verified. Inability to retrieve/verify the official source means **blocked**, not passed. \ No newline at end of file From 6c94c913b029d830dfb46d337f9470cd1ad227bb Mon Sep 17 00:00:00 2001 From: Devis Wawan Saputra Date: Thu, 22 Oct 2026 16:15:38 +0700 Subject: [PATCH 06/48] =?UTF-8?q?research:=20freeze=20independent=202015?= =?UTF-8?q?=20test=20=E2=80=94=20scripts/confirmatory=5F2015.py?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/confirmatory_2015.py | 174 +++++++++++++++++++++++++++++++++++ 1 file changed, 174 insertions(+) create mode 100644 scripts/confirmatory_2015.py diff --git a/scripts/confirmatory_2015.py b/scripts/confirmatory_2015.py new file mode 100644 index 0000000..c260f40 --- /dev/null +++ b/scripts/confirmatory_2015.py @@ -0,0 +1,174 @@ +"""One-shot, frozen independent ASSISTments-2015 cohort replication. + +Run from repository root: + PYTHONPATH=src python scripts/confirmatory_2015.py +Must not be retuned after inspecting holdout outcomes. See CONFIRMATORY_PROTOCOL_2015.md. +""" +from __future__ import annotations + +import hashlib +import json +import random +import sys +import warnings +from pathlib import Path + +import numpy as np +import pandas as pd +from sklearn.exceptions import ConvergenceWarning + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) +from robustness_extension import paired_cluster_bootstrap, stronger_pfa +import run_experiment as kt + +SOURCE_URL = "https://sites.google.com/site/assistmentsdata/datasets/2015-assistments-skill-builder-data" +GOOGLE_FILE_ID = "0B_hO8cnpcIMgUGZzRnh3bHJrSjQ" +SEED = 20261022 +N_SAMPLE = 5000 +BOOTS = 2000 +OUTPUT = Path("results/confirmatory_assist2015.json") + + +def download_official_csv(path: Path) -> None: + """Only the official ASSISTments-linked file ID is accepted for network retrieval.""" + if path.exists(): + print(f"Using locally supplied file {path}; source authenticity requires author audit") + return + import gdown + path.parent.mkdir(parents=True, exist_ok=True) + result = gdown.download(id=GOOGLE_FILE_ID, output=str(path), quiet=False) + if result is None or not path.exists(): + raise RuntimeError("Official Google Drive CSV download failed; no fallback to an unverified mirror") + with path.open("rb") as handle: + if handle.read(128).lstrip().startswith((b" tuple[list[dict], dict]: + digest = hashlib.sha256(csv_path.read_bytes()).hexdigest() + raw = pd.read_csv(csv_path, low_memory=False) + raw.columns = [str(c).strip().lower() for c in raw.columns] + required = ("user_id", "log_id", "sequence_id", "correct") + if len(set(raw.columns)) != len(raw.columns) or any(c not in raw for c in required): + raise ValueError(f"Expected unique columns {required}, found {list(raw.columns)}") + source_n = len(raw) + df = raw[list(required)].copy() + invalid_null = df.isna().any(axis=1) + for name in ("log_id", "correct"): + df[name] = pd.to_numeric(df[name], errors="coerce") + invalid = invalid_null | df.isna().any(axis=1) | ~df["correct"].isin([0, 1]) + bad_n = int(invalid.sum()) + if bad_n > 0: + # Source data may include missing records; the exclusion rule is declared ex ante. + df = df.loc[~invalid].copy() + if len(df) == 0 or bad_n / source_n > 0.05: + raise ValueError(f"Too few valid records: {bad_n}/{source_n} excluded (5% ceiling)") + if df.duplicated(["user_id", "log_id"]).any(): + raise ValueError("Duplicate learner-event key in 2015 raw file; manual provenance audit needed") + if not np.isfinite(df["log_id"]).all(): + raise ValueError("Nonfinite chronology field") + if (df["log_id"] != df["log_id"].astype("int64")).any(): + raise ValueError("Nonintegral chronology ID") + df["correct"] = df["correct"].astype(int) + df["user_id"] = df["user_id"].astype(str) + df["sequence_id"] = df["sequence_id"].astype(str) + df = df.sort_values(["user_id", "log_id"], kind="stable") + unique_users = sorted(df["user_id"].unique()) + if len(unique_users) < N_SAMPLE: + raise ValueError(f"Need {N_SAMPLE} learners, found {len(unique_users)}") + sampled = set(random.Random(SEED).sample(unique_users, N_SAMPLE)) + chosen = df[df["user_id"].isin(sampled)] + rows = [ + {"user_id": str(uid), "skill_ids": group["sequence_id"].tolist(), + "grades": group["correct"].tolist()} + for uid, group in chosen.groupby("user_id", sort=True) + ] + assert kt.validate_unique_learner_rows(rows) == N_SAMPLE + assert all(len(r["grades"]) == len(r["skill_ids"]) for r in rows) + metadata = { + "official_portal": SOURCE_URL, + "official_linked_drive_file_id": GOOGLE_FILE_ID, + "raw_filename": csv_path.name, + "source_sha256": digest, + "source_total_rows": source_n, + "excluded_invalid_rows": bad_n, + "source_unique_eligible_learners": len(unique_users), + "sample_seed": SEED, + "sample_learners": N_SAMPLE, + "sample_interactions": int(sum(len(r["grades"]) for r in rows)), + "chronology": "ascending log_id within user_id", + "concept": "sequence_id (skill builder; not 2009 mirror skill_id)", + } + return rows, metadata + + +def evaluate(csv_path: Path) -> dict: + rows, dataset = load_rows(csv_path) + train, validation, test = kt.split_rows(rows, seed=SEED) + ids = [set(str(r["user_id"]) for r in block) for block in (train, validation, test)] + assert not (ids[0] & ids[1] or ids[0] & ids[2] or ids[1] & ids[2]) + global_prior, skill_prior, _ = kt.train_priors(train) + # These fixed protocols were written before seeing any 2015 test outcomes. + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always", ConvergenceWarning) + reference = stronger_pfa(train, test) + convergence_warnings = [str(x.message) for x in w if issubclass(x.category, ConvergenceWarning)] + vocab = kt.skill_vocabulary(train) + kt.GRU_PATIENCE = 4 + model, history, best_epoch = kt.train_gru(train, validation, vocab, seed=42, epochs=25) + candidate = kt.predict_gru(model, test, vocab, global_prior, skill_prior) + comparison = paired_cluster_bootstrap(reference, candidate, repetitions=BOOTS, seed=SEED) + metrics = {"enriched_pfa": kt.metric_bundle(reference["y"], reference["p"]), + "extended_gru": kt.metric_bundle(candidate["y"], candidate["p"])} + positive = comparison["bootstrap_95_interval"][1] < 0 + validity_gate = ( + not convergence_warnings + and len(train) > 0 and len(validation) > 0 and len(test) > 0 + and not (best_epoch == 25) + and np.isfinite(comparison["difference_candidate_minus_reference"]) + ) + result = { + "status": "completed" if validity_gate else "completed_with_unresolved_diagnostics", + "directional_replication_confirmed": bool(positive and validity_gate), + "test_is_external_cohort_not_prior_2009_test": True, + "source_integrity_scope": "Official-source URL/id and downloaded-byte SHA; no independent publisher checksum available", + "data": dataset, + "protocol": {"sample_learners": N_SAMPLE, "sample_seed": SEED, + "split_seed": SEED, + "train_learners": len(train), "validation_learners": len(validation), "test_learners": len(test), + "gru_seed": 42, "gru_max_epochs": 25, "gru_patience": 4, + "bootstrap_seed": SEED, "bootstrap_draws": BOOTS, + "predeclared_primary_estimand": "GRU - enhanced PFA pooled interaction-weighted Brier", + "directional_rule": "Upper endpoint of 95% percentile learner-cluster bootstrap CI < 0; must pass diagnostic gates"}, + "metrics": metrics, + "brier_comparison": comparison, + "gru_best_validation_epoch": int(best_epoch), + "gru_epochs_completed": len(history), + "pfa_convergence_warnings": convergence_warnings, + "conclusion_scope": "Different-year ASSISTments learner-subset replication. Not source parity for 2009 and not a causal attribution of architecture.", + } + return result + + +def main(): + import argparse + parser = argparse.ArgumentParser() + parser.add_argument("--csv", default="data/cache/assistments_2015_official.csv") + parser.add_argument("--output", default=str(OUTPUT)) + args = parser.parse_args() + source = Path(args.csv) + download_official_csv(source) + result = evaluate(source) + path = Path(args.output) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(result, indent=2), encoding="utf-8") + print(json.dumps({"status": result["status"], + "confirmed": result["directional_replication_confirmed"], + "n_test_learners": result["protocol"]["test_learners"], + "metrics": result["metrics"], + "brier_difference": result["brier_comparison"]}, indent=2)) + + +if __name__ == "__main__": + main() \ No newline at end of file From 4c932c651f9c8d6caaf78b789948e19b5c308be5 Mon Sep 17 00:00:00 2001 From: Devis Wawan Saputra Date: Thu, 22 Oct 2026 16:15:41 +0700 Subject: [PATCH 07/48] =?UTF-8?q?research:=20freeze=20independent=202015?= =?UTF-8?q?=20test=20=E2=80=94=20.github/workflows/confirmatory-2015.yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/confirmatory-2015.yml | 62 +++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 .github/workflows/confirmatory-2015.yml diff --git a/.github/workflows/confirmatory-2015.yml b/.github/workflows/confirmatory-2015.yml new file mode 100644 index 0000000..51d9718 --- /dev/null +++ b/.github/workflows/confirmatory-2015.yml @@ -0,0 +1,62 @@ +name: External cohort confirmation — ASSISTments 2015 +on: + push: + branches: + - research/kt-fresh-cohort-20261022 + paths: + - scripts/confirmatory_2015.py + - .github/workflows/confirmatory-2015.yml + - CONFIRMATORY_PROTOCOL_2015.md + - scripts/robustness_extension.py + - src/run_experiment.py + workflow_dispatch: +permissions: + contents: write +jobs: + external-cohort: + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Install verified environment and Google Drive downloader + run: | + python -m pip install -r requirements-repro.txt + python -m pip install 'gdown>=5.2,<6' pytest + - name: Compile frozen confirmatory analysis + run: | + python -m py_compile scripts/confirmatory_2015.py + - name: Run independent-cohort analysis + run: PYTHONPATH=src python scripts/confirmatory_2015.py + - name: Validate output schema and data isolation + run: | + python - <<'PY' + import json + from pathlib import Path + a = json.loads(Path('results/confirmatory_assist2015.json').read_text()) + assert a['protocol']['sample_learners'] == 5000 + assert sum(a['protocol'][k] for k in ('train_learners','validation_learners','test_learners')) == 5000 + assert a['brier_comparison']['n_learners'] == a['protocol']['test_learners'] + assert a['brier_comparison']['n_bootstrap'] == 2000 + assert len(a['data']['source_sha256']) == 64 + print('External-cohort result recorded. Confirmation:', a['directional_replication_confirmed']) + PY + - name: Upload results and logs even if run fails + if: always() + uses: actions/upload-artifact@v4 + with: + name: knowledge-tracing-2015-confirmation + path: results/confirmatory_assist2015.json + if-no-files-found: warn + - name: Commit metrics to branch without changing previous paper results + if: success() + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add results/confirmatory_assist2015.json + if ! git diff --cached --quiet; then + git commit -m "research: freeze ASSISTments 2015 external-cohort test outcome" + git push origin HEAD:research/kt-fresh-cohort-20261022 + fi \ No newline at end of file From 35fb53785f558f0f386ee50db798312e8030800a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 22 Oct 2026 09:18:29 +0000 Subject: [PATCH 08/48] research: freeze ASSISTments 2015 external-cohort test outcome --- results/confirmatory_assist2015.json | 73 ++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 results/confirmatory_assist2015.json diff --git a/results/confirmatory_assist2015.json b/results/confirmatory_assist2015.json new file mode 100644 index 0000000..ffd25ec --- /dev/null +++ b/results/confirmatory_assist2015.json @@ -0,0 +1,73 @@ +{ + "status": "completed_with_unresolved_diagnostics", + "directional_replication_confirmed": false, + "test_is_external_cohort_not_prior_2009_test": true, + "source_integrity_scope": "Official-source URL/id and downloaded-byte SHA; no independent publisher checksum available", + "data": { + "official_portal": "https://sites.google.com/site/assistmentsdata/datasets/2015-assistments-skill-builder-data", + "official_linked_drive_file_id": "0B_hO8cnpcIMgUGZzRnh3bHJrSjQ", + "raw_filename": "assistments_2015_official.csv", + "source_sha256": "75e1f46131d5897a388c82411bc3032279899cb1f1e66a9043354d41234407a0", + "source_total_rows": 708631, + "excluded_invalid_rows": 24830, + "source_unique_eligible_learners": 19840, + "sample_seed": 20261022, + "sample_learners": 5000, + "sample_interactions": 170889, + "chronology": "ascending log_id within user_id", + "concept": "sequence_id (skill builder; not 2009 mirror skill_id)" + }, + "protocol": { + "sample_learners": 5000, + "sample_seed": 20261022, + "split_seed": 20261022, + "train_learners": 3500, + "validation_learners": 750, + "test_learners": 750, + "gru_seed": 42, + "gru_max_epochs": 25, + "gru_patience": 4, + "bootstrap_seed": 20261022, + "bootstrap_draws": 2000, + "predeclared_primary_estimand": "GRU - enhanced PFA pooled interaction-weighted Brier", + "directional_rule": "Upper endpoint of 95% percentile learner-cluster bootstrap CI < 0; must pass diagnostic gates" + }, + "metrics": { + "enriched_pfa": { + "n": 25089, + "positive_rate": 0.7285264458527642, + "brier": 0.18141712741703564, + "log_loss": 0.5433839611564283, + "average_precision": 0.8427901868527623, + "ece_10": 0.015065578937170483, + "roc_auc": 0.6895656527695257 + }, + "extended_gru": { + "n": 25089, + "positive_rate": 0.7285264458527642, + "brier": 0.17643847175634333, + "log_loss": 0.529606322036061, + "average_precision": 0.8597931958258713, + "ece_10": 0.014323575497811158, + "roc_auc": 0.7077904092022121 + } + }, + "brier_comparison": { + "difference_candidate_minus_reference": -0.004978655660692301, + "bootstrap_95_interval": [ + -0.0069731289320504345, + -0.0033563705430400648 + ], + "resampling_unit": "learner", + "estimand_unit": "interaction", + "n_learners": 750, + "n_interactions": 25089, + "n_bootstrap": 2000 + }, + "gru_best_validation_epoch": 9, + "gru_epochs_completed": 13, + "pfa_convergence_warnings": [ + "The max_iter was reached which means the coef_ did not converge" + ], + "conclusion_scope": "Different-year ASSISTments learner-subset replication. Not source parity for 2009 and not a causal attribution of architecture." +} \ No newline at end of file From 43a119bac1e240e49ca47c0fe9272394be68385d Mon Sep 17 00:00:00 2001 From: Devis Wawan Saputra Date: Thu, 22 Oct 2026 16:21:51 +0700 Subject: [PATCH 09/48] =?UTF-8?q?research:=20freeze=20disjoint=20learner-c?= =?UTF-8?q?ohort=20validation=20=E2=80=94=20CONFIRMATORY=5FSTAGE2=5FPROTOC?= =?UTF-8?q?OL=5F2015.md?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CONFIRMATORY_STAGE2_PROTOCOL_2015.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 CONFIRMATORY_STAGE2_PROTOCOL_2015.md diff --git a/CONFIRMATORY_STAGE2_PROTOCOL_2015.md b/CONFIRMATORY_STAGE2_PROTOCOL_2015.md new file mode 100644 index 0000000..1361365 --- /dev/null +++ b/CONFIRMATORY_STAGE2_PROTOCOL_2015.md @@ -0,0 +1,18 @@ +# Frozen stage-2, disjoint-cohort confirmatory replication + +**Reason for independent rerun.** The predeclared first external-cohort execution (GitHub run 39754046492; source SHA 75e1f46131d5897a388c82411bc3032279899cb1f1e66a9043354d41234407a0) generated estimates, but PFA `saga` emitted a nonconvergence warning and the planned diagnostic gate marked the test `completed_with_unresolved_diagnostics`. That result is retained without a confirmatory label. The correction addresses only numerical solver convergence; it must not select hyperparameters based on the observed direction or magnitude of the first test result. + +## New learner cohort and data provenance + +Use the **same official ASSISTments 2015 raw CSV bytes**, SHA-256 above, linked from the official ASSISTments portal. Apply exactly the same predeclared validity filters and chronological order (`log_id`) as in stage 1. Lexicographically sort eligible learner IDs, reconstruct the 5,000 participants originally sampled with `random.Random(20261022).sample(ids, 5000)`, exclude ALL 5,000 of those previously analysed learners, and sample 5,000 learners from the remaining eligible IDs with `random.Random(20261023).sample(remaining, 5000)`. Use a learner-disjoint 70/15/15 train/validation/test split with seed `20261023`. Assert zero overlap with stage 1 participants. No stage-2 outcomes have been inspected when this protocol was specified. + +## Fixed models and criterion + +- **PFA**: the same enriched information features (`skill`, within-skill previous correct/incorrect counts, across-skill previous correct/incorrect counts), StandardScaler for counts, OneHotEncoder for skill, logistic regression regularization `C=1.0`. Replace the nonconverged `saga` numerical solver by fixed `lbfgs` with `max_iter=2000`, `tol=1e-8`, to estimate the *same regularized logistic objective*. Record and reject nonconvergence warnings; **no alternative solvers, seeds, or hyperparameters after this check**. +- **GRU**: same implementation as first execution; train only on stage-2 training learners, select epoch only on stage-2 validation learners. Seed 42, up to 25 epochs, patience 4, Adam LR .002, token dim 32, query dim 16, hidden 64. +- Primary estimand: paired pooled interaction-weighted Brier loss GRU minus PFA across stage-2 test learners. Learner-cluster percentile bootstrap 2,000 replicates, seed 20261023. Directional result counts as *confirmed on this external 2015 subset* only if upper 95% percentile endpoint < 0 and all data/provenance/convergence guards pass. Otherwise it is a failed or indeterminate test; publish unchanged. Do not compare different PFA solvers on these stage-2 test outcomes to pick the favorable one. +- Secondary descriptive measures: ROC-AUC, log loss, ECE, sample sizes and observed correctness prevalence; not selection criteria. + +## Scope + +A successful stage-2 test would independently replicate the direction of a predictive difference on a different-year ASSISTments cohort, with a numerically converged baseline. It does not prove that GRU architecture alone causes the difference, demonstrate source parity between the original 2009 mirror and corrected official data, or establish generalizability outside the analyzed contexts. This is not a registered external trial or blind third-party peer review. \ No newline at end of file From d7dc193a1a2173e5eca958366f5fc97bc76ea184 Mon Sep 17 00:00:00 2001 From: Devis Wawan Saputra Date: Thu, 22 Oct 2026 16:21:54 +0700 Subject: [PATCH 10/48] =?UTF-8?q?research:=20freeze=20disjoint=20learner-c?= =?UTF-8?q?ohort=20validation=20=E2=80=94=20scripts/confirmatory=5F2015=5F?= =?UTF-8?q?stage2.py?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/confirmatory_2015_stage2.py | 152 ++++++++++++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 scripts/confirmatory_2015_stage2.py diff --git a/scripts/confirmatory_2015_stage2.py b/scripts/confirmatory_2015_stage2.py new file mode 100644 index 0000000..568d87e --- /dev/null +++ b/scripts/confirmatory_2015_stage2.py @@ -0,0 +1,152 @@ +"""Untouched 2015 learner cohort, disjoint from the nonconverged stage-1 pilot. + +Protocol frozen in CONFIRMATORY_STAGE2_PROTOCOL_2015.md before any stage-2 outcomes. +""" +from __future__ import annotations +import hashlib +import json +import random +import sys +import warnings +from pathlib import Path + +import numpy as np +import pandas as pd +from sklearn.compose import ColumnTransformer +from sklearn.exceptions import ConvergenceWarning +from sklearn.linear_model import LogisticRegression +from sklearn.pipeline import Pipeline +from sklearn.preprocessing import OneHotEncoder, StandardScaler + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'src')) +from robustness_extension import paired_cluster_bootstrap, pfa_plus_all_skill_history_frame +from confirmatory_2015 import download_official_csv, GOOGLE_FILE_ID, SOURCE_URL +import run_experiment as kt + +FIRST_SEED=20261022 +SECOND_SEED=20261023 +SAMPLE_SIZE=5000 +SOURCE_SHA256='75e1f46131d5897a388c82411bc3032279899cb1f1e66a9043354d41234407a0' + + +def cohort_from_official_csv(path:Path): + checksum=hashlib.sha256(path.read_bytes()).hexdigest() + if checksum!=SOURCE_SHA256: + raise ValueError('Official ASSISTments-2015 source checksum changed since stage 1; stop for provenance audit') + df=pd.read_csv(path,low_memory=False) + df.columns=[str(c).strip().lower() for c in df.columns] + columns=['user_id','log_id','sequence_id','correct'] + if len(set(df.columns))!=len(df.columns) or any(c not in df for c in columns): + raise ValueError('Missing/nonunique required source column') + orig_n=len(df) + df=df[columns].copy() + missing=df.isna().any(axis=1) + df['log_id']=pd.to_numeric(df['log_id'],errors='coerce') + df['correct']=pd.to_numeric(df['correct'],errors='coerce') + invalid=missing|df.isna().any(axis=1)|~df['correct'].isin([0,1]) + exclusions=int(invalid.sum()) + if orig_n==0 or exclusions/orig_n>0.05: + raise ValueError('Invalid record rate exceeds frozen threshold') + df=df.loc[~invalid].copy() + if df.duplicated(['user_id','log_id']).any(): + raise ValueError('Duplicate learner-event identifiers') + if not np.isfinite(df['log_id']).all() or (df['log_id']!=df['log_id'].astype('int64')).any(): + raise ValueError('Malformed chronological log IDs') + df['correct']=df['correct'].astype(int) + df['user_id']=df['user_id'].astype(str) + df['sequence_id']=df['sequence_id'].astype(str) + df=df.sort_values(['user_id','log_id'],kind='stable') + users=sorted(df['user_id'].unique()) + stage1=set(random.Random(FIRST_SEED).sample(users,SAMPLE_SIZE)) + remaining=[u for u in users if u not in stage1] + if len(remaining) Date: Thu, 22 Oct 2026 16:21:56 +0700 Subject: [PATCH 11/48] =?UTF-8?q?research:=20freeze=20disjoint=20learner-c?= =?UTF-8?q?ohort=20validation=20=E2=80=94=20.github/workflows/confirmatory?= =?UTF-8?q?-2015-stage2.yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../workflows/confirmatory-2015-stage2.yml | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 .github/workflows/confirmatory-2015-stage2.yml diff --git a/.github/workflows/confirmatory-2015-stage2.yml b/.github/workflows/confirmatory-2015-stage2.yml new file mode 100644 index 0000000..78ec559 --- /dev/null +++ b/.github/workflows/confirmatory-2015-stage2.yml @@ -0,0 +1,60 @@ +name: External cohort confirmation — second untouched ASSISTments 2015 cohort +on: + push: + branches: + - research/kt-fresh-cohort-20261022 + paths: + - CONFIRMATORY_STAGE2_PROTOCOL_2015.md + - scripts/confirmatory_2015_stage2.py + - .github/workflows/confirmatory-2015-stage2.yml + workflow_dispatch: +permissions: + contents: write +jobs: + second-cohort: + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install frozen computational packages + run: | + python -m pip install -r requirements-repro.txt + python -m pip install 'gdown>=5.2,<6' + - name: Compile and inspect frozen code + run: python -m py_compile scripts/confirmatory_2015.py scripts/confirmatory_2015_stage2.py + - name: Run untouched population study + run: PYTHONPATH=src python scripts/confirmatory_2015_stage2.py + - name: Check full independent source and test outputs + run: | + python - <<'PY' + import json + from pathlib import Path + a=json.loads(Path('results/confirmatory_assist2015_stage2.json').read_text()) + assert a['data']['sha256']=='75e1f46131d5897a388c82411bc3032279899cb1f1e66a9043354d41234407a0' + assert a['data']['stage2_intersection_stage1']==0 + assert a['data']['stage1_excluded_learners']==5000 + assert a['data']['stage2_sampled_learners']==5000 + assert sum(a['protocol'][x] for x in ('train_learners','validation_learners','test_learners'))==5000 + assert a['paired_brier']['n_learners']==750 + assert a['paired_brier']['n_bootstrap']==2000 + print('Recorded status:',a['status'],'directional replication:',a['directional_replication_confirmed']) + PY + - uses: actions/upload-artifact@v4 + if: always() + with: + name: assistments2015-stage2-validation + path: results/confirmatory_assist2015_stage2.json + if-no-files-found: warn + - name: Commit frozen results (not raw student records) + if: success() + run: | + git config user.name 'github-actions[bot]' + git config user.email '41898282+github-actions[bot]@users.noreply.github.com' + git add results/confirmatory_assist2015_stage2.json + if ! git diff --cached --quiet; then + git commit -m 'research: record untouched cohort confirmatory findings' + git push origin HEAD:research/kt-fresh-cohort-20261022 + fi \ No newline at end of file From 2df2f46b2126d1efe05480926a86f931dce365bb Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 22 Oct 2026 09:24:37 +0000 Subject: [PATCH 12/48] research: record untouched cohort confirmatory findings --- results/confirmatory_assist2015_stage2.json | 76 +++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 results/confirmatory_assist2015_stage2.json diff --git a/results/confirmatory_assist2015_stage2.json b/results/confirmatory_assist2015_stage2.json new file mode 100644 index 0000000..6383d28 --- /dev/null +++ b/results/confirmatory_assist2015_stage2.json @@ -0,0 +1,76 @@ +{ + "status": "passed_all_computational_gates", + "directional_replication_confirmed": true, + "test_supports_direction": true, + "stage1_result_used_for_model_selection": false, + "data": { + "source_url": "https://sites.google.com/site/assistmentsdata/datasets/2015-assistments-skill-builder-data", + "drive_id": "0B_hO8cnpcIMgUGZzRnh3bHJrSjQ", + "sha256": "75e1f46131d5897a388c82411bc3032279899cb1f1e66a9043354d41234407a0", + "total_source_rows": 708631, + "excluded_invalid_rows": 24830, + "eligible_learners": 19840, + "stage1_excluded_learners": 5000, + "stage2_sampling_pool": 14840, + "stage2_sampled_learners": 5000, + "stage2_sample_interactions": 172614, + "stage2_intersection_stage1": 0, + "sorting": "log_id ascending per learner, sequence_id as skill-builder concept" + }, + "protocol": { + "stage1_sample_seed": 20261022, + "stage2_sample_seed": 20261023, + "sampled_learners": 5000, + "train_learners": 3500, + "validation_learners": 750, + "test_learners": 750, + "gru_seed": 42, + "gru_max_epochs": 25, + "gru_patience": 4, + "pfa_solver": "lbfgs", + "pfa_C": 1.0, + "pfa_max_iter": 2000, + "pfa_tol": 1e-08, + "bootstrap_draws": 2000, + "bootstrap_seed": 20261023, + "primary_estimand": "GRU minus PFA pooled interaction-weighted Brier", + "success": "upper endpoint of 95% learner-cluster percentile interval <0; diagnostic gates pass" + }, + "test_metrics": { + "enriched_pfa": { + "n": 26697, + "positive_rate": 0.71697194441323, + "brier": 0.186954270145136, + "log_loss": 0.5566794636119368, + "average_precision": 0.8306718333720006, + "ece_10": 0.0169429951113543, + "roc_auc": 0.68128602293271 + }, + "extended_gru": { + "n": 26697, + "positive_rate": 0.71697194441323, + "brier": 0.18227301560902529, + "log_loss": 0.5434550333145024, + "average_precision": 0.8477370017121166, + "ece_10": 0.00819962269299951, + "roc_auc": 0.6996825804347548 + } + }, + "paired_brier": { + "difference_candidate_minus_reference": -0.004681254536110695, + "bootstrap_95_interval": [ + -0.0060729744844493895, + -0.0033456516490331786 + ], + "resampling_unit": "learner", + "estimand_unit": "interaction", + "n_learners": 750, + "n_interactions": 26697, + "n_bootstrap": 2000 + }, + "pfa_iterations": 484, + "pfa_convergence_warnings": [], + "gru_best_validation_epoch": 7, + "gru_epochs_completed": 11, + "limitation": "Replication of the directional performance difference on a newly sampled 2015 population; not causal architecture identification or universal transportability." +} \ No newline at end of file From f811ad802a2b81d99e43d32d0bc01f792c977dd9 Mon Sep 17 00:00:00 2001 From: Devis Wawan Saputra Date: Thu, 22 Oct 2026 16:26:20 +0700 Subject: [PATCH 13/48] research: document source-audited independent-cohort replication and interpretation limits --- paper/external_replication_assist2015.md | 35 ++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 paper/external_replication_assist2015.md diff --git a/paper/external_replication_assist2015.md b/paper/external_replication_assist2015.md new file mode 100644 index 0000000..8fe4033 --- /dev/null +++ b/paper/external_replication_assist2015.md @@ -0,0 +1,35 @@ +# External-cohort replication of the knowledge-tracing comparison (22 October 2026) + +## Rationale and source + +The original ASSISTments 2009 transformed-mirror model comparisons were exploratory: the 623-learner test partition had been inspected repeatedly, and the processed mirror had not been shown equivalent to the corrected official 2009 release. We therefore evaluated the previously reported direction of the predictive difference on **a separate-year source**, ASSISTments 2015 Skill Builder Data (official dataset portal: https://sites.google.com/site/assistmentsdata/datasets/2015-assistments-skill-builder-data). The 2015 raw file, linked by the official data portal, has SHA-256 `75e1f46131d5897a388c82411bc3032279899cb1f1e66a9043354d41234407a0`, 708,631 original event rows, and 19,840 eligible learner IDs after a fixed record-level validity screen; 24,830 invalid/missing rows were excluded. Response sequences were ordered by numeric `log_id`, with `sequence_id` representing the 2015 skill-builder identifier. + +## Prespecified evaluation and protocol deviation + +Before the first 2015 evaluation, a protocol fixed learner-based sampling, training/validation/test separation, model settings, the Brier-score estimand, and the bootstrap decision rule. The initial 5,000-learner random sample (seed 20261022) produced a nominal difference favoring GRU but failed its own numeric convergence gate: the enriched PFA SAGA optimizer reached its iteration limit. Its result is retained and **is not counted as confirmation** (GitHub Actions run [39754046492](https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39754046492)). + +A separately frozen corrective protocol replaced the numerical optimization algorithm only, to solve the same regularized logistic objective: enriched PFA with skill indicators, within-skill prior successes/failures, and across-skill prior successes/failures, using `lbfgs` with `C=1.0`, `max_iter=2000`, and `tol=1e-8`. Crucially, it **excluded all 5,000 participants from the first evaluation** before drawing a second random 5,000-learner sample with seed 20261023. The corrective decision was motivated by optimizer nonconvergence, not by searching test sets for favorable results. Within the second sample, learner-disjoint train/validation/test partitions contained 3,500/750/750 students. The GRU configuration remained unchanged: seed 42, fixed embedding and hidden sizes, Adam learning rate 0.002, up to 25 epochs, patience 4, with epoch selection restricted to validation data. The best validation epoch was 7 (11 epochs executed). The logistic solver converged in 484 iterations, with no convergence warnings. + +The primary comparison was fixed as the *interaction-weighted paired Brier loss difference (GRU minus enriched PFA)*, with 2,000 learner-cluster bootstrap resamples (seed 20261023). The stated directional success criterion was an upper 95% percentile-interval endpoint below zero, conditional on meeting data and numerical diagnostic gates. This is an intentionally narrow *predictive replication* criterion; it is not an intervention-effect or causal-model-attribution test. + +## New held-out findings + +The untouched stage-2 test set contained 750 learners and 26,697 response interactions (sampled 5,000 learners contained 172,614 interactions before splitting). Enriched PFA achieved ROC-AUC **0.681286**, Brier **0.186954**, log loss **0.556679**, and ECE-10 **0.016943**. Extended GRU achieved ROC-AUC **0.699683**, Brier **0.182273**, log loss **0.543455**, and ECE-10 **0.008200**. The primary pooled Brier difference (GRU minus PFA) was **−0.0046813**, with learner-cluster percentile 95% interval **[−0.0060730, −0.0033457]**. The interval excludes zero in the predicted direction, and all prespecified numerical/data gates passed. The observed correctness rate in the test sample was 0.716972. + +This second-stage result **replicates the direction of the predictive Brier advantage on a new ASSISTments 2015 learner subset**, without reusing any learner from the 5,000 selected in the first 2015 test. The earlier nonconverged experiment remains reported as a diagnostic pilot, not as an independent positive confirmation. + +## Limits on interpretation + +This is a new *cohort and learner population*, not an untouched reanalysis of the previous ASSISTments 2009 mirror. The skill-builder units in 2015 (`sequence_id`) are not guaranteed to be equivalent to 2009 skill codes. Models were retrained on 2015 training learners, so this is not direct frozen-weight cross-domain transport. The richer ordered history and greater capacity of the GRU remain confounded with architectural differences relative to aggregate-count PFA; **the result does not identify the causal effect of recurrence**. The study also does not prove equivalence between the transformed 2009 mirror and the corrected official 2009 event-level release, nor establish transportability to other schooling systems. The bootstrap interval is conditional on the selected sample and fixed code/analysis choices. Author and independent methodological review remain necessary. + +## Reproducibility evidence + +- Frozen first protocol: [CONFIRMATORY_PROTOCOL_2015.md](../CONFIRMATORY_PROTOCOL_2015.md). +- Corrective protocol fixed before the second test: [CONFIRMATORY_STAGE2_PROTOCOL_2015.md](../CONFIRMATORY_STAGE2_PROTOCOL_2015.md). +- First code/results: [scripts/confirmatory_2015.py](../scripts/confirmatory_2015.py), [stage-1 JSON](../results/confirmatory_assist2015.json). +- Second code/results: [scripts/confirmatory_2015_stage2.py](../scripts/confirmatory_2015_stage2.py), [stage-2 JSON](../results/confirmatory_assist2015_stage2.json). +- Official execution: [stage-2 GitHub Actions run 39754854426](https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39754854426). + +### Manuscript-ready concise paragraph + +> We conducted a distinct-year replication using the officially linked ASSISTments 2015 Skill Builder file. After a first 5,000-learner evaluation was deemed inconclusive because the enhanced PFA optimizer had not converged, we excluded every learner from that attempt and froze a second, nonoverlapping 5,000-learner sample, retaining learner-disjoint training (3,500), validation (750), and test (750) partitions. The numerical PFA solver was changed to a converged LBFGS fit of the same regularized logistic objective, while the GRU and primary Brier-score criterion remained prespecified. On 26,697 fresh test interactions, the extended GRU achieved Brier 0.18227 and AUC 0.69968 versus enriched PFA Brier 0.18695 and AUC 0.68129. The interaction-pooled paired Brier difference (GRU minus PFA) was −0.00468 (95% learner-cluster bootstrap interval [−0.00607, −0.00335]; 2,000 replicates). All convergence and provenance checks passed. This supports directional predictive replication within a distinct ASSISTments cohort but cannot isolate the contribution of recurrent architecture or demonstrate generalization beyond the tested contexts. From 67664724e21d545c7c0d878ac3a788361eaec21a Mon Sep 17 00:00:00 2001 From: Devis Wawan Saputra Date: Thu, 22 Oct 2026 17:00:22 +0700 Subject: [PATCH 14/48] paper: integrate independently verified 2015 cohort replication into final manuscript --- paper/main_integrated_20261022.tex | 188 +++++++++++++++++++++++++++++ 1 file changed, 188 insertions(+) create mode 100644 paper/main_integrated_20261022.tex diff --git a/paper/main_integrated_20261022.tex b/paper/main_integrated_20261022.tex new file mode 100644 index 0000000..2016d53 --- /dev/null +++ b/paper/main_integrated_20261022.tex @@ -0,0 +1,188 @@ +\documentclass[11pt,a4paper]{article} +\usepackage[margin=0.84in]{geometry} +\usepackage[T1]{fontenc} +\usepackage[utf8]{inputenc} +\usepackage{lmodern,microtype} +\usepackage{amsmath,amssymb,booktabs,array,graphicx,caption,subcaption} +\usepackage[colorlinks=true,urlcolor=blue,citecolor=blue,linkcolor=blue]{hyperref} +\usepackage{natbib} +\sloppy\emergencystretch=4em +\setlength{\parskip}{0.14em}\renewcommand{\baselinestretch}{0.96} +\setlength{\parindent}{0pt} +\title{Knowledge-Tracing Reproducibility and Information-Budget Auditing\\\large A Pinned 2009 Sequence Mirror and Fresh 2015 Learner-Cohort Replication} +\author{Devis Wawan Saputra\\\small Independent Researcher, Indonesia\\\small ORCID: \href{https://orcid.org/0000-0002-7133-9410}{0000-0002-7133-9410}} +\date{22 October 2026\\\small Empirical research manuscript; not peer reviewed} +\begin{document} +\maketitle +\begin{abstract} +Model comparisons in knowledge tracing depend on the learner split, source transformations, predictive information available to each method, and the population used for evaluation. We first reproduce a learner-disjoint, six-model study on a SHA-256-pinned transformed ASSISTments 2009 sequence mirror (4,148 learners; 274,331 responses). On 623 previously inspected test learners (41,228 responses), a seed-42 GRU achieves ROC-AUC 0.7470 and Brier 0.1800 against standard PFA AUC 0.6982 and Brier 0.1976. A same-test, \emph{exploratory} sensitivity comparison with enriched cross-skill PFA and longer GRU optimization yields Brier 0.1929 versus 0.1791, respectively; the paired response-pooled GRU-minus-PFA Brier difference is $-0.01387$ (95\% learner-cluster percentile interval $[-0.01723,-0.01105]$). To assess whether the predictive direction persists beyond the inspected 2009 mirror, we executed an independent-year ASSISTments 2015 replication. The initial 5,000-learner pilot was flagged as inconclusive because its PFA solver failed to converge. A separately fixed corrective protocol excluded \emph{all} pilot learners and evaluated new learners with a convergent optimizer for the same enriched-PFA logistic objective. On 750 learner-disjoint holdouts (26,697 responses), the GRU yields Brier 0.18227 and AUC 0.69968 against enriched PFA Brier 0.18695 and AUC 0.68129; the paired Brier contrast is $-0.00468$ (95\% learner-cluster interval $[-0.00607,-0.00335]$, 2,000 draws). This supports a bounded, different-year \emph{directional predictive replication}; it does not prove the effect of recurrent architecture, parity of the 2009 mirror with the officially corrected data, cross-institutional transportability, or instructional benefit. +\end{abstract} +\textbf{Keywords:} AI in education; knowledge tracing; ASSISTments; leakage-aware evaluation; BKT; PFA; recurrent networks; calibration; independent-cohort replication. + +\section{Introduction} +Knowledge tracing predicts a learner's subsequent response from observed practice history. The probabilistic outputs can be used to rank cases for inspection and to support adaptive content, but predictive validity must not be conflated with instructional value. A model that has learned response regularities on a historical tutoring platform may still fail under new curricula, item policies, or student populations. + +A second challenge concerns experimental comparison. A random row split can let responses from the same learner appear in both model development and evaluation. Such a split evaluates interpolation within known learner histories rather than the intended generalization to unseen learners. Another challenge is information symmetry: a recurrent model may use prior outcomes from other skills while a classical per-skill model sees only within-skill history. Predictive differences then reflect both representation and architecture. + +Rather than introduce a new model, this study audits the empirical scope of a controlled learner-disjoint benchmark, explicitly recording probabilistic performance, slice results, calibration, and uncertainty limits. It answers: (RQ1) How do conventional baselines compare on a frozen learner-disjoint partition? (RQ2) Does prediction differ between a learner's first observed and later encounters with a skill? (RQ3) What do the resampling intervals establish under a fixed learner split? (RQ4) Do extended training and a richer transparent baseline materially change the results, and what remains untested? (RQ5) Does the direction of the predictive difference replicate on an independent-year cohort? + +\section{Related Work and Positioning} +BKT recursively updates a latent mastery probability using learning-transition, guess, and slip assumptions \citep{corbett1995}. PFA predicts responses with logistic effects of skill identity and previous successes and failures \citep{pavlik2009}. Deep Knowledge Tracing introduced recurrent sequence representations \citep{piech2015}. Later evaluations demonstrated that claims of deep-model superiority are highly contingent on datasets, baselines, and evaluation design \citep{gervet2020}. Broader surveys emphasize inconsistent versions, preprocessing, and metric protocols across commonly named knowledge-tracing datasets \citep{abdelrahman2023}. More recent work directly addresses correction-aware and cross-cohort benchmarking. A 2026 replication package by Polydoros compares classical and deep approaches on the corrected ASSISTments 2009 and 2015 cohorts with learner-level resampling \citep{polydoros2026}. Padi and colleagues (2026) show empirically that duplicate multi-skill records and asymmetries in the information available to different model families can change reported rankings \citep{padi2026}. These advances preclude claiming novelty for learner-disjoint splitting, calibration analysis, or information matching in isolation. They also provide corrected-source and cross-cohort comparators, so we make no state-of-the-art or architecture-advantage claims. The distinctive contribution here is the auditable sequence of a frozen transformed-mirror reproduction, contrasting uncertainty targets, an explicitly labeled exploratory information-budget audit, and a separate-year predictive replication that retains rather than conceals a nonconverged first pilot. Unlike a model-development paper, this investigation treats exact source provenance, estimand distinctions, and explicit failure of source parity as empirical audit findings rather than claiming a state-of-the-art architecture. + +\section{Study Contribution and Evidence Hierarchy} +We provide four bounded outputs: (1) an executable source- and learner-split-pinned reproduction of six models and model-specific probability metrics on the transformed 2009 mirror; (2) a distinction between equal-learner and interaction-pooled Brier uncertainty targets; (3) an exploratory same-split sensitivity analysis comparing cross-skill PFA summary features with extended GRU training; and (4) a documented independent-year 2015 predictive replication on a distinct set of learners. The two evidence tiers must not be conflated: post-inspection 2009 improvements are exploratory, while the second 2015 cohort is fresh with respect to learner identity and outcome inspection. The second-stage 2015 protocol was nonetheless fixed \emph{after} observing a failed optimizer convergence diagnostic in the first 2015 pilot; its corrective choice and chronology are disclosed, and the result is not described as prospectively registered or independently peer-reviewed. These analyses compare unequal information representations and model capacity; they cannot identify a recurrence-only effect. + +\section{Data and Evaluation Protocol} +\subsection{Dataset origin and unresolved transformation question} +The intended underlying source is the ASSISTments 2009--2010 Skill Builder data documented by its maintainers \citep{assistmentsdata}; the ASSISTments tutoring platform and its formative-assessment context were described by \citet{feng2009}. The execution pipeline reads \texttt{Atomi/ASSISTments2009} at pinned revision \texttt{c72a664a9693547fb206652}\allowbreak\texttt{ed2ce18e62d320c7d} and file \path{data/train-00000-of-00001.parquet}. The archived file SHA-256 is \texttt{2053b7ea63f1d5ea953834bf82ed8954}\\\texttt{3e1f11bae1e3e7535c2f9d7f49d4dda7}. The mirror contains one serialized sequence per learner. The empirical record reports 4,148 unique learners and 274,331 binary response observations. + +\textbf{Provenance qualification.} The original ASSISTments maintainers published corrected skill-builder records after identifying duplicated rows \citep{assistmentsdata}. The pinned third-party mirror does not supply all original row-level identifiers needed to prove its equivalence to that corrected official release. Consequently, all numerical results below apply to \emph{this pinned mirror representation}. This is a meaningful, narrower replication claim, not a corrected-release benchmark. The public sequence mirror contains binary response arrays and skill identifiers, but the current audit cannot map these serialized events back to unique corrected-release student--problem identifiers or recover an independently timestamped ordering of the underlying raw events. If one underlying student--problem event was inadvertently repeated within a sequence, a learner-disjoint split alone could not prevent within-sequence target-label contamination. The chronological and unique-event interpretation of the mirror must therefore be treated as an \emph{unverified input assumption}, not as a validated property of the original ASSISTments log. + +\subsection{Officially linked 2015 independent-year source} +For the different-year replication we use the original ASSISTments 2015 Skill Builder CSV linked by the project's official dataset portal \citep{assistments2015}. The original file SHA-256 is \texttt{75e1f46131d5897a388c82411bc303}\texttt{2279899cb1f1e66a9043354d41234407a0}. It contains 708,631 records; 24,830 were excluded by the frozen missingness/validity filter, leaving 19,840 eligible learner IDs. The study constructs within-learner sequences in ascending numerical \texttt{log\_id} order, using \texttt{sequence\_id} as a 2015 skill-builder identifier; these identifiers are not the same construct as the 2009 mirror's \texttt{skill\_ids}. This different-year study requires new fitting on 2015 training learners and does not test a frozen 2009-trained model on 2015. The analysis source, hashes, sampling seeds, and both pilot and corrective-stage outcomes are archived in the repository's research branch. The 2015 result does not retroactively validate the 2009 mirror transformation. + +\subsection{Partition unit, leakage checks, and model selection} +Learners are shuffled at seed 42 and divided into training (2,903), validation (622), and test (623) partitions. The runner first rejects duplicated \texttt{user\_id} sequence rows, then forbids learner overlap by partitioning whole serialized rows. It reports 194,951 training responses and 41,228 test responses. This check establishes no repeated \emph{learner identifier} across partitions, but does not verify the identity or chronological ordering of individual raw student--problem events. The validation set determines the BKT parameter choice and neural stopping epoch. Global and skill priors, PFA coefficients, and recurrent embeddings are learned from the training partition only. Every stateful model emits its prediction \emph{before} observing the target response. + +\subsection{Model specifications} +\textbf{Global prior.} The binary response probability is the aggregate training correctness fraction. \textbf{Skill prior.} The response probability is the training correctness frequency for the queried skill, with a global fallback. \textbf{Fixed BKT.} Initial mastery, learning, guess, and slip probabilities are respectively 0.20, 0.15, 0.20, and 0.10. The prediction for a current response uses the previous posterior mastery state: +\begin{equation} +P(Y_t=1\mid H_{t-1})=m_{t-1}(1-s)+(1-m_{t-1})g. +\end{equation} +\textbf{Validation-tuned BKT.} The four parameters are selected from a fixed grid using validation log loss; the archived selection is initial mastery 0.40, learning 0.15, guess 0.30, and slip 0.20. \textbf{PFA.} A regularized logistic model uses a skill intercept and preceding learner--skill success/failure counts. \textbf{GRU.} A 64-dimensional recurrent hidden state processes previous response-conditioned skill tokens and is queried with the next skill embedding; seed-42 predictions for each learner's first response fall back to a training skill/global prior. Seeds 13, 42, and 73 were run for at most eight training epochs with validation-loss monitoring. + +\textbf{Comparability caveat.} PFA counts only prior outcomes on the queried skill; the GRU can exploit the complete previous cross-skill sequence. The observed gap does not isolate the effect of nonlinearity or recurrence. The executed extension adds a PFA comparator with prior cross-skill success/failure counts, and longer GRU training. Because this comparator is still a compact summary rather than a full ordered-history model, the test does not isolate the effect of model architecture. + +\subsection{Evaluation, uncertainty, and estimands} +ROC-AUC and average precision quantify discrimination and ranking. Brier score and log loss quantify probabilistic accuracy. Expected calibration error (ECE-10) compares confidence and observed response frequencies in ten probability bins, but is sample/bin-sensitive. A learner-first skill occurrence means that skill key has not appeared previously \emph{for that learner}; it is not necessarily unseen in training. + +There are two distinct aggregation targets in the existing report. The primary table pools response-level losses, giving interaction-weighted estimates: +\begin{equation} +\widehat{B}_{\mathrm{event}}(f)=\frac{1}{N}\sum_{i=1}^{n}\sum_{t=1}^{T_i}(y_{it}-\hat p_{it}^{(f)})^2,\quad N=\sum_i T_i. +\end{equation} +The archived cluster bootstrap instead averages per-learner loss differences: +\begin{equation} +\widehat{\Delta}_{\mathrm{learner}}=\frac{1}{n}\sum_i\frac{1}{T_i}\sum_t\left[(y_{it}-\hat p_{it}^{(f)})^2-(y_{it}-\hat p_{it}^{(g)})^2\right]. +\end{equation} +That is a defensible \emph{different} estimand. It must not be interpreted as an uncertainty interval for the event-pooled Brier difference. A paired learner-cluster bootstrap for the event-weighted difference resamples learners and recomputes the ratio of summed losses to summed response counts in each replicate. The executed supplement implements this correction, with 2,000 learner-resampling draws per comparison. Each draw recomputes the ratio of the sampled cluster loss sums to the sampled cluster response counts; the resulting percentile intervals are conditional on this one previously examined test partition. + +\section{Reproduction and Exploratory Robustness Results} +\subsection{Overall held-out comparisons} +Table~\ref{tab:results} reports the results from a new full execution of the original study on the pinned real-data mirror. Reproduced original values agree with the archived record to within $4\times10^{-8}$ for ROC-AUC and much more closely for Brier score. +\begin{table}[ht] +\centering\small +\caption{Reproduced results on 41,228 responses from 623 learner-disjoint test learners (seed-42 original GRU). Lower Brier, log loss, and ECE are better.} +\label{tab:results} +\begin{tabular}{lrrrrr}\toprule +Model & ROC-AUC & AP & Brier & Log loss & ECE-10\\\midrule +Global prior & 0.5000 & 0.6747 & 0.2197 & 0.6313 & 0.0160\\ +Skill prior & 0.6234 & 0.7669 & 0.2097 & 0.6096 & 0.0145\\ +Fixed BKT & 0.6746 & 0.7806 & 0.2187 & 0.6360 & 0.1226\\ +Validation-tuned BKT & 0.6722 & 0.7770 & 0.2024 & 0.5928 & 0.0279\\ +PFA logistic & 0.6982 & 0.8074 & 0.1976 & 0.5840 & 0.0413\\ +GRU (seed 42) & \textbf{0.7470} & \textbf{0.8439} & \textbf{0.1800} & \textbf{0.5382} & \textbf{0.0133}\\\bottomrule +\end{tabular} +\end{table} +The recorded GRU-minus-PFA Brier difference is approximately $-0.01765$ (an 8.9\% relative reduction with PFA as denominator), and ROC-AUC difference is $+0.04878$. These are \emph{descriptive differences} on one test partition. A separate \emph{exploratory} paired uncertainty calculation for the later strengthened comparison is reported below. In particular, the fixed-parameter BKT achieves higher AUC than the training-skill prior while having worse Brier score, illustrating why ranking and probability quality must be considered separately. + +\begin{figure}[ht]\centering\includegraphics[width=.83\linewidth]{kt_brier.png}\caption{Reproduced response-pooled Brier scores for the original six benchmark models on the pinned real-data mirror.}\end{figure} + +\subsection{First skill occurrence versus repeated occurrences} +\begin{table}[ht]\centering\small +\caption{Recorded PFA and seed-42 GRU performance by within-learner skill occurrence.}\begin{tabular}{lrrrrr}\toprule +Slice & N & PFA AUC & GRU AUC & PFA Brier & GRU Brier\\\midrule +First observed skill & 6,193 & 0.6499 & 0.7224 & 0.2244 & 0.2094\\ +Repeated skill & 35,035 & 0.7068 & 0.7548 & 0.1929 & 0.1747\\\bottomrule +\end{tabular}\end{table} +The GRU has lower recorded Brier scores in both slices. However, this does not imply validated mastery estimation or a benefit for instructional decisions. Difficulty mix and number of available previous responses vary by slice. + +\subsection{Random-seed variability and archived learner uncertainty} +Across the three prespecified GRU seeds the mean ROC-AUC is 0.74708 (sample SD 0.00083) and mean Brier score is 0.17975 (sample SD 0.00019). These quantify three optimization restarts on the same data split, not independent replications or evidence of robustness across populations. + +The original run reproduces a \emph{mean per-learner} Brier difference between the seed-42 GRU and skill prior of $-0.02888$, with 95\% learner-block percentile interval $[-0.03439,-0.02365]$ for 623 learners. This interval must be linked to the learner-averaged quantity given above and cannot be relabelled as the pooled response-level comparison. The new cluster bootstrap for the interaction-pooled GRU--PFA comparison is presented separately below; its estimand must not be conflated with the original equal-learner-weighted interval. + +\subsection{Independent execution reproducibility check} +The original end-to-end workflow was rerun in GitHub Actions (run \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39655888620}{39655888620}) on 22 October 2026. The downloaded real-data file matched the archived SHA-256 \texttt{2053b7ea63f1d5ea953834bf82ed89543e1f11bae1e3e7535c2f9d7f49d4dda7}. Learner and response counts, learner partitions, original baseline Brier scores, and full-run AUCs matched archived results at reporting precision; the maximum original AUC drift was below $4\times10^{-8}$. Both code tests and the full data-validation job finished successfully. This is a computational reproduction of \emph{the same pinned third-party mirror}, not a parity audit against the official corrected source. + +\subsection{Exploratory model and training-budget sensitivity} +To reduce the original information asymmetry, an enriched PFA baseline uses skill identity and both skill-specific and overall prior success/failure counts, with no access to the current response. The recurrent model still has richer ordered history and greater representational capacity; the comparison improves historical-information matching but does not equalize feature classes or prove an architectural mechanism. Extended GRU optimization used seeds 13, 42, and 73, at most 25 epochs, validation-loss early stopping and patience four. All three fits selected their best validation state before the cap. Table~\ref{tab:robust} summarizes the recorded out-of-sample scores \emph{on the same already-inspected test partition}. + +\begin{table}[ht]\centering\small +\caption{Exploratory robustness results from real-data reruns. All rows use the same 623 test learners (41,228 responses); GRU settings are exploratory revisions after earlier test inspection.} +\label{tab:robust} +\begin{tabular}{lrrrr}\toprule +Model & ROC-AUC & Brier & Log loss & Best epoch\\\midrule +Standard PFA & 0.6982 & 0.1976 & 0.5840 & --\\ +PFA + cross-skill counts & 0.7135 & 0.1929 & 0.5709 & --\\ +GRU (seed 13, extended) & 0.7471 & 0.1793 & 0.5367 & 13\\ +GRU (seed 42, extended) & 0.7491 & 0.1791 & 0.5357 & 13\\ +GRU (seed 73, extended) & 0.7498 & 0.1789 & 0.5358 & 11\\\bottomrule +\end{tabular} +\end{table} + +Relative to the stronger PFA, the extended seed-42 GRU has a response-pooled Brier improvement of $-0.013874$ (paired learner-cluster 95\% percentile interval $[-0.017231,-0.011049]$). Results for seeds 13 and 73 are $-0.013583$ [$-0.017004,-0.010911$] and $-0.014020$ [$-0.017418,-0.011123$], respectively. These bootstrap intervals (2,000 resamples of learners) describe conditional uncertainty \emph{on the fixed test partition after exploratory model changes}; they should not be interpreted as confirmatory evidence from an untouched test sample. The stronger PFA itself improves Brier score relative to the original PFA by $-0.004680$ [$-0.005900,-0.003537$], underscoring the effect of history-feature availability. + +\begin{figure}[ht]\centering\includegraphics[width=.90\linewidth]{robust_comparison.png}\caption{Exploratory robustness: real-data ROC-AUC and Brier score for PFA variants and three extended GRU seeds. This is a same-split sensitivity comparison, not an independent validation sample.}\end{figure} + +\begin{figure}[ht]\centering\includegraphics[width=.83\linewidth]{kt_auc.png}\caption{Reproduced test ROC-AUC across original models, shown descriptively for the fixed split.}\end{figure} + +\section{Independent-Year ASSISTments 2015 Replication} +\subsection{Frozen sampling sequence and diagnostic correction} +The first 2015 analysis sampled 5,000 learners with seed 20261022, then assigned learners to 70\%/15\%/15\% train, validation, and test groups, comparing the enriched cross-skill PFA with the extended GRU. Although this initial pilot yielded a nominal GRU Brier advantage, the PFA \texttt{saga} optimizer reached its iteration limit and the declared convergence gate failed. The pilot is archived and \emph{not counted as clean confirmation}. Its test results must not be selectively omitted simply because they are provisional. + +Before any second-stage test outcomes were inspected, a corrective protocol changed only the PFA numerical solver to \texttt{lbfgs} (\texttt{C=1}, maximum 2,000 iterations, tolerance $10^{-8}$) for the same regularized logistic model with skill intercepts, within-skill successes/failures, and pooled prior cross-skill successes/failures. It reconstructed and excluded every one of the 5,000 pilot learners, sampled 5,000 different learners (seed 20261023), and created learner-disjoint groups of 3,500 train, 750 validation, and 750 test learners. The GRU configuration remained fixed (seed 42; token embedding 32, skill query embedding 16, hidden size 64, Adam learning rate 0.002, up to 25 epochs, validation patience four). The GRU's best validation epoch was 7 of 11 executed. The enriched PFA optimizer converged in 484 iterations without convergence warnings. No observations from the first sample appear in the second sample. + +\subsection{Untouched second-sample results and estimand} +The primary estimand is the interaction-pooled paired Brier loss difference, GRU minus enriched PFA, on the \emph{second} sample's final holdout. Two thousand learner-cluster bootstrap replicates (seed 20261023) resample whole learners and preserve the pooled-interaction weighting. A directional replication was counted only if the upper 95\% percentile interval endpoint was below zero and all provenance, disjointness, and convergence checks passed. This is a one-sided directional assessment using a displayed two-sided interval, not a preregistered general benchmark claim. + +\begin{table}[ht]\centering\small +\caption{ASSISTments 2015 independent learner-cohort test (750 learners, 26,697 test responses). Higher AUC and lower Brier, log loss and ECE-10 are favorable. The second sample excludes all 5,000 learners from the first diagnostic pilot.} +\label{tab:cohort2015} +\begin{tabular}{lrrrr}\toprule +Model & ROC-AUC & Brier & Log loss & ECE-10\\\midrule +Enriched PFA (converged) & 0.68129 & 0.18695 & 0.55668 & 0.01694\\ +Extended GRU (seed 42) & 0.69968 & 0.18227 & 0.54346 & 0.00820\\\bottomrule +\end{tabular} +\end{table} + +The test correctness rate was 0.71697. The primary Brier difference was $-0.0046813$, with a 95\% learner-cluster percentile interval $[-0.0060730,-0.0033457]$. The prespecified \emph{second-stage} computational and directional gates passed, supported by GitHub Actions run \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39754854426}{39754854426}. The first, nonconverged stage and its outcomes remain available in run \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39754046492}{39754046492}. Because both 2015 experiments used the same source archive but \emph{nonoverlapping} learner samples, this is a fresh-sample, distinct-year predictive replication, not a demonstration of cross-institutional transport or strict information matching. + +\section{Discussion and Threats to Validity} +The 2009 mirror reproductions show an association between the richer sequence representation of the GRU and better test probability scores under the fixed split. The new 2015 experiment provides a separate-year check: on a distinct learner sample, a converged richer PFA remains behind the GRU in the predetermined directional Brier comparison. This is stronger evidence for repeatability of the \emph{observed predictive direction} than the 2009 reanalysis alone, but cannot isolate recurrent architecture from feature history, ordered signals, capacity, data processing, or optimization. + +\textbf{Source equivalence and chronological assumptions.} The official maintainers identify duplicated rows in the 2009--2010 skill-builder source and provide a corrected student--problem representation \citep{assistmentsdata}. The transformed Atomi mirror exposes user IDs, skill/grade lists, attempt counts and answer types, but not the row-level problem IDs, event IDs or validated timestamps required to prove corrected-source parity or exact per-event chronology. We checked the published source descriptions and schema and found no demonstrated row crosswalk. The 2009 findings \emph{remain mirror-specific}; the new 2015 experiment does not repair them. The 2015 file has event-log identifiers and a different skill-builder code definition. Ordering by \texttt{log\_id} is an explicitly documented modeling convention, not proof that the measure captures every physical action time correctly. + +\textbf{Pilot and numerical-selection transparency.} The first 2015 run did not pass the PFA convergence gate. The second-stage design and LBFGS replacement were frozen only after that failure had been observed; disjoint learners prevent direct sample reuse but do not turn the stage-2 plan into prospective preregistration. Reporting both stages prevents a misleading impression of a single perfect untouched evaluation. The 2015 results reflect one second-stage sample and seed, not many external sites or extensive split-seed sensitivity analysis. + +\textbf{Information and attribution.} The enriched PFA uses compressed aggregate per-skill and across-skill histories; the GRU uses ordered prior interactions and a different parameter budget. Its superiority cannot be assigned specifically to recurrence. Existing correction-aware, cross-cohort, and information-controlled comparisons \citep{polydoros2026,padi2026} further constrain claims of novelty or state-of-the-art ranking. + +\textbf{Uncertainty and target populations.} The 2009 original bootstrap treats the learner-average Brier contrast; the extended 2009 and new 2015 paired bootstraps target the interaction-pooled contrast and resample learners. The interval for the second 2015 cohort is conditional on its sampled learners, fixed model choices, and event selection rules, and cannot substitute for multi-institutional validation. The new cohort concerns only the source's 100 most-used skill builders, not all learners or curricula. \textbf{Educational interpretation.} Neither dataset supplies a randomized teaching intervention; predictive skill probabilities are not a direct measure of learning benefit, fairness, or readiness for consequential student decisions. + +\section{Research Questions Revisited} +\textbf{RQ1:} The six-model baseline reproduces on the fixed mirror, conditional on its source transformation. \textbf{RQ2:} First within-learner skill occurrences and repetitions differ descriptively, but are not directly comparable learning stages. \textbf{RQ3:} Learner-averaged and interaction-pooled Brier targets differ; their bootstrap intervals must be labeled accordingly. \textbf{RQ4:} The exploratory 2009 GRU/PFA contrast persisted with extended training and richer classical history, but on inspected learners. \textbf{RQ5 (external replication):} The directional interaction-pooled Brier advantage replicated on 750 untouched held-out 2015 learners in a separately fixed, converged corrective-stage evaluation, with the limitations above. + +\section{Conclusion} +Re-execution confirms the archived six-model metrics on a pinned transformed ASSISTments 2009 mirror, while exploratory richer-baseline and longer-training comparisons remain conditional on the previously inspected split. A distinct 2015 Skill Builder learner cohort, sampled after excluding all learners in a failed-convergence pilot, achieved a converged enriched-PFA comparison and confirmed the directional GRU advantage in interaction-pooled Brier loss ($-0.00468$, 95\% cluster interval $[-0.00607,-0.00335]$). The combined evidence supports a \emph{limited predictive replication} claim, not proof that recurrence itself is superior under information-matched conditions. Official corrected-2009 row-parity, broad generalization, and instructional utility remain unverified; the paper should be read as an auditable empirical working preprint, not a new knowledge-tracing algorithm or universal model ranking. + +\section*{Data availability and research integrity} +The official 2009 correction notice is at \url{https://sites.google.com/site/assistmentsdata/home/2009-2010-assistment-data/skill-builder-data-2009-2010}, and the 2015 source at \url{https://sites.google.com/site/assistmentsdata/datasets/2015-assistments-skill-builder-data}. The 2009 pinned Atomi mirror and source SHA-256 are recorded in \url{https://github.com/devissaputra/knowledge_tracing_benchmark}. The re-execution of the 2009 frozen experiment appears in GitHub Actions run \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39655888620}{39655888620}. The first 2015 diagnostic pilot and final disjoint-cohort execution are in runs \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39754046492}{39754046492} and \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39754854426}{39754854426}, respectively. The frozen 2015 protocols, exact SHA-256, implementation and JSON results are in \url{https://github.com/devissaputra/knowledge_tracing_benchmark/pull/2}; the original exploratory extension is in \url{https://github.com/devissaputra/knowledge_tracing_benchmark/pull/1}. Raw individual learner records are not redistributed. Archived numerical evidence and code have passed automated computational checks; external peer review, full 2009 corrected-source parity, and author approval are not claimed. + +\section*{Limitations of authorship and interpretation} +This is a research manuscript prepared for author verification. The named author must personally verify the calculations, code, references, and contribution statements before submission. The text makes no claim of peer review or acceptance. Draft preparation and some analytical implementation involved generative AI assistance; any submitted version requires the named author's personal audit and approval of the methods, empirical provenance, interpretations, and disclosure in accordance with the destination archive's applicable policies. + +\begingroup\small +\setlength{\itemsep}{0pt} +\begin{thebibliography}{99} +\bibitem[Abdelrahman et al.(2023)]{abdelrahman2023} Abdelrahman, G., Wang, Q., and Nunes, B. (2023). Knowledge tracing: A survey. \emph{ACM Computing Surveys}, 55(11), Article 224. \url{https://doi.org/10.1145/3569576}. +\bibitem[Corbett and Anderson(1995)]{corbett1995} Corbett, A. T. and Anderson, J. R. (1995). Knowledge tracing: Modeling the acquisition of procedural knowledge. \emph{User Modeling and User-Adapted Interaction}, 4, 253--278. \url{https://doi.org/10.1007/BF01099821}. +\bibitem[Feng et al.(2009)]{feng2009} Feng, M., Heffernan, N. T., and Koedinger, K. R. (2009). Addressing the assessment challenge with an online system that tutors as it assesses. \emph{User Modeling and User-Adapted Interaction}, 19(3), 243--266. \url{https://doi.org/10.1007/s11257-009-9063-7}. +\bibitem[Gervet et al.(2020)]{gervet2020} Gervet, T., Koedinger, K., Schneider, J., and Mitchell, T. (2020). When is deep learning the best approach to knowledge tracing? \emph{Journal of Educational Data Mining}, 12(3), 31--54. \url{https://doi.org/10.5281/zenodo.4143614}. +\bibitem[Polydoros(2026)]{polydoros2026} Polydoros, G. (2026). Cross-Cohort Knowledge-Tracing Benchmark: ASSISTments 2009--2010 and 2015 Replication Package (v2.0.1, corrected statistical reporting). Zenodo software. \url{https://doi.org/10.5281/zenodo.22131530}. +\bibitem[Padi et al.(2026)]{padi2026} Padi, P., Morampudi, A., Irrinki, U. S. G., and Kakitapelli, P. K. D. (2026). Stable and faithful explanations for knowledge tracing. arXiv:2609.28502. \url{https://arxiv.org/abs/2609.28502}. +\bibitem[Pavlik et al.(2009)]{pavlik2009} Pavlik, P. I., Cen, H., and Koedinger, K. R. (2009). Performance factors analysis: A new alternative to knowledge tracing. \emph{Proceedings of AIED}, 531--538. \url{https://doi.org/10.3233/978-1-60750-028-5-531}. +\bibitem[Piech et al.(2015)]{piech2015} Piech, C., Bassen, J., Huang, J., Ganguli, S., Sahami, M., Guibas, L., and Sohl-Dickstein, J. (2015). Deep knowledge tracing. \emph{NeurIPS 28}. \url{https://arxiv.org/abs/1506.05908}. +\bibitem[ASSISTmentsData(2015)]{assistments2015} ASSISTmentsData (2015). 2015 ASSISTments Skill Builder Data, original dataset portal (100 skill builders). \url{https://sites.google.com/site/assistmentsdata/datasets/2015-assistments-skill-builder-data}. +\bibitem[ASSISTmentsData(n.d.)]{assistmentsdata} ASSISTmentsData (n.d.). Skill-builder data 2009--2010, correction notice and official data. \url{https://sites.google.com/site/assistmentsdata/home/2009-2010-assistment-data/skill-builder-data-2009-2010}. +\end{thebibliography} +\endgroup +\end{document} \ No newline at end of file From f9832652c845d98d1c356a39b8d1f8252d911450 Mon Sep 17 00:00:00 2001 From: Devis Wawan Saputra Date: Thu, 22 Oct 2026 17:00:32 +0700 Subject: [PATCH 15/48] paper: add documented PROVENANCE_NOVELTY_AUDIT_20261022.md --- paper/PROVENANCE_NOVELTY_AUDIT_20261022.md | 34 ++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 paper/PROVENANCE_NOVELTY_AUDIT_20261022.md diff --git a/paper/PROVENANCE_NOVELTY_AUDIT_20261022.md b/paper/PROVENANCE_NOVELTY_AUDIT_20261022.md new file mode 100644 index 0000000..34e985f --- /dev/null +++ b/paper/PROVENANCE_NOVELTY_AUDIT_20261022.md @@ -0,0 +1,34 @@ +# Paper 1: source-parity and contribution audit (22 October 2026) + +## Evidentiary boundary + +The existing 2009 experiment uses SHA-256-pinned transformed `Atomi/ASSISTments2009` parquet, revision `c72a664a9693547fb206652ed2ce18e62d320c7d`, digest `2053b7ea63f1d5ea953834bf82ed89543e1f11bae1e3e7535c2f9d7f49d4dda7`. The mirrored record exposes `user_id`, `skill_ids`, `skill_names`, `grades`, `attempt_counts`, and `answer_types` as *sequences* (Hugging Face dataset card, checked 22 October 2026). It has 4,148 serialized learner rows. The data adapter verifies unique learner IDs and aligned binary responses but does not reconstruct raw record identities or validated event chronology. + +The official corrected 2009-2010 Skill Builder release documents that duplicate records existed in the earlier dataset and publishes a corrected representation with **one row per student-problem**, collapsing multiple skills into one row. The official source supplies row-level identifiers in its original release that are not present in the Atomi sequence mirror. No pre-transformation crosswalk or data lineage statement establishing that the Atomi parquet came from that corrected exact release was located in its exposed card. Exact source equivalence is therefore **NOT VERIFIED**. Similar variable names or comparable learner counts would not suffice as proof. The paper must continue using *pinned transformed mirror*, not *verified corrected 2009 dataset*, in empirical claims. + +Official 2009 corrected source and correction notice: https://sites.google.com/site/assistmentsdata/home/2009-2010-assistment-data/skill-builder-data-2009-2010 + +Public transformed mirror metadata and version: https://huggingface.co/datasets/Atomi/ASSISTments2009 and https://huggingface.co/datasets/Atomi/ASSISTments2009/tree/c72a664a9693547fb206652ed2ce18e62d320c7d + +The official 2015 Skill Builder source has a different schema and includes `user_id`, `log_id`, `sequence_id`, `correct`. Its 2015 event-log ID is used as a within-learner ordering proxy in the new analysis. Replicating the *direction* of a model gap on disjoint 2015 learners is a separate finding, not validation of historical 2009 source parity. + +Official 2015 source: https://sites.google.com/site/assistmentsdata/datasets/2015-assistments-skill-builder-data + +## Literature and novelty positioning + +- Abdelrahman, Wang and Nunes, *Knowledge Tracing: A Survey* (ACM CSUR; https://doi.org/10.1145/3569576) already synthesizes model families and evaluation practices. +- Gervet et al., *When is Deep Learning the Best Approach to Knowledge Tracing?* (2020; https://zenodo.org/records/4143614) directly cautions against generic recurrent-vs-classical superiority claims. +- Polydoros (2026; https://doi.org/10.5281/zenodo.22131530) offers a corrected-2009 / official-2015 multi-model benchmark; therefore this manuscript cannot claim to introduce cross-cohort knowledge tracing, learner splits, or calibration evaluation. +- Padi et al. (2026; https://arxiv.org/abs/2609.28502) addresses duplicated histories and information symmetry, so a claim that this paper first discovers either issue is unsupportable. + +**Defensible incremental contribution:** a fully pinned reproduction of the *specific* 2009 transformed mirror with two explicitly distinguished Brier-weighting estimands, a transparent post-inspection enhanced-baseline/GRU optimization audit, and an independently sampled 2015 directional replication that retains its initial PFA-convergence failure and disjoint corrective cohort. This is a contribution in research auditability and evidence hierarchy, not new model theory or a state-of-the-art predictor. Its publication significance ultimately requires human editorial/peer judgment. + +## Release-gate determination + +- Public documentation / source-schema audit: completed. +- Independent 2015 direction-of-predictive-difference replication: **passed narrow numerical gate** per GitHub Actions run 39754854426; first pilot run 39754046492 retained as nonconverged. +- Corrected official 2009 row equivalence / verified chronology: **not established**; impossible to infer from the mirror columns alone. +- Architecture-only effect and general applicability: **not established**. +- Author sign-off, contribution originality, arXiv metadata and submission: pending human author. + +The final manuscript is a carefully bounded working preprint candidate; do not state that corrected-source parity or peer review has been achieved. \ No newline at end of file From b1dab5e514c890be3449f9e3acc3e4917dbb0369 Mon Sep 17 00:00:00 2001 From: Devis Wawan Saputra Date: Thu, 22 Oct 2026 17:00:35 +0700 Subject: [PATCH 16/48] paper: add documented FINAL_QA_RESULTS_20261022.json --- paper/FINAL_QA_RESULTS_20261022.json | 295 +++++++++++++++++++++++++++ 1 file changed, 295 insertions(+) create mode 100644 paper/FINAL_QA_RESULTS_20261022.json diff --git a/paper/FINAL_QA_RESULTS_20261022.json b/paper/FINAL_QA_RESULTS_20261022.json new file mode 100644 index 0000000..fdc14da --- /dev/null +++ b/paper/FINAL_QA_RESULTS_20261022.json @@ -0,0 +1,295 @@ +{ + "date": "2026-10-22", + "checks_total": 56, + "passed": 56, + "failed": 0, + "checks": [ + { + "check": "Expected evidence: 4,148", + "passed": true, + "detail": "" + }, + { + "check": "Expected evidence: 274,331", + "passed": true, + "detail": "" + }, + { + "check": "Expected evidence: 623", + "passed": true, + "detail": "" + }, + { + "check": "Expected evidence: 41,228", + "passed": true, + "detail": "" + }, + { + "check": "Expected evidence: 0.7491", + "passed": true, + "detail": "" + }, + { + "check": "Expected evidence: 0.1791", + "passed": true, + "detail": "" + }, + { + "check": "Expected evidence: 0.7135", + "passed": true, + "detail": "" + }, + { + "check": "Expected evidence: 0.1929", + "passed": true, + "detail": "" + }, + { + "check": "Expected evidence: 708,631", + "passed": true, + "detail": "" + }, + { + "check": "Expected evidence: 24,830", + "passed": true, + "detail": "" + }, + { + "check": "Expected evidence: 19,840", + "passed": true, + "detail": "" + }, + { + "check": "Expected evidence: 3,500", + "passed": true, + "detail": "" + }, + { + "check": "Expected evidence: 750", + "passed": true, + "detail": "" + }, + { + "check": "Expected evidence: 26,697", + "passed": true, + "detail": "" + }, + { + "check": "Expected evidence: 0.68129", + "passed": true, + "detail": "" + }, + { + "check": "Expected evidence: 0.69968", + "passed": true, + "detail": "" + }, + { + "check": "Expected evidence: 0.18695", + "passed": true, + "detail": "" + }, + { + "check": "Expected evidence: 0.18227", + "passed": true, + "detail": "" + }, + { + "check": "Expected evidence: -0.0046813", + "passed": true, + "detail": "" + }, + { + "check": "Expected evidence: -0.0060730", + "passed": true, + "detail": "" + }, + { + "check": "Expected evidence: -0.0033457", + "passed": true, + "detail": "" + }, + { + "check": "Expected evidence: 2,000", + "passed": true, + "detail": "" + }, + { + "check": "Expected evidence: 484", + "passed": true, + "detail": "" + }, + { + "check": "Expected evidence: 20261023", + "passed": true, + "detail": "" + }, + { + "check": "Expected evidence: 75e1f46131d5897a388c82411bc303", + "passed": true, + "detail": "" + }, + { + "check": "Expected evidence: 39754854426", + "passed": true, + "detail": "" + }, + { + "check": "Expected evidence: 39754046492", + "passed": true, + "detail": "" + }, + { + "check": "Expected evidence: 2053b7ea63f1d5ea953834bf82ed8954", + "passed": true, + "detail": "" + }, + { + "check": "Scope guardrail: mirror-specific", + "passed": true, + "detail": "" + }, + { + "check": "Scope guardrail: not counted as clean confirmation", + "passed": true, + "detail": "" + }, + { + "check": "Scope guardrail: cannot isolate recurrent architecture", + "passed": true, + "detail": "" + }, + { + "check": "Scope guardrail: does not retroactively validate", + "passed": true, + "detail": "" + }, + { + "check": "Scope guardrail: does not prove", + "passed": true, + "detail": "" + }, + { + "check": "Scope guardrail: not peer reviewed", + "passed": true, + "detail": "" + }, + { + "check": "Scope guardrail: cannot identify", + "passed": true, + "detail": "" + }, + { + "check": "All citation keys resolved", + "passed": true, + "detail": "cites=14, unique=10, reference entries=10" + }, + { + "check": "2015 official source cited", + "passed": true, + "detail": "" + }, + { + "check": "No raw individual dataset embedded", + "passed": true, + "detail": "" + }, + { + "check": "Figures available", + "passed": true, + "detail": "" + }, + { + "check": "Pilot nonconvergence documented", + "passed": true, + "detail": "" + }, + { + "check": "Corrected-stage method documented", + "passed": true, + "detail": "" + }, + { + "check": "Data source contrast maintained", + "passed": true, + "detail": "" + }, + { + "check": "Log no LaTeX Error", + "passed": true, + "detail": "" + }, + { + "check": "Log no Citation `", + "passed": true, + "detail": "" + }, + { + "check": "Log no Reference `", + "passed": true, + "detail": "" + }, + { + "check": "Log no Overfull \\hbox", + "passed": true, + "detail": "" + }, + { + "check": "Log no ! Undefined control sequence", + "passed": true, + "detail": "" + }, + { + "check": "PDF is valid", + "passed": true, + "detail": "" + }, + { + "check": "PDF has multiple pages", + "passed": true, + "detail": "" + }, + { + "check": "PDF extracted text contains new empirical result", + "passed": true, + "detail": "" + }, + { + "check": "All pages have extracted text", + "passed": true, + "detail": "" + }, + { + "check": "No runtime outputs in arXiv archive", + "passed": true, + "detail": "" + }, + { + "check": "Expected arXiv source files", + "passed": true, + "detail": "" + }, + { + "check": "Clean-room arXiv source compiles twice", + "passed": true, + "detail": "" + }, + { + "check": "Clean-room PDF exists", + "passed": true, + "detail": "" + }, + { + "check": "Clean-room log no overfull boxes", + "passed": true, + "detail": "" + } + ], + "files": { + "main.pdf": "ba643d4581d860bcf0bd0cd2eca60f628ec7143bdf30803f607acfc536eeef5e", + "Paper1_arXiv_Updated_20261022.zip": "358472bb04f92f931c4f7f7cf8e989ac974d703d1f918ee70ab097c274a1f65b", + "PROVENANCE_AND_NOVELTY_AUDIT.md": "08c3adc7f889beefec8d7ce74ebc1d8d760ec1d6421935ba44731b89a00fef5b" + }, + "scientific_reproducibility": "Externally executed and previously verified via GitHub Actions; this QA does not re-execute raw-data study.", + "author_gate": "Pending" +} \ No newline at end of file From a5ad5802105c03ae3f8fa110f30874af04047fbd Mon Sep 17 00:00:00 2001 From: Devis Wawan Saputra Date: Thu, 22 Oct 2026 17:01:02 +0700 Subject: [PATCH 17/48] paper: correct outdated research overview and link integrated 2015 preprint --- paper/paper.md | 106 +++++++++---------------------------------------- 1 file changed, 18 insertions(+), 88 deletions(-) diff --git a/paper/paper.md b/paper/paper.md index 4d3dae4..7e4a07e 100644 --- a/paper/paper.md +++ b/paper/paper.md @@ -1,97 +1,27 @@ -# A Learner-Disjoint Benchmark of Priors, BKT, PFA and Recurrent Knowledge Tracing on ASSISTments 2009 +# Knowledge-Tracing Reproducibility and Information-Budget Auditing -## Abstract +**Current research status (22 October 2026):** The final author-review manuscript is [main_integrated_20261022.tex](main_integrated_20261022.tex), together with the three figures in the separately packaged arXiv source archive. This repository text is a project overview, **not** a second competing manuscript. -Knowledge-tracing comparisons are easy to overstate when learners cross data partitions or when a neural model is compared only with a weak baseline. This study uses learner-disjoint train, validation and test partitions from ASSISTments 2009 and compares global and skill priors, fixed and validation-tuned Bayesian Knowledge Tracing, a Performance Factors Analysis-style logistic model, and a compact GRU knowledge tracer. Evaluation includes probability quality, discrimination, calibration, first-seen versus repeated-skill slices, learner-block uncertainty, repeated neural seeds, and skill-level error analysis. +## Empirical findings -## Research questions +- **ASSISTments 2009 transformed mirror, learner-disjoint original test:** 4,148 learners, 274,331 interactions in the full representation, 623 test learners, 41,228 test responses. Standard PFA ROC-AUC 0.6982 and Brier 0.1976; original seed-42 GRU AUC 0.7470 and Brier 0.1800. Full original workflow reproduced on the *same pinned mirror*, GitHub Actions [39655888620](https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39655888620). +- **Exploratory 2009 fixed-test robustness:** Stronger PFA with prior across-skill counts: AUC 0.7135, Brier 0.1929. Extended seed-42 GRU: AUC 0.7491, Brier 0.1791. GRU-minus-enriched-PFA pooled Brier difference -0.013874, learner-cluster 95% interval [-0.017231, -0.011049]. All these changes were evaluated after the original test was inspected and therefore are explicitly **exploratory**. +- **Independent-year ASSISTments 2015, preliminary test:** A first 5,000-learner sampling protocol produced nominal favorable differences but the enriched PFA optimizer failed to converge. [Run 39754046492](https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39754046492) is an **inconclusive diagnostic pilot**, not valid confirmation. +- **New, nonoverlapping 2015 cohort, convergence-corrected protocol:** All 5,000 pilot learners were excluded; second sample 5,000 different learners (3,500 train, 750 validation, 750 test). On 26,697 fresh held-out responses, enriched PFA achieved AUC 0.68129, Brier 0.18695; extended GRU AUC 0.69968, Brier 0.18227. Interaction-weighted paired Brier delta -0.0046813, 95% learner-cluster bootstrap interval [-0.0060730,-0.0033457], 2,000 draws. All numerical/data gates and PFA convergence passed. Evidence: [run 39754854426](https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39754854426), [final JSON](../results/confirmatory_assist2015_stage2.json). **This supports only a narrow directional predictive replication on the tested 2015 sample.** -1. Do stateful models improve held-out next-response prediction over global and skill priors? -2. Does validation-tuned BKT improve over a fixed BKT parameterization? -3. Does a transparent PFA-style model explain enough history to compete with a recurrent model? -4. Are conclusions stable for first-seen versus repeated skill encounters? -5. How large are learner-block uncertainty intervals around Brier-score differences? -6. How sensitive is the recurrent result to the prespecified GRU seeds? +## Manuscript and audit sources -## Background +- [Final integrated LaTeX manuscript](main_integrated_20261022.tex) +- [Final numerical/source-package QA results](FINAL_QA_RESULTS_20261022.json) +- [Source-parity and novelty assessment](PROVENANCE_NOVELTY_AUDIT_20261022.md) +- [External cohort replication details](external_replication_assist2015.md) +- [First 2015 protocol](../CONFIRMATORY_PROTOCOL_2015.md) and [corrective-stage protocol](../CONFIRMATORY_STAGE2_PROTOCOL_2015.md) +- [Exploratory code PR #1](https://github.com/devissaputra/knowledge_tracing_benchmark/pull/1) and [independent-cohort code PR #2](https://github.com/devissaputra/knowledge_tracing_benchmark/pull/2) -Bayesian Knowledge Tracing models a learner's changing latent state for individual knowledge components (Corbett & Anderson, 1995). Performance Factors Analysis provides a transparent logistic alternative based on prior successes and failures (Pavlik, Cen, & Koedinger, 2009). Recurrent neural models later expanded knowledge tracing by learning sequence representations directly from interaction histories (Piech et al., 2015). +## Non-negotiable interpretation bounds -This benchmark deliberately keeps classical and recurrent comparators together. That choice follows evidence that conclusions about deep versus classical knowledge-tracing models can depend strongly on dataset characteristics, features, and evaluation design (Gervet et al., 2020). Attention-based models such as SAKT and AKT are important extensions but are outside the current frozen model set (Pandey & Karypis, 2019; Ghosh, Heffernan, & Lan, 2020). +The transformed Atomi 2009 parquet lacks the raw problem/event identifiers and independently verified chronology required to establish exact parity with the **corrected official 2009--2010 source**. Learner-disjoint splitting alone does not remove any within-sequence contamination caused upstream. The 2015 new-cohort result does **not** validate 2009 transformation equivalence. -## Data +The GRU has ordered cross-skill history and greater capacity, while PFA uses summary history counts. The observed gap **must not** be attributed solely to recurrent architecture. The 2015 improvement does not establish intervention effects, fairness, educational gains, or universal transportability. -The scientific source is ASSISTments 2009. The executable pipeline retrieves the public Atomi serialization pinned to revision `c72a664a9693547fb206652ed2ce18e62d320c7d` and records the exact parquet SHA-256. Before splitting, the runner verifies that the serialized learner rows contain unique `user_id` values. See `DATA.md`. - -## Design - -Learners are shuffled with a fixed seed and assigned 70%/15%/15% to train, validation and test partitions. No learner crosses partitions. Stateful predictions are generated before the current response is observed. - -The validation partition is used for BKT parameter selection and GRU early stopping. The final test partition is reserved for evaluation. - -## Models - -The benchmark includes a global response prior, a training-skill prior, fixed BKT, validation-tuned BKT, PFA-style logistic regression using prior learner-skill successes and failures, and a compact GRU sequence model. - -The GRU is evaluated at the prespecified seeds 13, 42, and 73 in the full run. The implementation is a compact recurrent benchmark and is not presented as an exact reproduction of every canonical Deep Knowledge Tracing implementation. - -## Evaluation - -The final test set reports ROC-AUC, average precision, Brier score, log loss and ECE-10. Metrics are sliced by first-seen and repeated skill encounters. Here, first-seen means the first occurrence of the observed skill key within that learner's sequence, not a skill globally unseen in training. - -Brier-score differences are bootstrapped by learner block, skills with sufficiently many test interactions are inspected for high prediction error, and repeated GRU runs are summarized to expose random-seed sensitivity. - -## Results - -Numerical evidence is generated by `src/run_experiment.py` into `results/metrics.json`, `results/summary.md`, `paper/results.md`, and figures. This methods manuscript intentionally contains no independently hand-entered winner claim. - -## Limitations - -Skill annotations and response sequences are imperfect proxies for learning. Results are dataset- and protocol-specific; they do not establish teaching effectiveness, causal learning gains, fairness, or suitability for consequential learner decisions. The learner bootstrap is conditional on one prespecified train/validation/test partition and does not test sensitivity to alternative split seeds. Additional datasets, split-sensitivity analysis, and modern attention-based knowledge-tracing models are appropriate extensions before making broader state-of-the-art claims. - -## References - -Corbett, A. T., & Anderson, J. R. (1995). Knowledge tracing: Modeling the acquisition of procedural knowledge. *User Modeling and User-Adapted Interaction, 4*, 253–278. https://doi.org/10.1007/BF01099821 - -Feng, M., Heffernan, N. T., & Koedinger, K. R. (2009). Addressing the assessment challenge in an intelligent tutoring system that tutors as it assesses. *User Modeling and User-Adapted Interaction, 19*, 243–266. - -Gervet, T., Koedinger, K., Schneider, J., & Mitchell, T. (2020). When is deep learning the best approach to knowledge tracing? *Journal of Educational Data Mining, 12*(3), 31–54. https://doi.org/10.5281/zenodo.4143614 - -Ghosh, A., Heffernan, N., & Lan, A. S. (2020). Context-Aware Attentive Knowledge Tracing. *Proceedings of the 26th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining*. https://arxiv.org/abs/2007.12324 - -Pandey, S., & Karypis, G. (2019). A Self-Attentive Model for Knowledge Tracing. *Proceedings of the 12th International Conference on Educational Data Mining*. https://arxiv.org/abs/1907.06837 - -Pavlik, P. I., Cen, H., & Koedinger, K. R. (2009). Performance Factors Analysis — A New Alternative to Knowledge Tracing. *Proceedings of the 14th International Conference on Artificial Intelligence in Education*, 531–538. https://doi.org/10.3233/978-1-60750-028-5-531 - -Piech, C., Spencer, J., Huang, J., Ganguli, S., Sahami, M., Guibas, L., & Sohl-Dickstein, J. (2015). Deep Knowledge Tracing. *Advances in Neural Information Processing Systems, 28*. https://arxiv.org/abs/1506.05908 - - -## Calculation definitions and evidence audit - -BKT prediction = mastery×(1-slip) + (1-mastery)×guess. - -After a response, Bayes updating and the learning transition produce the next mastery state. Brier and log loss assess probabilities; learner-block bootstrap respects within-learner dependence. Model mastery is not directly measured knowledge. - -On 623 held-out learners, the seed-42 GRU achieves ROC-AUC 0.7470 and Brier score 0.1800, compared with 0.6982 and 0.1976 for PFA. The three GRU seeds average ROC-AUC 0.7471 with sample SD 0.0008. These results support better response prediction under this historical benchmark; they do not establish instructional benefit or justify high-stakes interpretations of a learner’s knowledge. - -The [calculation guide](../CALCULATIONS.md) provides exact evidence paths and a function-level implementation map. - -![Study design](../assets/review_overview.svg) - -![Calculation and selected evidence](../assets/review_calculations.svg) - -### Selected evidence and interpretation - -| Quantity | Value | Unit / meaning | JSON path | -|---|---:|---|---| -| skill_prior | 0.20965724966908303 | Brier ↓ | `test_metrics.skill_prior.all.brier` | -| bkt_fixed | 0.21870446416122305 | Brier ↓ | `test_metrics.bkt_fixed.all.brier` | -| bkt_validation_tuned | 0.20237712440747457 | Brier ↓ | `test_metrics.bkt_validation_tuned.all.brier` | -| pfa_logistic | 0.19760582856803 | Brier ↓ | `test_metrics.pfa_logistic.all.brier` | -| gru_kt_seed_42 | 0.1799516879180387 | Brier ↓ | `test_metrics.gru_kt_seed_42.all.brier` | - -These values are read from `results/metrics.json`. They must be interpreted with the split, data status and limitations above. The complete data/model experiment was not rerun in this review. Stored empirical results were inspected, not independently reproduced from raw data. - -### Reproduction and claim boundaries - -The existing suite requires unavailable dependencies; no full-suite pass is claimed. The figure generator can be checked with `python scripts/build_review_figures.py --check`. This verifies the displayed calculation evidence, not an independent replication of the complete scientific experiment. The manuscript is a working report, not a peer-reviewed publication. +The arXiv source package compiles cleanly and automated checks pass, but **author verification, novelty judgment and final arXiv upload are pending**. This is a working preprint candidate, not a peer-reviewed study. From 455e328a987c7695d916eb167eec7feca7ec6f3b Mon Sep 17 00:00:00 2001 From: Devis Wawan Saputra Date: Thu, 22 Oct 2026 17:02:29 +0700 Subject: [PATCH 18/48] ci: quote workflow name to repair YAML parse error --- .github/workflows/arxiv-verification.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/arxiv-verification.yml b/.github/workflows/arxiv-verification.yml index 978ad43..dab9831 100644 --- a/.github/workflows/arxiv-verification.yml +++ b/.github/workflows/arxiv-verification.yml @@ -1,4 +1,4 @@ -name: arXiv verification: real-data robustness +name: "arXiv verification: real-data robustness" on: push: branches: From b761cb54ba3036c98b142734d5c6c0c4a95358b6 Mon Sep 17 00:00:00 2001 From: Devis Wawan Saputra Date: Thu, 22 Oct 2026 17:34:33 +0700 Subject: [PATCH 19/48] paper: integrate pilot diagnostics and confirmed 2015 results in manuscript itself --- paper/main_integrated_20261022.tex | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/paper/main_integrated_20261022.tex b/paper/main_integrated_20261022.tex index 2016d53..81a3e28 100644 --- a/paper/main_integrated_20261022.tex +++ b/paper/main_integrated_20261022.tex @@ -15,7 +15,7 @@ \begin{document} \maketitle \begin{abstract} -Model comparisons in knowledge tracing depend on the learner split, source transformations, predictive information available to each method, and the population used for evaluation. We first reproduce a learner-disjoint, six-model study on a SHA-256-pinned transformed ASSISTments 2009 sequence mirror (4,148 learners; 274,331 responses). On 623 previously inspected test learners (41,228 responses), a seed-42 GRU achieves ROC-AUC 0.7470 and Brier 0.1800 against standard PFA AUC 0.6982 and Brier 0.1976. A same-test, \emph{exploratory} sensitivity comparison with enriched cross-skill PFA and longer GRU optimization yields Brier 0.1929 versus 0.1791, respectively; the paired response-pooled GRU-minus-PFA Brier difference is $-0.01387$ (95\% learner-cluster percentile interval $[-0.01723,-0.01105]$). To assess whether the predictive direction persists beyond the inspected 2009 mirror, we executed an independent-year ASSISTments 2015 replication. The initial 5,000-learner pilot was flagged as inconclusive because its PFA solver failed to converge. A separately fixed corrective protocol excluded \emph{all} pilot learners and evaluated new learners with a convergent optimizer for the same enriched-PFA logistic objective. On 750 learner-disjoint holdouts (26,697 responses), the GRU yields Brier 0.18227 and AUC 0.69968 against enriched PFA Brier 0.18695 and AUC 0.68129; the paired Brier contrast is $-0.00468$ (95\% learner-cluster interval $[-0.00607,-0.00335]$, 2,000 draws). This supports a bounded, different-year \emph{directional predictive replication}; it does not prove the effect of recurrent architecture, parity of the 2009 mirror with the officially corrected data, cross-institutional transportability, or instructional benefit. +Model comparisons in knowledge tracing depend on the learner split, source transformations, predictive information available to each method, and the population used for evaluation. We first reproduce a learner-disjoint, six-model study on a SHA-256-pinned transformed ASSISTments 2009 sequence mirror (4,148 learners; 274,331 responses). On 623 previously inspected test learners (41,228 responses), a seed-42 GRU achieves ROC-AUC 0.7470 and Brier 0.1800 against standard PFA AUC 0.6982 and Brier 0.1976. A same-test, \emph{exploratory} sensitivity comparison with enriched cross-skill PFA and longer GRU optimization yields Brier 0.1929 versus 0.1791, respectively; the paired response-pooled GRU-minus-PFA Brier difference is $-0.01387$ (95\% learner-cluster percentile interval $[-0.01723,-0.01105]$). To assess whether the predictive direction persists beyond the inspected 2009 mirror, we executed an independent-year ASSISTments 2015 replication. The initial 5,000-learner pilot was flagged as inconclusive because its PFA solver failed to converge. A separately fixed corrective protocol excluded \emph{all} pilot learners and evaluated new learners with a convergent optimizer for the same enriched-PFA logistic objective. On 750 learner-disjoint holdouts (26,697 responses), the GRU yields Brier 0.18227 and AUC 0.69968 against enriched PFA Brier 0.18695 and AUC 0.68129; the interaction-pooled paired Brier contrast is $-0.00468$ (learner-cluster bootstrap 95\% interval $[-0.00607,-0.00335]$, 2,000 draws, conditional on the fitted models). This supports a bounded, different-year \emph{directional predictive replication}; it does not prove the effect of recurrent architecture, parity of the 2009 mirror with the officially corrected data, cross-institutional transportability, or instructional benefit. \end{abstract} \textbf{Keywords:} AI in education; knowledge tracing; ASSISTments; leakage-aware evaluation; BKT; PFA; recurrent networks; calibration; independent-cohort replication. @@ -128,7 +128,7 @@ \subsection{Exploratory model and training-budget sensitivity} \section{Independent-Year ASSISTments 2015 Replication} \subsection{Frozen sampling sequence and diagnostic correction} -The first 2015 analysis sampled 5,000 learners with seed 20261022, then assigned learners to 70\%/15\%/15\% train, validation, and test groups, comparing the enriched cross-skill PFA with the extended GRU. Although this initial pilot yielded a nominal GRU Brier advantage, the PFA \texttt{saga} optimizer reached its iteration limit and the declared convergence gate failed. The pilot is archived and \emph{not counted as clean confirmation}. Its test results must not be selectively omitted simply because they are provisional. +The first 2015 analysis sampled 5,000 learners with seed 20261022, then assigned learners to 70\%/15\%/15\% train, validation, and test groups, comparing the enriched cross-skill PFA with the extended GRU. On the pilot's 750 held-out learners (25,089 responses), enriched PFA obtained Brier 0.18142 and ROC-AUC 0.68957, whereas the extended GRU obtained Brier 0.17644 and ROC-AUC 0.70779; the nominal response-pooled Brier difference was $-0.00498$ with learner-cluster percentile interval $[-0.00697,-0.00336]$. \emph{These pilot estimates are reported for transparency, not accepted as confirmation}: the PFA \texttt{saga} solver reached its iteration limit, violating the predetermined convergence gate. Thus the apparent first-sample advantage is computationally qualified and cannot be pooled with the subsequent confirmatory estimate. Before any second-stage test outcomes were inspected, a corrective protocol changed only the PFA numerical solver to \texttt{lbfgs} (\texttt{C=1}, maximum 2,000 iterations, tolerance $10^{-8}$) for the same regularized logistic model with skill intercepts, within-skill successes/failures, and pooled prior cross-skill successes/failures. It reconstructed and excluded every one of the 5,000 pilot learners, sampled 5,000 different learners (seed 20261023), and created learner-disjoint groups of 3,500 train, 750 validation, and 750 test learners. The GRU configuration remained fixed (seed 42; token embedding 32, skill query embedding 16, hidden size 64, Adam learning rate 0.002, up to 25 epochs, validation patience four). The GRU's best validation epoch was 7 of 11 executed. The enriched PFA optimizer converged in 484 iterations without convergence warnings. No observations from the first sample appear in the second sample. @@ -145,6 +145,12 @@ \subsection{Untouched second-sample results and estimand} \end{tabular} \end{table} +\begin{figure}[ht]\centering +\includegraphics[width=.88\linewidth]{cohort2015_brier.png} +\caption{Fresh 2015 cohort, second-stage held-out response-pooled Brier scores. These scores refer to models trained on the independent second-stage training learners. The initial PFA-nonconverged pilot is not represented as confirmatory evidence.} +\label{fig:stage2brier} +\end{figure} + The test correctness rate was 0.71697. The primary Brier difference was $-0.0046813$, with a 95\% learner-cluster percentile interval $[-0.0060730,-0.0033457]$. The prespecified \emph{second-stage} computational and directional gates passed, supported by GitHub Actions run \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39754854426}{39754854426}. The first, nonconverged stage and its outcomes remain available in run \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39754046492}{39754046492}. Because both 2015 experiments used the same source archive but \emph{nonoverlapping} learner samples, this is a fresh-sample, distinct-year predictive replication, not a demonstration of cross-institutional transport or strict information matching. \section{Discussion and Threats to Validity} @@ -156,13 +162,13 @@ \section{Discussion and Threats to Validity} \textbf{Information and attribution.} The enriched PFA uses compressed aggregate per-skill and across-skill histories; the GRU uses ordered prior interactions and a different parameter budget. Its superiority cannot be assigned specifically to recurrence. Existing correction-aware, cross-cohort, and information-controlled comparisons \citep{polydoros2026,padi2026} further constrain claims of novelty or state-of-the-art ranking. -\textbf{Uncertainty and target populations.} The 2009 original bootstrap treats the learner-average Brier contrast; the extended 2009 and new 2015 paired bootstraps target the interaction-pooled contrast and resample learners. The interval for the second 2015 cohort is conditional on its sampled learners, fixed model choices, and event selection rules, and cannot substitute for multi-institutional validation. The new cohort concerns only the source's 100 most-used skill builders, not all learners or curricula. \textbf{Educational interpretation.} Neither dataset supplies a randomized teaching intervention; predictive skill probabilities are not a direct measure of learning benefit, fairness, or readiness for consequential student decisions. +\textbf{Uncertainty and target populations.} The 2009 original bootstrap treats the learner-average Brier contrast; the extended 2009 and new 2015 paired bootstraps target the interaction-pooled contrast and resample learners. The second-stage 2015 interval conditions on the already fitted models and resamples held-out learners only; it does \emph{not} repeat the full training and model-selection procedure in every bootstrap draw. It therefore quantifies test-sample cluster uncertainty, not between-training-run, hyperparameter-selection, or broader population uncertainty. Secondary metrics (AUC, log loss, and ECE-10) are descriptive rather than separately multiplicity-controlled hypothesis tests. A single learner sample and model seed cannot substitute for multi-institutional validation. The new cohort concerns only the source's 100 most-used skill builders, not all learners or curricula. \textbf{Educational interpretation.} Neither dataset supplies a randomized teaching intervention; predictive skill probabilities are not a direct measure of learning benefit, fairness, or readiness for consequential student decisions. \section{Research Questions Revisited} \textbf{RQ1:} The six-model baseline reproduces on the fixed mirror, conditional on its source transformation. \textbf{RQ2:} First within-learner skill occurrences and repetitions differ descriptively, but are not directly comparable learning stages. \textbf{RQ3:} Learner-averaged and interaction-pooled Brier targets differ; their bootstrap intervals must be labeled accordingly. \textbf{RQ4:} The exploratory 2009 GRU/PFA contrast persisted with extended training and richer classical history, but on inspected learners. \textbf{RQ5 (external replication):} The directional interaction-pooled Brier advantage replicated on 750 untouched held-out 2015 learners in a separately fixed, converged corrective-stage evaluation, with the limitations above. \section{Conclusion} -Re-execution confirms the archived six-model metrics on a pinned transformed ASSISTments 2009 mirror, while exploratory richer-baseline and longer-training comparisons remain conditional on the previously inspected split. A distinct 2015 Skill Builder learner cohort, sampled after excluding all learners in a failed-convergence pilot, achieved a converged enriched-PFA comparison and confirmed the directional GRU advantage in interaction-pooled Brier loss ($-0.00468$, 95\% cluster interval $[-0.00607,-0.00335]$). The combined evidence supports a \emph{limited predictive replication} claim, not proof that recurrence itself is superior under information-matched conditions. Official corrected-2009 row-parity, broad generalization, and instructional utility remain unverified; the paper should be read as an auditable empirical working preprint, not a new knowledge-tracing algorithm or universal model ranking. +Re-execution confirms the archived six-model metrics on a pinned transformed ASSISTments 2009 mirror, while exploratory richer-baseline and longer-training comparisons remain conditional on the previously inspected split. A distinct 2015 Skill Builder learner cohort, sampled after excluding all learners in a failed-convergence pilot, achieved a converged enriched-PFA comparison and confirmed the directional GRU advantage in interaction-pooled Brier loss ($-0.00468$, 95\% cluster interval $[-0.00607,-0.00335]$). The combined evidence supports a \emph{limited predictive replication} claim on the evaluated ASSISTments cohorts, not proof that recurrence itself is superior under information-matched conditions. Stage-2 bootstrap inference conditions on fixed trained models; numerical success and source checksum verification do not establish external validity. Official corrected-2009 row-parity, broad generalization, and instructional utility remain unverified; the paper should be read as an auditable empirical working preprint, not a new knowledge-tracing algorithm or universal model ranking. \section*{Data availability and research integrity} The official 2009 correction notice is at \url{https://sites.google.com/site/assistmentsdata/home/2009-2010-assistment-data/skill-builder-data-2009-2010}, and the 2015 source at \url{https://sites.google.com/site/assistmentsdata/datasets/2015-assistments-skill-builder-data}. The 2009 pinned Atomi mirror and source SHA-256 are recorded in \url{https://github.com/devissaputra/knowledge_tracing_benchmark}. The re-execution of the 2009 frozen experiment appears in GitHub Actions run \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39655888620}{39655888620}. The first 2015 diagnostic pilot and final disjoint-cohort execution are in runs \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39754046492}{39754046492} and \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39754854426}{39754854426}, respectively. The frozen 2015 protocols, exact SHA-256, implementation and JSON results are in \url{https://github.com/devissaputra/knowledge_tracing_benchmark/pull/2}; the original exploratory extension is in \url{https://github.com/devissaputra/knowledge_tracing_benchmark/pull/1}. Raw individual learner records are not redistributed. Archived numerical evidence and code have passed automated computational checks; external peer review, full 2009 corrected-source parity, and author approval are not claimed. From fe430813ff444fe1922e7258fc008134357f99af Mon Sep 17 00:00:00 2001 From: Devis Wawan Saputra Date: Thu, 22 Oct 2026 17:34:36 +0700 Subject: [PATCH 20/48] paper: establish updated manuscript as canonical LaTeX main.tex --- paper/main.tex | 194 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 194 insertions(+) create mode 100644 paper/main.tex diff --git a/paper/main.tex b/paper/main.tex new file mode 100644 index 0000000..81a3e28 --- /dev/null +++ b/paper/main.tex @@ -0,0 +1,194 @@ +\documentclass[11pt,a4paper]{article} +\usepackage[margin=0.84in]{geometry} +\usepackage[T1]{fontenc} +\usepackage[utf8]{inputenc} +\usepackage{lmodern,microtype} +\usepackage{amsmath,amssymb,booktabs,array,graphicx,caption,subcaption} +\usepackage[colorlinks=true,urlcolor=blue,citecolor=blue,linkcolor=blue]{hyperref} +\usepackage{natbib} +\sloppy\emergencystretch=4em +\setlength{\parskip}{0.14em}\renewcommand{\baselinestretch}{0.96} +\setlength{\parindent}{0pt} +\title{Knowledge-Tracing Reproducibility and Information-Budget Auditing\\\large A Pinned 2009 Sequence Mirror and Fresh 2015 Learner-Cohort Replication} +\author{Devis Wawan Saputra\\\small Independent Researcher, Indonesia\\\small ORCID: \href{https://orcid.org/0000-0002-7133-9410}{0000-0002-7133-9410}} +\date{22 October 2026\\\small Empirical research manuscript; not peer reviewed} +\begin{document} +\maketitle +\begin{abstract} +Model comparisons in knowledge tracing depend on the learner split, source transformations, predictive information available to each method, and the population used for evaluation. We first reproduce a learner-disjoint, six-model study on a SHA-256-pinned transformed ASSISTments 2009 sequence mirror (4,148 learners; 274,331 responses). On 623 previously inspected test learners (41,228 responses), a seed-42 GRU achieves ROC-AUC 0.7470 and Brier 0.1800 against standard PFA AUC 0.6982 and Brier 0.1976. A same-test, \emph{exploratory} sensitivity comparison with enriched cross-skill PFA and longer GRU optimization yields Brier 0.1929 versus 0.1791, respectively; the paired response-pooled GRU-minus-PFA Brier difference is $-0.01387$ (95\% learner-cluster percentile interval $[-0.01723,-0.01105]$). To assess whether the predictive direction persists beyond the inspected 2009 mirror, we executed an independent-year ASSISTments 2015 replication. The initial 5,000-learner pilot was flagged as inconclusive because its PFA solver failed to converge. A separately fixed corrective protocol excluded \emph{all} pilot learners and evaluated new learners with a convergent optimizer for the same enriched-PFA logistic objective. On 750 learner-disjoint holdouts (26,697 responses), the GRU yields Brier 0.18227 and AUC 0.69968 against enriched PFA Brier 0.18695 and AUC 0.68129; the interaction-pooled paired Brier contrast is $-0.00468$ (learner-cluster bootstrap 95\% interval $[-0.00607,-0.00335]$, 2,000 draws, conditional on the fitted models). This supports a bounded, different-year \emph{directional predictive replication}; it does not prove the effect of recurrent architecture, parity of the 2009 mirror with the officially corrected data, cross-institutional transportability, or instructional benefit. +\end{abstract} +\textbf{Keywords:} AI in education; knowledge tracing; ASSISTments; leakage-aware evaluation; BKT; PFA; recurrent networks; calibration; independent-cohort replication. + +\section{Introduction} +Knowledge tracing predicts a learner's subsequent response from observed practice history. The probabilistic outputs can be used to rank cases for inspection and to support adaptive content, but predictive validity must not be conflated with instructional value. A model that has learned response regularities on a historical tutoring platform may still fail under new curricula, item policies, or student populations. + +A second challenge concerns experimental comparison. A random row split can let responses from the same learner appear in both model development and evaluation. Such a split evaluates interpolation within known learner histories rather than the intended generalization to unseen learners. Another challenge is information symmetry: a recurrent model may use prior outcomes from other skills while a classical per-skill model sees only within-skill history. Predictive differences then reflect both representation and architecture. + +Rather than introduce a new model, this study audits the empirical scope of a controlled learner-disjoint benchmark, explicitly recording probabilistic performance, slice results, calibration, and uncertainty limits. It answers: (RQ1) How do conventional baselines compare on a frozen learner-disjoint partition? (RQ2) Does prediction differ between a learner's first observed and later encounters with a skill? (RQ3) What do the resampling intervals establish under a fixed learner split? (RQ4) Do extended training and a richer transparent baseline materially change the results, and what remains untested? (RQ5) Does the direction of the predictive difference replicate on an independent-year cohort? + +\section{Related Work and Positioning} +BKT recursively updates a latent mastery probability using learning-transition, guess, and slip assumptions \citep{corbett1995}. PFA predicts responses with logistic effects of skill identity and previous successes and failures \citep{pavlik2009}. Deep Knowledge Tracing introduced recurrent sequence representations \citep{piech2015}. Later evaluations demonstrated that claims of deep-model superiority are highly contingent on datasets, baselines, and evaluation design \citep{gervet2020}. Broader surveys emphasize inconsistent versions, preprocessing, and metric protocols across commonly named knowledge-tracing datasets \citep{abdelrahman2023}. More recent work directly addresses correction-aware and cross-cohort benchmarking. A 2026 replication package by Polydoros compares classical and deep approaches on the corrected ASSISTments 2009 and 2015 cohorts with learner-level resampling \citep{polydoros2026}. Padi and colleagues (2026) show empirically that duplicate multi-skill records and asymmetries in the information available to different model families can change reported rankings \citep{padi2026}. These advances preclude claiming novelty for learner-disjoint splitting, calibration analysis, or information matching in isolation. They also provide corrected-source and cross-cohort comparators, so we make no state-of-the-art or architecture-advantage claims. The distinctive contribution here is the auditable sequence of a frozen transformed-mirror reproduction, contrasting uncertainty targets, an explicitly labeled exploratory information-budget audit, and a separate-year predictive replication that retains rather than conceals a nonconverged first pilot. Unlike a model-development paper, this investigation treats exact source provenance, estimand distinctions, and explicit failure of source parity as empirical audit findings rather than claiming a state-of-the-art architecture. + +\section{Study Contribution and Evidence Hierarchy} +We provide four bounded outputs: (1) an executable source- and learner-split-pinned reproduction of six models and model-specific probability metrics on the transformed 2009 mirror; (2) a distinction between equal-learner and interaction-pooled Brier uncertainty targets; (3) an exploratory same-split sensitivity analysis comparing cross-skill PFA summary features with extended GRU training; and (4) a documented independent-year 2015 predictive replication on a distinct set of learners. The two evidence tiers must not be conflated: post-inspection 2009 improvements are exploratory, while the second 2015 cohort is fresh with respect to learner identity and outcome inspection. The second-stage 2015 protocol was nonetheless fixed \emph{after} observing a failed optimizer convergence diagnostic in the first 2015 pilot; its corrective choice and chronology are disclosed, and the result is not described as prospectively registered or independently peer-reviewed. These analyses compare unequal information representations and model capacity; they cannot identify a recurrence-only effect. + +\section{Data and Evaluation Protocol} +\subsection{Dataset origin and unresolved transformation question} +The intended underlying source is the ASSISTments 2009--2010 Skill Builder data documented by its maintainers \citep{assistmentsdata}; the ASSISTments tutoring platform and its formative-assessment context were described by \citet{feng2009}. The execution pipeline reads \texttt{Atomi/ASSISTments2009} at pinned revision \texttt{c72a664a9693547fb206652}\allowbreak\texttt{ed2ce18e62d320c7d} and file \path{data/train-00000-of-00001.parquet}. The archived file SHA-256 is \texttt{2053b7ea63f1d5ea953834bf82ed8954}\\\texttt{3e1f11bae1e3e7535c2f9d7f49d4dda7}. The mirror contains one serialized sequence per learner. The empirical record reports 4,148 unique learners and 274,331 binary response observations. + +\textbf{Provenance qualification.} The original ASSISTments maintainers published corrected skill-builder records after identifying duplicated rows \citep{assistmentsdata}. The pinned third-party mirror does not supply all original row-level identifiers needed to prove its equivalence to that corrected official release. Consequently, all numerical results below apply to \emph{this pinned mirror representation}. This is a meaningful, narrower replication claim, not a corrected-release benchmark. The public sequence mirror contains binary response arrays and skill identifiers, but the current audit cannot map these serialized events back to unique corrected-release student--problem identifiers or recover an independently timestamped ordering of the underlying raw events. If one underlying student--problem event was inadvertently repeated within a sequence, a learner-disjoint split alone could not prevent within-sequence target-label contamination. The chronological and unique-event interpretation of the mirror must therefore be treated as an \emph{unverified input assumption}, not as a validated property of the original ASSISTments log. + +\subsection{Officially linked 2015 independent-year source} +For the different-year replication we use the original ASSISTments 2015 Skill Builder CSV linked by the project's official dataset portal \citep{assistments2015}. The original file SHA-256 is \texttt{75e1f46131d5897a388c82411bc303}\texttt{2279899cb1f1e66a9043354d41234407a0}. It contains 708,631 records; 24,830 were excluded by the frozen missingness/validity filter, leaving 19,840 eligible learner IDs. The study constructs within-learner sequences in ascending numerical \texttt{log\_id} order, using \texttt{sequence\_id} as a 2015 skill-builder identifier; these identifiers are not the same construct as the 2009 mirror's \texttt{skill\_ids}. This different-year study requires new fitting on 2015 training learners and does not test a frozen 2009-trained model on 2015. The analysis source, hashes, sampling seeds, and both pilot and corrective-stage outcomes are archived in the repository's research branch. The 2015 result does not retroactively validate the 2009 mirror transformation. + +\subsection{Partition unit, leakage checks, and model selection} +Learners are shuffled at seed 42 and divided into training (2,903), validation (622), and test (623) partitions. The runner first rejects duplicated \texttt{user\_id} sequence rows, then forbids learner overlap by partitioning whole serialized rows. It reports 194,951 training responses and 41,228 test responses. This check establishes no repeated \emph{learner identifier} across partitions, but does not verify the identity or chronological ordering of individual raw student--problem events. The validation set determines the BKT parameter choice and neural stopping epoch. Global and skill priors, PFA coefficients, and recurrent embeddings are learned from the training partition only. Every stateful model emits its prediction \emph{before} observing the target response. + +\subsection{Model specifications} +\textbf{Global prior.} The binary response probability is the aggregate training correctness fraction. \textbf{Skill prior.} The response probability is the training correctness frequency for the queried skill, with a global fallback. \textbf{Fixed BKT.} Initial mastery, learning, guess, and slip probabilities are respectively 0.20, 0.15, 0.20, and 0.10. The prediction for a current response uses the previous posterior mastery state: +\begin{equation} +P(Y_t=1\mid H_{t-1})=m_{t-1}(1-s)+(1-m_{t-1})g. +\end{equation} +\textbf{Validation-tuned BKT.} The four parameters are selected from a fixed grid using validation log loss; the archived selection is initial mastery 0.40, learning 0.15, guess 0.30, and slip 0.20. \textbf{PFA.} A regularized logistic model uses a skill intercept and preceding learner--skill success/failure counts. \textbf{GRU.} A 64-dimensional recurrent hidden state processes previous response-conditioned skill tokens and is queried with the next skill embedding; seed-42 predictions for each learner's first response fall back to a training skill/global prior. Seeds 13, 42, and 73 were run for at most eight training epochs with validation-loss monitoring. + +\textbf{Comparability caveat.} PFA counts only prior outcomes on the queried skill; the GRU can exploit the complete previous cross-skill sequence. The observed gap does not isolate the effect of nonlinearity or recurrence. The executed extension adds a PFA comparator with prior cross-skill success/failure counts, and longer GRU training. Because this comparator is still a compact summary rather than a full ordered-history model, the test does not isolate the effect of model architecture. + +\subsection{Evaluation, uncertainty, and estimands} +ROC-AUC and average precision quantify discrimination and ranking. Brier score and log loss quantify probabilistic accuracy. Expected calibration error (ECE-10) compares confidence and observed response frequencies in ten probability bins, but is sample/bin-sensitive. A learner-first skill occurrence means that skill key has not appeared previously \emph{for that learner}; it is not necessarily unseen in training. + +There are two distinct aggregation targets in the existing report. The primary table pools response-level losses, giving interaction-weighted estimates: +\begin{equation} +\widehat{B}_{\mathrm{event}}(f)=\frac{1}{N}\sum_{i=1}^{n}\sum_{t=1}^{T_i}(y_{it}-\hat p_{it}^{(f)})^2,\quad N=\sum_i T_i. +\end{equation} +The archived cluster bootstrap instead averages per-learner loss differences: +\begin{equation} +\widehat{\Delta}_{\mathrm{learner}}=\frac{1}{n}\sum_i\frac{1}{T_i}\sum_t\left[(y_{it}-\hat p_{it}^{(f)})^2-(y_{it}-\hat p_{it}^{(g)})^2\right]. +\end{equation} +That is a defensible \emph{different} estimand. It must not be interpreted as an uncertainty interval for the event-pooled Brier difference. A paired learner-cluster bootstrap for the event-weighted difference resamples learners and recomputes the ratio of summed losses to summed response counts in each replicate. The executed supplement implements this correction, with 2,000 learner-resampling draws per comparison. Each draw recomputes the ratio of the sampled cluster loss sums to the sampled cluster response counts; the resulting percentile intervals are conditional on this one previously examined test partition. + +\section{Reproduction and Exploratory Robustness Results} +\subsection{Overall held-out comparisons} +Table~\ref{tab:results} reports the results from a new full execution of the original study on the pinned real-data mirror. Reproduced original values agree with the archived record to within $4\times10^{-8}$ for ROC-AUC and much more closely for Brier score. +\begin{table}[ht] +\centering\small +\caption{Reproduced results on 41,228 responses from 623 learner-disjoint test learners (seed-42 original GRU). Lower Brier, log loss, and ECE are better.} +\label{tab:results} +\begin{tabular}{lrrrrr}\toprule +Model & ROC-AUC & AP & Brier & Log loss & ECE-10\\\midrule +Global prior & 0.5000 & 0.6747 & 0.2197 & 0.6313 & 0.0160\\ +Skill prior & 0.6234 & 0.7669 & 0.2097 & 0.6096 & 0.0145\\ +Fixed BKT & 0.6746 & 0.7806 & 0.2187 & 0.6360 & 0.1226\\ +Validation-tuned BKT & 0.6722 & 0.7770 & 0.2024 & 0.5928 & 0.0279\\ +PFA logistic & 0.6982 & 0.8074 & 0.1976 & 0.5840 & 0.0413\\ +GRU (seed 42) & \textbf{0.7470} & \textbf{0.8439} & \textbf{0.1800} & \textbf{0.5382} & \textbf{0.0133}\\\bottomrule +\end{tabular} +\end{table} +The recorded GRU-minus-PFA Brier difference is approximately $-0.01765$ (an 8.9\% relative reduction with PFA as denominator), and ROC-AUC difference is $+0.04878$. These are \emph{descriptive differences} on one test partition. A separate \emph{exploratory} paired uncertainty calculation for the later strengthened comparison is reported below. In particular, the fixed-parameter BKT achieves higher AUC than the training-skill prior while having worse Brier score, illustrating why ranking and probability quality must be considered separately. + +\begin{figure}[ht]\centering\includegraphics[width=.83\linewidth]{kt_brier.png}\caption{Reproduced response-pooled Brier scores for the original six benchmark models on the pinned real-data mirror.}\end{figure} + +\subsection{First skill occurrence versus repeated occurrences} +\begin{table}[ht]\centering\small +\caption{Recorded PFA and seed-42 GRU performance by within-learner skill occurrence.}\begin{tabular}{lrrrrr}\toprule +Slice & N & PFA AUC & GRU AUC & PFA Brier & GRU Brier\\\midrule +First observed skill & 6,193 & 0.6499 & 0.7224 & 0.2244 & 0.2094\\ +Repeated skill & 35,035 & 0.7068 & 0.7548 & 0.1929 & 0.1747\\\bottomrule +\end{tabular}\end{table} +The GRU has lower recorded Brier scores in both slices. However, this does not imply validated mastery estimation or a benefit for instructional decisions. Difficulty mix and number of available previous responses vary by slice. + +\subsection{Random-seed variability and archived learner uncertainty} +Across the three prespecified GRU seeds the mean ROC-AUC is 0.74708 (sample SD 0.00083) and mean Brier score is 0.17975 (sample SD 0.00019). These quantify three optimization restarts on the same data split, not independent replications or evidence of robustness across populations. + +The original run reproduces a \emph{mean per-learner} Brier difference between the seed-42 GRU and skill prior of $-0.02888$, with 95\% learner-block percentile interval $[-0.03439,-0.02365]$ for 623 learners. This interval must be linked to the learner-averaged quantity given above and cannot be relabelled as the pooled response-level comparison. The new cluster bootstrap for the interaction-pooled GRU--PFA comparison is presented separately below; its estimand must not be conflated with the original equal-learner-weighted interval. + +\subsection{Independent execution reproducibility check} +The original end-to-end workflow was rerun in GitHub Actions (run \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39655888620}{39655888620}) on 22 October 2026. The downloaded real-data file matched the archived SHA-256 \texttt{2053b7ea63f1d5ea953834bf82ed89543e1f11bae1e3e7535c2f9d7f49d4dda7}. Learner and response counts, learner partitions, original baseline Brier scores, and full-run AUCs matched archived results at reporting precision; the maximum original AUC drift was below $4\times10^{-8}$. Both code tests and the full data-validation job finished successfully. This is a computational reproduction of \emph{the same pinned third-party mirror}, not a parity audit against the official corrected source. + +\subsection{Exploratory model and training-budget sensitivity} +To reduce the original information asymmetry, an enriched PFA baseline uses skill identity and both skill-specific and overall prior success/failure counts, with no access to the current response. The recurrent model still has richer ordered history and greater representational capacity; the comparison improves historical-information matching but does not equalize feature classes or prove an architectural mechanism. Extended GRU optimization used seeds 13, 42, and 73, at most 25 epochs, validation-loss early stopping and patience four. All three fits selected their best validation state before the cap. Table~\ref{tab:robust} summarizes the recorded out-of-sample scores \emph{on the same already-inspected test partition}. + +\begin{table}[ht]\centering\small +\caption{Exploratory robustness results from real-data reruns. All rows use the same 623 test learners (41,228 responses); GRU settings are exploratory revisions after earlier test inspection.} +\label{tab:robust} +\begin{tabular}{lrrrr}\toprule +Model & ROC-AUC & Brier & Log loss & Best epoch\\\midrule +Standard PFA & 0.6982 & 0.1976 & 0.5840 & --\\ +PFA + cross-skill counts & 0.7135 & 0.1929 & 0.5709 & --\\ +GRU (seed 13, extended) & 0.7471 & 0.1793 & 0.5367 & 13\\ +GRU (seed 42, extended) & 0.7491 & 0.1791 & 0.5357 & 13\\ +GRU (seed 73, extended) & 0.7498 & 0.1789 & 0.5358 & 11\\\bottomrule +\end{tabular} +\end{table} + +Relative to the stronger PFA, the extended seed-42 GRU has a response-pooled Brier improvement of $-0.013874$ (paired learner-cluster 95\% percentile interval $[-0.017231,-0.011049]$). Results for seeds 13 and 73 are $-0.013583$ [$-0.017004,-0.010911$] and $-0.014020$ [$-0.017418,-0.011123$], respectively. These bootstrap intervals (2,000 resamples of learners) describe conditional uncertainty \emph{on the fixed test partition after exploratory model changes}; they should not be interpreted as confirmatory evidence from an untouched test sample. The stronger PFA itself improves Brier score relative to the original PFA by $-0.004680$ [$-0.005900,-0.003537$], underscoring the effect of history-feature availability. + +\begin{figure}[ht]\centering\includegraphics[width=.90\linewidth]{robust_comparison.png}\caption{Exploratory robustness: real-data ROC-AUC and Brier score for PFA variants and three extended GRU seeds. This is a same-split sensitivity comparison, not an independent validation sample.}\end{figure} + +\begin{figure}[ht]\centering\includegraphics[width=.83\linewidth]{kt_auc.png}\caption{Reproduced test ROC-AUC across original models, shown descriptively for the fixed split.}\end{figure} + +\section{Independent-Year ASSISTments 2015 Replication} +\subsection{Frozen sampling sequence and diagnostic correction} +The first 2015 analysis sampled 5,000 learners with seed 20261022, then assigned learners to 70\%/15\%/15\% train, validation, and test groups, comparing the enriched cross-skill PFA with the extended GRU. On the pilot's 750 held-out learners (25,089 responses), enriched PFA obtained Brier 0.18142 and ROC-AUC 0.68957, whereas the extended GRU obtained Brier 0.17644 and ROC-AUC 0.70779; the nominal response-pooled Brier difference was $-0.00498$ with learner-cluster percentile interval $[-0.00697,-0.00336]$. \emph{These pilot estimates are reported for transparency, not accepted as confirmation}: the PFA \texttt{saga} solver reached its iteration limit, violating the predetermined convergence gate. Thus the apparent first-sample advantage is computationally qualified and cannot be pooled with the subsequent confirmatory estimate. + +Before any second-stage test outcomes were inspected, a corrective protocol changed only the PFA numerical solver to \texttt{lbfgs} (\texttt{C=1}, maximum 2,000 iterations, tolerance $10^{-8}$) for the same regularized logistic model with skill intercepts, within-skill successes/failures, and pooled prior cross-skill successes/failures. It reconstructed and excluded every one of the 5,000 pilot learners, sampled 5,000 different learners (seed 20261023), and created learner-disjoint groups of 3,500 train, 750 validation, and 750 test learners. The GRU configuration remained fixed (seed 42; token embedding 32, skill query embedding 16, hidden size 64, Adam learning rate 0.002, up to 25 epochs, validation patience four). The GRU's best validation epoch was 7 of 11 executed. The enriched PFA optimizer converged in 484 iterations without convergence warnings. No observations from the first sample appear in the second sample. + +\subsection{Untouched second-sample results and estimand} +The primary estimand is the interaction-pooled paired Brier loss difference, GRU minus enriched PFA, on the \emph{second} sample's final holdout. Two thousand learner-cluster bootstrap replicates (seed 20261023) resample whole learners and preserve the pooled-interaction weighting. A directional replication was counted only if the upper 95\% percentile interval endpoint was below zero and all provenance, disjointness, and convergence checks passed. This is a one-sided directional assessment using a displayed two-sided interval, not a preregistered general benchmark claim. + +\begin{table}[ht]\centering\small +\caption{ASSISTments 2015 independent learner-cohort test (750 learners, 26,697 test responses). Higher AUC and lower Brier, log loss and ECE-10 are favorable. The second sample excludes all 5,000 learners from the first diagnostic pilot.} +\label{tab:cohort2015} +\begin{tabular}{lrrrr}\toprule +Model & ROC-AUC & Brier & Log loss & ECE-10\\\midrule +Enriched PFA (converged) & 0.68129 & 0.18695 & 0.55668 & 0.01694\\ +Extended GRU (seed 42) & 0.69968 & 0.18227 & 0.54346 & 0.00820\\\bottomrule +\end{tabular} +\end{table} + +\begin{figure}[ht]\centering +\includegraphics[width=.88\linewidth]{cohort2015_brier.png} +\caption{Fresh 2015 cohort, second-stage held-out response-pooled Brier scores. These scores refer to models trained on the independent second-stage training learners. The initial PFA-nonconverged pilot is not represented as confirmatory evidence.} +\label{fig:stage2brier} +\end{figure} + +The test correctness rate was 0.71697. The primary Brier difference was $-0.0046813$, with a 95\% learner-cluster percentile interval $[-0.0060730,-0.0033457]$. The prespecified \emph{second-stage} computational and directional gates passed, supported by GitHub Actions run \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39754854426}{39754854426}. The first, nonconverged stage and its outcomes remain available in run \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39754046492}{39754046492}. Because both 2015 experiments used the same source archive but \emph{nonoverlapping} learner samples, this is a fresh-sample, distinct-year predictive replication, not a demonstration of cross-institutional transport or strict information matching. + +\section{Discussion and Threats to Validity} +The 2009 mirror reproductions show an association between the richer sequence representation of the GRU and better test probability scores under the fixed split. The new 2015 experiment provides a separate-year check: on a distinct learner sample, a converged richer PFA remains behind the GRU in the predetermined directional Brier comparison. This is stronger evidence for repeatability of the \emph{observed predictive direction} than the 2009 reanalysis alone, but cannot isolate recurrent architecture from feature history, ordered signals, capacity, data processing, or optimization. + +\textbf{Source equivalence and chronological assumptions.} The official maintainers identify duplicated rows in the 2009--2010 skill-builder source and provide a corrected student--problem representation \citep{assistmentsdata}. The transformed Atomi mirror exposes user IDs, skill/grade lists, attempt counts and answer types, but not the row-level problem IDs, event IDs or validated timestamps required to prove corrected-source parity or exact per-event chronology. We checked the published source descriptions and schema and found no demonstrated row crosswalk. The 2009 findings \emph{remain mirror-specific}; the new 2015 experiment does not repair them. The 2015 file has event-log identifiers and a different skill-builder code definition. Ordering by \texttt{log\_id} is an explicitly documented modeling convention, not proof that the measure captures every physical action time correctly. + +\textbf{Pilot and numerical-selection transparency.} The first 2015 run did not pass the PFA convergence gate. The second-stage design and LBFGS replacement were frozen only after that failure had been observed; disjoint learners prevent direct sample reuse but do not turn the stage-2 plan into prospective preregistration. Reporting both stages prevents a misleading impression of a single perfect untouched evaluation. The 2015 results reflect one second-stage sample and seed, not many external sites or extensive split-seed sensitivity analysis. + +\textbf{Information and attribution.} The enriched PFA uses compressed aggregate per-skill and across-skill histories; the GRU uses ordered prior interactions and a different parameter budget. Its superiority cannot be assigned specifically to recurrence. Existing correction-aware, cross-cohort, and information-controlled comparisons \citep{polydoros2026,padi2026} further constrain claims of novelty or state-of-the-art ranking. + +\textbf{Uncertainty and target populations.} The 2009 original bootstrap treats the learner-average Brier contrast; the extended 2009 and new 2015 paired bootstraps target the interaction-pooled contrast and resample learners. The second-stage 2015 interval conditions on the already fitted models and resamples held-out learners only; it does \emph{not} repeat the full training and model-selection procedure in every bootstrap draw. It therefore quantifies test-sample cluster uncertainty, not between-training-run, hyperparameter-selection, or broader population uncertainty. Secondary metrics (AUC, log loss, and ECE-10) are descriptive rather than separately multiplicity-controlled hypothesis tests. A single learner sample and model seed cannot substitute for multi-institutional validation. The new cohort concerns only the source's 100 most-used skill builders, not all learners or curricula. \textbf{Educational interpretation.} Neither dataset supplies a randomized teaching intervention; predictive skill probabilities are not a direct measure of learning benefit, fairness, or readiness for consequential student decisions. + +\section{Research Questions Revisited} +\textbf{RQ1:} The six-model baseline reproduces on the fixed mirror, conditional on its source transformation. \textbf{RQ2:} First within-learner skill occurrences and repetitions differ descriptively, but are not directly comparable learning stages. \textbf{RQ3:} Learner-averaged and interaction-pooled Brier targets differ; their bootstrap intervals must be labeled accordingly. \textbf{RQ4:} The exploratory 2009 GRU/PFA contrast persisted with extended training and richer classical history, but on inspected learners. \textbf{RQ5 (external replication):} The directional interaction-pooled Brier advantage replicated on 750 untouched held-out 2015 learners in a separately fixed, converged corrective-stage evaluation, with the limitations above. + +\section{Conclusion} +Re-execution confirms the archived six-model metrics on a pinned transformed ASSISTments 2009 mirror, while exploratory richer-baseline and longer-training comparisons remain conditional on the previously inspected split. A distinct 2015 Skill Builder learner cohort, sampled after excluding all learners in a failed-convergence pilot, achieved a converged enriched-PFA comparison and confirmed the directional GRU advantage in interaction-pooled Brier loss ($-0.00468$, 95\% cluster interval $[-0.00607,-0.00335]$). The combined evidence supports a \emph{limited predictive replication} claim on the evaluated ASSISTments cohorts, not proof that recurrence itself is superior under information-matched conditions. Stage-2 bootstrap inference conditions on fixed trained models; numerical success and source checksum verification do not establish external validity. Official corrected-2009 row-parity, broad generalization, and instructional utility remain unverified; the paper should be read as an auditable empirical working preprint, not a new knowledge-tracing algorithm or universal model ranking. + +\section*{Data availability and research integrity} +The official 2009 correction notice is at \url{https://sites.google.com/site/assistmentsdata/home/2009-2010-assistment-data/skill-builder-data-2009-2010}, and the 2015 source at \url{https://sites.google.com/site/assistmentsdata/datasets/2015-assistments-skill-builder-data}. The 2009 pinned Atomi mirror and source SHA-256 are recorded in \url{https://github.com/devissaputra/knowledge_tracing_benchmark}. The re-execution of the 2009 frozen experiment appears in GitHub Actions run \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39655888620}{39655888620}. The first 2015 diagnostic pilot and final disjoint-cohort execution are in runs \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39754046492}{39754046492} and \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39754854426}{39754854426}, respectively. The frozen 2015 protocols, exact SHA-256, implementation and JSON results are in \url{https://github.com/devissaputra/knowledge_tracing_benchmark/pull/2}; the original exploratory extension is in \url{https://github.com/devissaputra/knowledge_tracing_benchmark/pull/1}. Raw individual learner records are not redistributed. Archived numerical evidence and code have passed automated computational checks; external peer review, full 2009 corrected-source parity, and author approval are not claimed. + +\section*{Limitations of authorship and interpretation} +This is a research manuscript prepared for author verification. The named author must personally verify the calculations, code, references, and contribution statements before submission. The text makes no claim of peer review or acceptance. Draft preparation and some analytical implementation involved generative AI assistance; any submitted version requires the named author's personal audit and approval of the methods, empirical provenance, interpretations, and disclosure in accordance with the destination archive's applicable policies. + +\begingroup\small +\setlength{\itemsep}{0pt} +\begin{thebibliography}{99} +\bibitem[Abdelrahman et al.(2023)]{abdelrahman2023} Abdelrahman, G., Wang, Q., and Nunes, B. (2023). Knowledge tracing: A survey. \emph{ACM Computing Surveys}, 55(11), Article 224. \url{https://doi.org/10.1145/3569576}. +\bibitem[Corbett and Anderson(1995)]{corbett1995} Corbett, A. T. and Anderson, J. R. (1995). Knowledge tracing: Modeling the acquisition of procedural knowledge. \emph{User Modeling and User-Adapted Interaction}, 4, 253--278. \url{https://doi.org/10.1007/BF01099821}. +\bibitem[Feng et al.(2009)]{feng2009} Feng, M., Heffernan, N. T., and Koedinger, K. R. (2009). Addressing the assessment challenge with an online system that tutors as it assesses. \emph{User Modeling and User-Adapted Interaction}, 19(3), 243--266. \url{https://doi.org/10.1007/s11257-009-9063-7}. +\bibitem[Gervet et al.(2020)]{gervet2020} Gervet, T., Koedinger, K., Schneider, J., and Mitchell, T. (2020). When is deep learning the best approach to knowledge tracing? \emph{Journal of Educational Data Mining}, 12(3), 31--54. \url{https://doi.org/10.5281/zenodo.4143614}. +\bibitem[Polydoros(2026)]{polydoros2026} Polydoros, G. (2026). Cross-Cohort Knowledge-Tracing Benchmark: ASSISTments 2009--2010 and 2015 Replication Package (v2.0.1, corrected statistical reporting). Zenodo software. \url{https://doi.org/10.5281/zenodo.22131530}. +\bibitem[Padi et al.(2026)]{padi2026} Padi, P., Morampudi, A., Irrinki, U. S. G., and Kakitapelli, P. K. D. (2026). Stable and faithful explanations for knowledge tracing. arXiv:2609.28502. \url{https://arxiv.org/abs/2609.28502}. +\bibitem[Pavlik et al.(2009)]{pavlik2009} Pavlik, P. I., Cen, H., and Koedinger, K. R. (2009). Performance factors analysis: A new alternative to knowledge tracing. \emph{Proceedings of AIED}, 531--538. \url{https://doi.org/10.3233/978-1-60750-028-5-531}. +\bibitem[Piech et al.(2015)]{piech2015} Piech, C., Bassen, J., Huang, J., Ganguli, S., Sahami, M., Guibas, L., and Sohl-Dickstein, J. (2015). Deep knowledge tracing. \emph{NeurIPS 28}. \url{https://arxiv.org/abs/1506.05908}. +\bibitem[ASSISTmentsData(2015)]{assistments2015} ASSISTmentsData (2015). 2015 ASSISTments Skill Builder Data, original dataset portal (100 skill builders). \url{https://sites.google.com/site/assistmentsdata/datasets/2015-assistments-skill-builder-data}. +\bibitem[ASSISTmentsData(n.d.)]{assistmentsdata} ASSISTmentsData (n.d.). Skill-builder data 2009--2010, correction notice and official data. \url{https://sites.google.com/site/assistmentsdata/home/2009-2010-assistment-data/skill-builder-data-2009-2010}. +\end{thebibliography} +\endgroup +\end{document} \ No newline at end of file From 57d7b66319627abf3bb3e59236e26ac81db81b4d Mon Sep 17 00:00:00 2001 From: Devis Wawan Saputra Date: Thu, 22 Oct 2026 17:34:41 +0700 Subject: [PATCH 21/48] paper: add reproducible figure generator from executed metrics --- paper/generate_figures.py | 68 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 paper/generate_figures.py diff --git a/paper/generate_figures.py b/paper/generate_figures.py new file mode 100644 index 0000000..565d1c3 --- /dev/null +++ b/paper/generate_figures.py @@ -0,0 +1,68 @@ +"""Rebuild manuscript figures from previously archived real-data GitHub Actions JSON. + +Run from repository root: python paper/generate_figures.py +Requires matplotlib and numpy. Does not download learner-level data or refit models. +""" +from __future__ import annotations +import json +from pathlib import Path +import matplotlib.pyplot as plt +import numpy as np + +ROOT=Path(__file__).resolve().parents[1] +OUT=Path(__file__).resolve().parent + +def read(rel): + path=ROOT/rel + if not path.is_file(): raise FileNotFoundError(f'Required executed output: {path}') + return json.loads(path.read_text(encoding='utf-8')) + +def bars(items,metric,title,filename,ylim=None): + fig,ax=plt.subplots(figsize=(7,3.35)) + names=[str(n) for n,_ in items] + values=[float(d[metric]) for _,d in items] + x=np.arange(len(items)) + ax.bar(x,values,width=.68) + ax.set_xticks(x,names,rotation=18,ha='right') + ax.set_ylabel(metric.replace('_',' ').upper() if metric=='roc_auc' else 'Brier score (lower is better)') + ax.set_title(title,fontsize=10) + if ylim is not None:ax.set_ylim(*ylim) + ax.spines['top'].set_visible(False) + ax.spines['right'].set_visible(False) + fig.tight_layout(pad=1.5) + fig.savefig(OUT/filename,dpi=180) + plt.close(fig) + +original=read('results/metrics.json') +robust=read('results/robustness_extension.json') +new=read('results/confirmatory_assist2015_stage2.json') +assert original['dataset']['n_learners']==4148 +assert robust['status']=='complete_exploratory_real_data' +assert new['status']=='passed_all_computational_gates' and new['directional_replication_confirmed'] +assert new['paired_brier']['n_interactions']==26697 + +choices=[('Global prior','global_rate'),('Skill prior','skill_prior'),('Fixed BKT','bkt_fixed'),('Tuned BKT','bkt_validation_tuned'),('PFA','pfa_logistic'),('GRU','gru_kt_seed_42')] +m=[(name,original['test_metrics'][key]['all']) for name,key in choices] +bars(m,'brier','2009 mirror: original six-model test','kt_brier.png',ylim=(0,.25)) +bars(m,'roc_auc','2009 mirror: original six-model test','kt_auc.png',ylim=(0,.85)) + +fig,axes=plt.subplots(1,2,figsize=(8,3.1)) +labels=['PFA','Enriched PFA','GRU 13','GRU 42','GRU 73'] +rs=[robust['pfa'],robust['stronger_pfa']]+[robust['extended_gru'][str(s)]['metrics'] for s in [13,42,73]] +for ax,key,desc in zip(axes,['roc_auc','brier'],['ROC-AUC (higher is better)','Brier (lower is better)']): + vals=[float(t[key]) for t in rs] + ax.bar(range(len(rs)),vals,width=.68) + ax.set_xticks(range(len(rs)),labels,rotation=28,ha='right') + ax.set_ylabel(desc) + ax.spines['top'].set_visible(False) + ax.spines['right'].set_visible(False) +fig.suptitle('2009 exploratory same-test information-budget sensitivity',fontsize=10) +fig.tight_layout(pad=1.2) +fig.savefig(OUT/'robust_comparison.png',dpi=180) +plt.close(fig) + +ms=new['test_metrics'] +bars([('Enriched PFA (converged)',ms['enriched_pfa']),('Extended GRU',ms['extended_gru'])], + 'brier','Fresh ASSISTments 2015: 750 learners; 26,697 held-out responses', + 'cohort2015_brier.png',ylim=(0,.23)) +print('Generated four manuscript figures from executed JSON.') \ No newline at end of file From 7e4683b089f87837b918c2745ca2deff32e8e5ee Mon Sep 17 00:00:00 2001 From: Devis Wawan Saputra Date: Thu, 22 Oct 2026 17:36:02 +0700 Subject: [PATCH 22/48] paper: document canonical manuscript build and author review gates --- paper/BUILD_AND_AUTHOR_REVIEW.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 paper/BUILD_AND_AUTHOR_REVIEW.md diff --git a/paper/BUILD_AND_AUTHOR_REVIEW.md b/paper/BUILD_AND_AUTHOR_REVIEW.md new file mode 100644 index 0000000..a1ba9e9 --- /dev/null +++ b/paper/BUILD_AND_AUTHOR_REVIEW.md @@ -0,0 +1,32 @@ +# Canonical Paper 1 — build and author review + +The **actual** updated manuscript is `paper/main.tex` on the research branch `research/kt-fresh-cohort-20261022`. The previous `paper/main_integrated_20261022.tex` is synchronized for revision history. The distribution in ChatGPT Library `/Paper1_Final_20261022/Paper1_Actual_Manuscript_Source_20261022.zip` includes four required PNG figures and is directly compilable. + +## Build from repository root + +```bash +python paper/generate_figures.py +cd paper +latexmk -pdf -interaction=nonstopmode -halt-on-error main.tex +``` + +The figure generator uses the actual archived GitHub Actions JSON outputs; it does not fabricate any research data. It needs `matplotlib` and `numpy`. When working from the downloadable source ZIP, figures are already provided, so `latexmk -pdf main.tex` suffices. + +## What changed in the manuscript + +1. Preserves the frozen 2009 transformed-mirror benchmark and the exploratory enhanced-PFA/GRU analyses. +2. Documents the initial ASSISTments 2015 test (25,089 held-out responses) with actual model scores and its failed numerical convergence gate. The pilot is **not** counted as confirmatory evidence. +3. Integrates the independent, learner-disjoint 2015 corrective-stage test (750 held-out learners; 26,697 responses), numerical point estimates, bootstrap interval and figure. +4. States explicitly that the bootstrap resampled learners **without refitting the training models**. This captures held-out sample uncertainty conditional on fitted models, not optimization instability or between-cohort variability. +5. Preserves caveats: 2009 mirror not proven equivalent to corrected official data; model information/capacity unmatched; no causal or instructional-effect claims. + +## Record of local QA + +- 10-page A4, selectable text, all pages checked by rendering. +- Four figures included; no missing references or LaTeX errors; no overfull LaTeX boxes. +- PDF successfully rebuilt from source in a clean temporary directory. +- 19/19 consistency/build checks passed in the accompanying `MANUSCRIPT_QA.json`. + +## Final gates not delegated + +The author must review the science, verify the provenance and calculations personally, approve contribution/originality and disclosure statements, and choose the arXiv category and submit. The branch must not be silently treated as peer-reviewed or as evidence of data parity with corrected ASSISTments 2009-2010. \ No newline at end of file From b1f4939766a230342583e6b0d440a7dae98b6020 Mon Sep 17 00:00:00 2001 From: Devis Wawan Saputra Date: Thu, 22 Oct 2026 17:36:04 +0700 Subject: [PATCH 23/48] paper: link canonical main.tex and build instructions --- paper/paper.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/paper/paper.md b/paper/paper.md index 7e4a07e..787a2a8 100644 --- a/paper/paper.md +++ b/paper/paper.md @@ -11,8 +11,8 @@ ## Manuscript and audit sources -- [Final integrated LaTeX manuscript](main_integrated_20261022.tex) -- [Final numerical/source-package QA results](FINAL_QA_RESULTS_20261022.json) +- [Canonical updated LaTeX manuscript](main.tex) +- [Manuscript build and review instructions](BUILD_AND_AUTHOR_REVIEW.md) · [Final numerical/source-package QA results](FINAL_QA_RESULTS_20261022.json) - [Source-parity and novelty assessment](PROVENANCE_NOVELTY_AUDIT_20261022.md) - [External cohort replication details](external_replication_assist2015.md) - [First 2015 protocol](../CONFIRMATORY_PROTOCOL_2015.md) and [corrective-stage protocol](../CONFIRMATORY_STAGE2_PROTOCOL_2015.md) From 29c60191f1c86590b266a982a4ae0b9a8382a32e Mon Sep 17 00:00:00 2001 From: Devis Wawan Saputra Date: Thu, 22 Oct 2026 18:52:47 +0700 Subject: [PATCH 24/48] research: investigate corrected 2009 source parity (scripts/audit_corrected_2009.py) --- scripts/audit_corrected_2009.py | 212 ++++++++++++++++++++++++++++++++ 1 file changed, 212 insertions(+) create mode 100644 scripts/audit_corrected_2009.py diff --git a/scripts/audit_corrected_2009.py b/scripts/audit_corrected_2009.py new file mode 100644 index 0000000..fc20e15 --- /dev/null +++ b/scripts/audit_corrected_2009.py @@ -0,0 +1,212 @@ +"""Public-data provenance audit; never stores or prints student-level records. + +Compares the pinned Atomi serialized sequences to the original-data maintainers' +corrected 2009 Skill Builder release. The corrected release retains student/problem +identifiers that the third-party transformed mirror omits, so this can reject +exact parity but cannot in general prove event-by-event identity or chronology. + +Run: python scripts/audit_corrected_2009.py +Data downloads stay in the untracked data/cache folder. +""" +from __future__ import annotations +import hashlib +import json +from collections import Counter +from pathlib import Path +import re + +CORRECTED_FILE_ID = '1NNXHFRxcArrU0ZJSb9BIL56vmUt5FhlE' +SOURCE_PAGE = 'https://sites.google.com/site/assistmentsdata/home/2009-2010-assistment-data/skill-builder-data-2009-2010' +MIRROR_REPO = 'Atomi/ASSISTments2009' +MIRROR_REV = 'c72a664a9693547fb206652ed2ce18e62d320c7d' +MIRROR_FILE = 'data/train-00000-of-00001.parquet' +MIRROR_SHA = '2053b7ea63f1d5ea953834bf82ed89543e1f11bae1e3e7535c2f9d7f49d4dda7' + + +def identifier(x): + """Normalize simple integer/decimal IDs, but not compound skill strings.""" + s = str(x).strip() + if s.endswith('.0') and s[:-2].lstrip('-').isdigit(): + return s[:-2] + return s + + +def skill_token(x): + s = str(x).strip() + if s.lower() in {'nan', 'none', '', 'null'}: + return '' + # Corrected data use skill1_skill2; some public CSVs use semicolon lists. + parts = re.split(r'[_;]', s) + return '_'.join(identifier(z) for z in parts if z.strip()) + + +def load_csv_or_archive(path): + import io + import zipfile + import pandas as pd + if zipfile.is_zipfile(path): + with zipfile.ZipFile(path) as z: + members = [x for x in z.namelist() if x.lower().endswith('.csv') and not x.startswith('__MACOSX/')] + if not members: + raise ValueError('Official corrected archive lacks a CSV') + members.sort(key=lambda m: z.getinfo(m).file_size, reverse=True) + with z.open(members[0]) as f: + return pd.read_csv(f, low_memory=False, encoding='latin-1') + if path.read_bytes()[:32].lower().lstrip().startswith((b' Date: Thu, 22 Oct 2026 18:52:50 +0700 Subject: [PATCH 25/48] research: investigate corrected 2009 source parity (.github/workflows/official-2009-parity.yml) --- .github/workflows/official-2009-parity.yml | 52 ++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 .github/workflows/official-2009-parity.yml diff --git a/.github/workflows/official-2009-parity.yml b/.github/workflows/official-2009-parity.yml new file mode 100644 index 0000000..b41b840 --- /dev/null +++ b/.github/workflows/official-2009-parity.yml @@ -0,0 +1,52 @@ +name: "Audit corrected official ASSISTments 2009 against pinned third-party mirror" +on: + push: + branches: + - research/kt-fresh-cohort-20261022 + paths: + - scripts/audit_corrected_2009.py + - .github/workflows/official-2009-parity.yml + workflow_dispatch: +permissions: + contents: write +jobs: + source-parity: + runs-on: ubuntu-latest + timeout-minutes: 50 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install source-audit dependencies + run: | + python -m pip install 'pandas==2.3.3' 'pyarrow==23.0.1' 'huggingface-hub==1.33.0' 'gdown>=5.2,<6' + - name: Verify audit script syntax + run: python -m py_compile scripts/audit_corrected_2009.py + - name: Compare official corrected raw records against pinned mirror + run: python scripts/audit_corrected_2009.py + - name: Verify audit records uncertainty explicitly + run: | + python - <<'PY' + import json + from pathlib import Path + r=json.loads(Path('results/official_2009_mirror_parity_audit.json').read_text()) + assert r['parity_status'] in ('blocked_not_established','exact_full_dataset_parity_disproved_by_necessary_conditions','aggregate_necessary_conditions_pass_but_event_parity_not_proven') + assert 'student_records' not in r + print('SOURCE AUDIT STATUS:',r['parity_status']) + PY + - uses: actions/upload-artifact@v4 + if: always() + with: + name: official-2009-mirror-parity-metadata-only + path: results/official_2009_mirror_parity_audit.json + - name: Preserve report in research branch + if: success() + run: | + git config user.name 'github-actions[bot]' + git config user.email '41898282+github-actions[bot]@users.noreply.github.com' + git add results/official_2009_mirror_parity_audit.json + if ! git diff --cached --quiet; then + git commit -m 'research: archive corrected 2009 versus mirror provenance audit' + git push origin HEAD:research/kt-fresh-cohort-20261022 + fi \ No newline at end of file From b276fcafc4204e12ba48c85b62e5b1b16d8528c0 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 22 Oct 2026 11:53:19 +0000 Subject: [PATCH 26/48] research: archive corrected 2009 versus mirror provenance audit --- .../official_2009_mirror_parity_audit.json | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 results/official_2009_mirror_parity_audit.json diff --git a/results/official_2009_mirror_parity_audit.json b/results/official_2009_mirror_parity_audit.json new file mode 100644 index 0000000..0c22620 --- /dev/null +++ b/results/official_2009_mirror_parity_audit.json @@ -0,0 +1,40 @@ +{ + "source": { + "official_documentation_url": "https://sites.google.com/site/assistmentsdata/home/2009-2010-assistment-data/skill-builder-data-2009-2010", + "official_corrected_drive_id": "1NNXHFRxcArrU0ZJSb9BIL56vmUt5FhlE", + "corrected_file_sha256": "162ef8d2d28bcbfea6591a282994062bd8d5eaa00636544292a0d268dca6e5da", + "corrected_file_bytes": 64412812, + "source_columns": [ + "user_id", + "skill_id", + "correct", + "order_id" + ], + "corrected_source_records_total": 346860, + "records_valid_binary": 346860, + "records_valid_binary_and_labeled_skill": 283105, + "source_unique_learners_with_skills": 4163, + "source_rows_repeating_student_order_key": 0 + }, + "mirror": { + "repository": "Atomi/ASSISTments2009", + "revision": "c72a664a9693547fb206652ed2ce18e62d320c7d", + "sha256": "2053b7ea63f1d5ea953834bf82ed89543e1f11bae1e3e7535c2f9d7f49d4dda7", + "serialized_learners": 4148, + "total_interactions": 274331 + }, + "comparison": { + "shared_learner_ids": 4148, + "official_only_learner_ids": 15, + "mirror_only_learner_ids": 0, + "shared_learner_ids_same_number_of_interactions": 3331, + "shared_learner_ids_identical_skill_response_multisets": 3331, + "shared_learner_ids_identical_native_CSV_order": 1441, + "shared_learner_ids_identical_numeric_order_id_order": 3331, + "mirror_interactions_shared_learners": 274331, + "corrected_interactions_shared_learners": 283014 + }, + "parity_status": "exact_full_dataset_parity_disproved_by_necessary_conditions", + "interpretation": "Necessary count and (skill, correctness) multiset conditions only; no mirror original event IDs, no independent chronology guarantee. Equal aggregates cannot prove row identity; unequal totals/multisets disprove exact full parity.", + "privacy": "No learner IDs, rows, or raw source bytes are written to output." +} \ No newline at end of file From 52706062b8cd7277070ea7ff7e781cbd907d9a11 Mon Sep 17 00:00:00 2001 From: Devis Wawan Saputra Date: Thu, 22 Oct 2026 18:56:38 +0700 Subject: [PATCH 27/48] research: frozen corrected-2009 sensitivity CORRECTED_2009_REANALYSIS_PROTOCOL.md --- CORRECTED_2009_REANALYSIS_PROTOCOL.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 CORRECTED_2009_REANALYSIS_PROTOCOL.md diff --git a/CORRECTED_2009_REANALYSIS_PROTOCOL.md b/CORRECTED_2009_REANALYSIS_PROTOCOL.md new file mode 100644 index 0000000..3ab106d --- /dev/null +++ b/CORRECTED_2009_REANALYSIS_PROTOCOL.md @@ -0,0 +1,9 @@ +# Exploratory official-corrected-2009 source sensitivity protocol (22 Oct 2026) + +**Rationale:** Direct independent source audit proves that the frozen 2009 Atomi learner-sequence mirror and the official corrected 2009--2010 skill-builder release are not event-equivalent. The previously published model differences are conditional on the mirror. Execute one source-corrected sensitivity comparison on the source bytes whose provenance was just verified. + +**Source:** The exact corrected file linked from the ASSISTments maintainers' 2009--2010 Skill Builder page: Google Drive ID `1NNXHFRxcArrU0ZJSb9BIL56vmUt5FhlE`, SHA-256 `162ef8d2d28bcbfea6591a282994062bd8d5eaa00636544292a0d268dca6e5da`. Preserve original file; never redistribute its raw student records. Exclude only rows with absent/invalid binary outcome or unlabeled skill; retain one record per corrected student--problem event; source has 346,860 rows and 283,105 labeled rows spanning 4,163 learners. Normalize simple numeric skill IDs, preserve `_` composite skill keys. Order events within learner by increasing numeric `order_id`. Assert unique (student, order_id) and deterministic tie treatment. Event chronology depends on the source documentation's `order_id` semantics; treat this as an assumption, not independently reconstructed timestamps. + +**Preselected experiment:** Train/test split by whole learner with deterministic fixed seed 20261024 and `kt.split_rows` 70/15/15. Fit *enriched PFA*, using skill identity, prior same-skill successes/failures and prior across-skill successes/failures, `StandardScaler` for counts, `OneHotEncoder(handle_unknown='ignore')`, regularized logistic regression with `solver=lbfgs`, `C=1.0`, `max_iter=2000`, `tol=1e-8`. Reject nonconvergent fits. Fit the existing compact GRU, seed 42, token_dim=32, query_dim=16, hidden=64, max 25 epochs, patience 4, Adam 0.002; select stop epoch with validation learners only. First test event uses training skill/global prior. Compare test Brier scores (primary), AUC, log loss and calibration (secondary). Use 2,000 paired learner-cluster bootstrap draws seed 20261024 on pooled interaction-weighted GRU-minus-PFA Brier. Do not choose a different seed/hyperparameter to rescue a failed result. Record null/failure status honestly if invalid or interrupted. + +**Interpretation:** This analysis tests a corrected-source *sensitivity*, not an independent untouched 2009 hypothesis confirmation, because the original 2009 population and comparison direction were already analyzed. It does **not** establish a recurrence-only architecture effect, direct parity of the mirror with official data, or generalizability beyond the cohort. Do not edit the original mirror outcome or supplant the separate fresh 2015 replication. \ No newline at end of file From 9592c30097a1731871494cd47a866e5f5f2985bc Mon Sep 17 00:00:00 2001 From: Devis Wawan Saputra Date: Thu, 22 Oct 2026 18:56:41 +0700 Subject: [PATCH 28/48] research: frozen corrected-2009 sensitivity scripts/corrected_2009_reanalysis.py --- scripts/corrected_2009_reanalysis.py | 101 +++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 scripts/corrected_2009_reanalysis.py diff --git a/scripts/corrected_2009_reanalysis.py b/scripts/corrected_2009_reanalysis.py new file mode 100644 index 0000000..7fcd354 --- /dev/null +++ b/scripts/corrected_2009_reanalysis.py @@ -0,0 +1,101 @@ +"""Exploratory, source-corrected 2009 learning-sequence analysis, uses official raw data. +The source hash and protocol are fixed in CORRECTED_2009_REANALYSIS_PROTOCOL.md. +Outputs aggregate metadata only, not learner records. +""" +from __future__ import annotations +import hashlib, json, sys, warnings +from pathlib import Path +import numpy as np +import pandas as pd +from sklearn.exceptions import ConvergenceWarning +ROOT=Path(__file__).resolve().parents[1] +sys.path.insert(0,str(ROOT/'src')) +sys.path.insert(0,str(ROOT/'scripts')) +from audit_corrected_2009 import get_source,load_csv_or_archive,identifier,skill_token +from robustness_extension import paired_cluster_bootstrap, pfa_plus_all_skill_history_frame +import run_experiment as kt +from sklearn.compose import ColumnTransformer +from sklearn.linear_model import LogisticRegression +from sklearn.pipeline import Pipeline +from sklearn.preprocessing import OneHotEncoder,StandardScaler + +SOURCE_HASH='162ef8d2d28bcbfea6591a282994062bd8d5eaa00636544292a0d268dca6e5da' +SEED=20261024 + +def load(): + p=get_source() + sha=hashlib.sha256(p.read_bytes()).hexdigest() + if sha!=SOURCE_HASH:raise RuntimeError('Corrected 2009 official raw source hash does not match audited file') + raw=load_csv_or_archive(p) + raw.columns=[str(c).strip().lower().replace(' ','_') for c in raw.columns] + required=('user_id','order_id','skill_id','correct') + if any(c not in raw.columns for c in required):raise ValueError('Corrected source lacks essential fields') + n_all=len(raw) + source=raw[list(required)].copy() + source=source.dropna(subset=['user_id','order_id','skill_id','correct']).copy() + source['user_id']=source['user_id'].map(identifier) + source['skill_id']=source['skill_id'].map(skill_token) + source['correct']=pd.to_numeric(source['correct'],errors='coerce') + source['order_id']=pd.to_numeric(source['order_id'],errors='coerce') + valid=source['correct'].isin([0,1]) & source['order_id'].notna() & (source['skill_id']!='') + source=source.loc[valid].copy() + source['correct']=source['correct'].astype(int) + if source.duplicated(['user_id','order_id']).any():raise ValueError('Duplicated learner-event order_id') + source=source.sort_values(['user_id','order_id'],kind='stable') + rows=[{'user_id':str(user),'skill_ids':g['skill_id'].tolist(),'grades':g['correct'].tolist()} + for user,g in source.groupby('user_id',sort=True)] + if len(rows)!=4163 or len(source)!=283105 or n_all!=346860: + raise ValueError('Corrected source aggregate record count differs from independent provenance audit') + return rows,{'corrected_sha256':sha,'original_source_rows':n_all,'labeled_rows':len(source), + 'excluded_unlabeled_or_invalid':n_all-len(source), + 'learner_count':len(rows),'chronology':'numeric order_id; source-identified sequence order'} + +def pfa(train,test): + X,y,*_=pfa_plus_all_skill_history_frame(train) + XX,yy,learners,skills,first=pfa_plus_all_skill_history_frame(test) + pipe=Pipeline([('prep',ColumnTransformer([ + ('skill',OneHotEncoder(handle_unknown='ignore'),['skill']), + ('counts',StandardScaler(),['skill_success','skill_failure','all_success','all_failure'])])), + ('clf',LogisticRegression(solver='lbfgs',max_iter=2000,tol=1e-8,C=1.0))]) + with warnings.catch_warnings(record=True) as captured: + warnings.simplefilter('always',ConvergenceWarning) + pipe.fit(X,y) + messages=[str(q.message) for q in captured if issubclass(q.category,ConvergenceWarning)] + return kt.prediction_frame(yy,pipe.predict_proba(XX)[:,1],learners,skills,first),messages,int(pipe.named_steps['clf'].n_iter_[0]) + +def run(): + rows,source=load() + train,val,test=kt.split_rows(rows,seed=SEED) + ids=[{str(q['user_id']) for q in group} for group in (train,val,test)] + assert not(ids[0]&ids[1] or ids[0]&ids[2] or ids[1]&ids[2]) + pfa_pred,warnings_list,pfa_n=pfa(train,test) + global_rate,skills,_=kt.train_priors(train) + vocab=kt.skill_vocabulary(train) + kt.GRU_PATIENCE=4 + model,history,best=kt.train_gru(train,val,vocab,seed=42,epochs=25) + gru_pred=kt.predict_gru(model,test,vocab,global_rate,skills) + delta=paired_cluster_bootstrap(pfa_pred,gru_pred,repetitions=2000,seed=SEED) + guards=not warnings_list and len(history)>=1 and delta['n_learners']==len(test) + return {'status':'completed_exploratory_corrected_source' if guards else 'inconclusive_diagnostics', + 'analysis_type':'exploratory source-sensitivity, not confirmatory', + 'source':source, + 'protocol':{'split_seed':SEED,'gru_seed':42,'train_learners':len(train),'validation_learners':len(val), + 'test_learners':len(test),'pfa_solver':'lbfgs','pfa_C':1.0,'pfa_max_iter':2000,'pfa_tol':1e-8, + 'gru_max_epochs':25,'gru_patience':4,'bootstrap_resamples':2000,'bootstrap_seed':SEED}, + 'pfa_convergence_warnings':warnings_list,'pfa_iterations':pfa_n,'gru_epochs_completed':len(history), + 'gru_best_epoch':best, + 'metrics':{'enriched_pfa':kt.metric_bundle(pfa_pred['y'],pfa_pred['p']), + 'extended_gru':kt.metric_bundle(gru_pred['y'],gru_pred['p'])}, + 'paired_brier':delta, + 'interpretation':'Numbers are conditional on corrected source and selected splitting/representation. This is not a new untouched 2009 holdout or causal architecture effect.'} + +def main(): + out=Path('results/corrected_2009_sensitivity.json') + out.parent.mkdir(parents=True,exist_ok=True) + try:r=run() + except Exception as e: + r={'status':'blocked_or_failed','error_type':type(e).__name__, + 'diagnostic':str(e)[:700],'interpretation':'Never substitute expected results or claim success on a blocked source-corrected run.'} + out.write_text(json.dumps(r,indent=2),encoding='utf-8') + print(json.dumps(r,indent=2)) +if __name__=='__main__':main() \ No newline at end of file From 7651f5d98835d9ccb31392b0c66ad4577d43e46f Mon Sep 17 00:00:00 2001 From: Devis Wawan Saputra Date: Thu, 22 Oct 2026 18:56:44 +0700 Subject: [PATCH 29/48] research: frozen corrected-2009 sensitivity .github/workflows/corrected-2009-reanalysis.yml --- .../workflows/corrected-2009-reanalysis.yml | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 .github/workflows/corrected-2009-reanalysis.yml diff --git a/.github/workflows/corrected-2009-reanalysis.yml b/.github/workflows/corrected-2009-reanalysis.yml new file mode 100644 index 0000000..1fe98f3 --- /dev/null +++ b/.github/workflows/corrected-2009-reanalysis.yml @@ -0,0 +1,55 @@ +name: "Exploratory corrected-source 2009 GRU versus enriched PFA" +on: + push: + branches: + - research/kt-fresh-cohort-20261022 + paths: + - scripts/corrected_2009_reanalysis.py + - .github/workflows/corrected-2009-reanalysis.yml + - CORRECTED_2009_REANALYSIS_PROTOCOL.md + workflow_dispatch: +permissions: + contents: write +jobs: + reanalysis: + runs-on: ubuntu-latest + timeout-minutes: 100 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install frozen numeric and raw-source packages + run: | + python -m pip install -r requirements-repro.txt + python -m pip install 'gdown>=5.2,<6' + - name: Inspect Python code + run: python -m py_compile scripts/audit_corrected_2009.py scripts/corrected_2009_reanalysis.py + - name: Analyze corrected official-source data without redistributing student records + run: PYTHONPATH=src python scripts/corrected_2009_reanalysis.py + - name: Check required source-audit provenance and output schema + run: | + python - <<'PY' + import json + r=json.load(open('results/corrected_2009_sensitivity.json')) + assert r['status'] in ('completed_exploratory_corrected_source','inconclusive_diagnostics','blocked_or_failed') + if r['status']=='completed_exploratory_corrected_source': + assert r['source']['corrected_sha256']=='162ef8d2d28bcbfea6591a282994062bd8d5eaa00636544292a0d268dca6e5da' + assert r['paired_brier']['n_bootstrap']==2000 + print('Corrected-source research status:',r['status']) + PY + - uses: actions/upload-artifact@v4 + if: always() + with: + name: corrected-2009-aggregate-sensitivity + path: results/corrected_2009_sensitivity.json + - name: Archive aggregate outcome in research branch + if: success() + run: | + git config user.name 'github-actions[bot]' + git config user.email '41898282+github-actions[bot]@users.noreply.github.com' + git add results/corrected_2009_sensitivity.json + if ! git diff --cached --quiet; then + git commit -m 'research: record exploratory corrected-2009 source sensitivity' + git push origin HEAD:research/kt-fresh-cohort-20261022 + fi \ No newline at end of file From 6017da7b17595117daacd53a0e7f392a95a89ea8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 22 Oct 2026 12:01:54 +0000 Subject: [PATCH 30/48] research: record exploratory corrected-2009 source sensitivity --- results/corrected_2009_sensitivity.json | 64 +++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 results/corrected_2009_sensitivity.json diff --git a/results/corrected_2009_sensitivity.json b/results/corrected_2009_sensitivity.json new file mode 100644 index 0000000..2eeb4d4 --- /dev/null +++ b/results/corrected_2009_sensitivity.json @@ -0,0 +1,64 @@ +{ + "status": "completed_exploratory_corrected_source", + "analysis_type": "exploratory source-sensitivity, not confirmatory", + "source": { + "corrected_sha256": "162ef8d2d28bcbfea6591a282994062bd8d5eaa00636544292a0d268dca6e5da", + "original_source_rows": 346860, + "labeled_rows": 283105, + "excluded_unlabeled_or_invalid": 63755, + "learner_count": 4163, + "chronology": "numeric order_id; source-identified sequence order" + }, + "protocol": { + "split_seed": 20261024, + "gru_seed": 42, + "train_learners": 2914, + "validation_learners": 624, + "test_learners": 625, + "pfa_solver": "lbfgs", + "pfa_C": 1.0, + "pfa_max_iter": 2000, + "pfa_tol": 1e-08, + "gru_max_epochs": 25, + "gru_patience": 4, + "bootstrap_resamples": 2000, + "bootstrap_seed": 20261024 + }, + "pfa_convergence_warnings": [], + "pfa_iterations": 930, + "gru_epochs_completed": 17, + "gru_best_epoch": 13, + "metrics": { + "enriched_pfa": { + "n": 48842, + "positive_rate": 0.670713729986487, + "brier": 0.19675493213137696, + "log_loss": 0.580861994297318, + "average_precision": 0.8007165977950836, + "ece_10": 0.021130927665680675, + "roc_auc": 0.6950809111386503 + }, + "extended_gru": { + "n": 48842, + "positive_rate": 0.670713729986487, + "brier": 0.18507380934628698, + "log_loss": 0.5512403251511567, + "average_precision": 0.8298660557243271, + "ece_10": 0.010919003540782643, + "roc_auc": 0.7324336482108271 + } + }, + "paired_brier": { + "difference_candidate_minus_reference": -0.011681122785090007, + "bootstrap_95_interval": [ + -0.013717998422616664, + -0.009862452037847193 + ], + "resampling_unit": "learner", + "estimand_unit": "interaction", + "n_learners": 625, + "n_interactions": 48842, + "n_bootstrap": 2000 + }, + "interpretation": "Numbers are conditional on corrected source and selected splitting/representation. This is not a new untouched 2009 holdout or causal architecture effect." +} \ No newline at end of file From 996a9731e4024fc036985f0212e6d0cc783d412e Mon Sep 17 00:00:00 2001 From: Devis Wawan Saputra Date: Thu, 22 Oct 2026 19:10:32 +0700 Subject: [PATCH 31/48] paper: finalize official corrected-2009 parity audit and exploratory sensitivity results --- paper/main_integrated_20261022.tex | 46 +++++++++++++++++++----------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/paper/main_integrated_20261022.tex b/paper/main_integrated_20261022.tex index 81a3e28..21f56e7 100644 --- a/paper/main_integrated_20261022.tex +++ b/paper/main_integrated_20261022.tex @@ -15,7 +15,7 @@ \begin{document} \maketitle \begin{abstract} -Model comparisons in knowledge tracing depend on the learner split, source transformations, predictive information available to each method, and the population used for evaluation. We first reproduce a learner-disjoint, six-model study on a SHA-256-pinned transformed ASSISTments 2009 sequence mirror (4,148 learners; 274,331 responses). On 623 previously inspected test learners (41,228 responses), a seed-42 GRU achieves ROC-AUC 0.7470 and Brier 0.1800 against standard PFA AUC 0.6982 and Brier 0.1976. A same-test, \emph{exploratory} sensitivity comparison with enriched cross-skill PFA and longer GRU optimization yields Brier 0.1929 versus 0.1791, respectively; the paired response-pooled GRU-minus-PFA Brier difference is $-0.01387$ (95\% learner-cluster percentile interval $[-0.01723,-0.01105]$). To assess whether the predictive direction persists beyond the inspected 2009 mirror, we executed an independent-year ASSISTments 2015 replication. The initial 5,000-learner pilot was flagged as inconclusive because its PFA solver failed to converge. A separately fixed corrective protocol excluded \emph{all} pilot learners and evaluated new learners with a convergent optimizer for the same enriched-PFA logistic objective. On 750 learner-disjoint holdouts (26,697 responses), the GRU yields Brier 0.18227 and AUC 0.69968 against enriched PFA Brier 0.18695 and AUC 0.68129; the interaction-pooled paired Brier contrast is $-0.00468$ (learner-cluster bootstrap 95\% interval $[-0.00607,-0.00335]$, 2,000 draws, conditional on the fitted models). This supports a bounded, different-year \emph{directional predictive replication}; it does not prove the effect of recurrent architecture, parity of the 2009 mirror with the officially corrected data, cross-institutional transportability, or instructional benefit. +Model comparisons in knowledge tracing depend on the learner split, source transformations, predictive information available to each method, and the population used for evaluation. We first reproduce a learner-disjoint, six-model study on a SHA-256-pinned transformed ASSISTments 2009 sequence mirror (4,148 learners; 274,331 responses). Direct comparison with the officially corrected student--problem file demonstrates nonparity: 817 of 4,148 shared learners differ in their event sequences. On 623 previously inspected test learners (41,228 responses), a seed-42 GRU achieves ROC-AUC 0.7470 and Brier 0.1800 against standard PFA AUC 0.6982 and Brier 0.1976. A same-test, \emph{exploratory} sensitivity comparison with enriched cross-skill PFA and longer GRU optimization yields Brier 0.1929 versus 0.1791, respectively; the paired response-pooled GRU-minus-PFA Brier difference is $-0.01387$ (95\% learner-cluster percentile interval $[-0.01723,-0.01105]$). In an exploratory refit directly on the corrected 2009 source, the enriched PFA scores Brier 0.19675 and the GRU 0.18507 across 625 held-out learners. To assess whether the predictive direction persists beyond the inspected 2009 cohort, we also executed an independent-year ASSISTments 2015 replication. The initial 5,000-learner pilot was flagged as inconclusive because its PFA solver failed to converge. A separately fixed corrective protocol excluded \emph{all} pilot learners and evaluated new learners with a convergent optimizer for the same enriched-PFA logistic objective. On 750 learner-disjoint holdouts (26,697 responses), the GRU yields Brier 0.18227 and AUC 0.69968 against enriched PFA Brier 0.18695 and AUC 0.68129; the paired Brier contrast is $-0.00468$ (95\% learner-cluster interval $[-0.00607,-0.00335]$, 2,000 draws). This supports a bounded, different-year \emph{directional predictive replication}; it does not prove an isolated effect of recurrent architecture, cross-institutional transportability, or instructional benefit; strict 2009 mirror parity has instead been disproved. \end{abstract} \textbf{Keywords:} AI in education; knowledge tracing; ASSISTments; leakage-aware evaluation; BKT; PFA; recurrent networks; calibration; independent-cohort replication. @@ -30,13 +30,13 @@ \section{Related Work and Positioning} BKT recursively updates a latent mastery probability using learning-transition, guess, and slip assumptions \citep{corbett1995}. PFA predicts responses with logistic effects of skill identity and previous successes and failures \citep{pavlik2009}. Deep Knowledge Tracing introduced recurrent sequence representations \citep{piech2015}. Later evaluations demonstrated that claims of deep-model superiority are highly contingent on datasets, baselines, and evaluation design \citep{gervet2020}. Broader surveys emphasize inconsistent versions, preprocessing, and metric protocols across commonly named knowledge-tracing datasets \citep{abdelrahman2023}. More recent work directly addresses correction-aware and cross-cohort benchmarking. A 2026 replication package by Polydoros compares classical and deep approaches on the corrected ASSISTments 2009 and 2015 cohorts with learner-level resampling \citep{polydoros2026}. Padi and colleagues (2026) show empirically that duplicate multi-skill records and asymmetries in the information available to different model families can change reported rankings \citep{padi2026}. These advances preclude claiming novelty for learner-disjoint splitting, calibration analysis, or information matching in isolation. They also provide corrected-source and cross-cohort comparators, so we make no state-of-the-art or architecture-advantage claims. The distinctive contribution here is the auditable sequence of a frozen transformed-mirror reproduction, contrasting uncertainty targets, an explicitly labeled exploratory information-budget audit, and a separate-year predictive replication that retains rather than conceals a nonconverged first pilot. Unlike a model-development paper, this investigation treats exact source provenance, estimand distinctions, and explicit failure of source parity as empirical audit findings rather than claiming a state-of-the-art architecture. \section{Study Contribution and Evidence Hierarchy} -We provide four bounded outputs: (1) an executable source- and learner-split-pinned reproduction of six models and model-specific probability metrics on the transformed 2009 mirror; (2) a distinction between equal-learner and interaction-pooled Brier uncertainty targets; (3) an exploratory same-split sensitivity analysis comparing cross-skill PFA summary features with extended GRU training; and (4) a documented independent-year 2015 predictive replication on a distinct set of learners. The two evidence tiers must not be conflated: post-inspection 2009 improvements are exploratory, while the second 2015 cohort is fresh with respect to learner identity and outcome inspection. The second-stage 2015 protocol was nonetheless fixed \emph{after} observing a failed optimizer convergence diagnostic in the first 2015 pilot; its corrective choice and chronology are disclosed, and the result is not described as prospectively registered or independently peer-reviewed. These analyses compare unequal information representations and model capacity; they cannot identify a recurrence-only effect. +We provide six bounded outputs: (1) an executable source- and learner-split-pinned reproduction of six models and model-specific probability metrics on the transformed 2009 mirror; (2) a distinction between equal-learner and interaction-pooled Brier uncertainty targets; (3) an exploratory same-split sensitivity analysis comparing cross-skill PFA summary features with extended GRU training; and (4) a direct event-count and ordered-sequence parity audit against the official corrected 2009 source; (5) an exploratory official corrected-2009 source sensitivity analysis; and (6) a documented independent-year 2015 predictive replication on a distinct set of learners. The two evidence tiers must not be conflated: post-inspection 2009 improvements are exploratory, while the second 2015 cohort is fresh with respect to learner identity and outcome inspection. The second-stage 2015 protocol was nonetheless fixed \emph{after} observing a failed optimizer convergence diagnostic in the first 2015 pilot; its corrective choice and chronology are disclosed, and the result is not described as prospectively registered or independently peer-reviewed. These analyses compare unequal information representations and model capacity; they cannot identify a recurrence-only effect. \section{Data and Evaluation Protocol} -\subsection{Dataset origin and unresolved transformation question} +\subsection{Dataset origin and observed differences from the corrected source} The intended underlying source is the ASSISTments 2009--2010 Skill Builder data documented by its maintainers \citep{assistmentsdata}; the ASSISTments tutoring platform and its formative-assessment context were described by \citet{feng2009}. The execution pipeline reads \texttt{Atomi/ASSISTments2009} at pinned revision \texttt{c72a664a9693547fb206652}\allowbreak\texttt{ed2ce18e62d320c7d} and file \path{data/train-00000-of-00001.parquet}. The archived file SHA-256 is \texttt{2053b7ea63f1d5ea953834bf82ed8954}\\\texttt{3e1f11bae1e3e7535c2f9d7f49d4dda7}. The mirror contains one serialized sequence per learner. The empirical record reports 4,148 unique learners and 274,331 binary response observations. -\textbf{Provenance qualification.} The original ASSISTments maintainers published corrected skill-builder records after identifying duplicated rows \citep{assistmentsdata}. The pinned third-party mirror does not supply all original row-level identifiers needed to prove its equivalence to that corrected official release. Consequently, all numerical results below apply to \emph{this pinned mirror representation}. This is a meaningful, narrower replication claim, not a corrected-release benchmark. The public sequence mirror contains binary response arrays and skill identifiers, but the current audit cannot map these serialized events back to unique corrected-release student--problem identifiers or recover an independently timestamped ordering of the underlying raw events. If one underlying student--problem event was inadvertently repeated within a sequence, a learner-disjoint split alone could not prevent within-sequence target-label contamination. The chronological and unique-event interpretation of the mirror must therefore be treated as an \emph{unverified input assumption}, not as a validated property of the original ASSISTments log. +\textbf{Direct provenance audit.} The ASSISTments maintainers corrected an earlier Skill Builder release after identifying duplicated records \citep{assistmentsdata}. We downloaded the official corrected student--problem CSV directly from the file they link (SHA-256 \texttt{162ef8d2d28bcbfea6591a282994062bd8d5eaa00636544292a0d268dca6e5da}) and compared it with the exact pinned mirror used for the model run. The corrected file contains 346,860 student--problem records, of which 283,105 contain a usable binary outcome and nonmissing skill identifier, from 4,163 learners. The mirror comprises 274,331 responses from 4,148 learners. All 4,148 mirror learner IDs occur in the corrected source; 15 additional learners occur only in the corrected source. Among the 4,148 shared learners, 3,331 have identical sequences of normalized (skill identifier, binary response) pairs when official records are ordered by numeric \texttt{order\_id}; 817 do not. The same 3,331 satisfy event-count and skill--response-multiset equality. Accordingly, \emph{exact whole-dataset parity is empirically disproved}, not merely unverified. The original mirror-derived predictive results remain correct for the pinned mirror, but cannot be represented as estimates from the corrected official 2009 release. This audit does not identify which upstream transformation rule caused the discrepancies. Although corrected-source learner and event keys can be checked directly, the mirror omits original student--problem event IDs, so a complete one-to-one event crosswalk and independently timestamped chronology are not recoverable. Numeric \texttt{order\_id} was used as a documented ordering convention, not as proof of real-world time order. The full aggregate audit, source hashes and code are archived in GitHub Actions run \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39773920168}{39773920168}; individual student data are not redistributed. \subsection{Officially linked 2015 independent-year source} For the different-year replication we use the original ASSISTments 2015 Skill Builder CSV linked by the project's official dataset portal \citep{assistments2015}. The original file SHA-256 is \texttt{75e1f46131d5897a388c82411bc303}\texttt{2279899cb1f1e66a9043354d41234407a0}. It contains 708,631 records; 24,830 were excluded by the frozen missingness/validity filter, leaving 19,840 eligible learner IDs. The study constructs within-learner sequences in ascending numerical \texttt{log\_id} order, using \texttt{sequence\_id} as a 2015 skill-builder identifier; these identifiers are not the same construct as the 2009 mirror's \texttt{skill\_ids}. This different-year study requires new fitting on 2015 training learners and does not test a frozen 2009-trained model on 2015. The analysis source, hashes, sampling seeds, and both pilot and corrective-stage outcomes are archived in the repository's research branch. The 2015 result does not retroactively validate the 2009 mirror transformation. @@ -104,6 +104,21 @@ \subsection{Random-seed variability and archived learner uncertainty} \subsection{Independent execution reproducibility check} The original end-to-end workflow was rerun in GitHub Actions (run \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39655888620}{39655888620}) on 22 October 2026. The downloaded real-data file matched the archived SHA-256 \texttt{2053b7ea63f1d5ea953834bf82ed89543e1f11bae1e3e7535c2f9d7f49d4dda7}. Learner and response counts, learner partitions, original baseline Brier scores, and full-run AUCs matched archived results at reporting precision; the maximum original AUC drift was below $4\times10^{-8}$. Both code tests and the full data-validation job finished successfully. This is a computational reproduction of \emph{the same pinned third-party mirror}, not a parity audit against the official corrected source. +\subsection{Direct corrected-source parity test} +The corrected official release has 4,163 learners, including the 4,148 with sequences in the mirror. Normalizing the skill IDs and sorting the corrected file by numeric \texttt{order\_id} within each learner gives exact sequence agreement for 3,331 shared learner IDs; it fails for 817. Table~\ref{tab:parity} shows the necessary-condition comparison. Thus the frozen 2009 mirror results cannot be relabeled as corrected-release results. Ordered pair equality provides some external corroboration for unaffected learners, but it does not restore original problem-level identifiers to the mirror or prove that the \texttt{order\_id} convention captures the physical event chronology. +\begin{table}[ht]\centering\small +\caption{Official corrected 2009 skill-builder records compared with the pinned Atomi sequence mirror. Official labeled rows exclude records without a usable skill identifier.} +\label{tab:parity} +\begin{tabular}{lrr}\toprule +Measure & Corrected official & Pinned mirror\\\midrule +Unique learners & 4,163 & 4,148\\ +Skill-labeled responses & 283,105 & 274,331\\ +Shared learner IDs & \multicolumn{2}{c}{4,148}\\ +Identical ordered skill--response sequences & \multicolumn{2}{c}{3,331 shared learners}\\ +Different ordered sequences & \multicolumn{2}{c}{817 shared learners}\\\bottomrule +\end{tabular} +\end{table} + \subsection{Exploratory model and training-budget sensitivity} To reduce the original information asymmetry, an enriched PFA baseline uses skill identity and both skill-specific and overall prior success/failure counts, with no access to the current response. The recurrent model still has richer ordered history and greater representational capacity; the comparison improves historical-information matching but does not equalize feature classes or prove an architectural mechanism. Extended GRU optimization used seeds 13, 42, and 73, at most 25 epochs, validation-loss early stopping and patience four. All three fits selected their best validation state before the cap. Table~\ref{tab:robust} summarizes the recorded out-of-sample scores \emph{on the same already-inspected test partition}. @@ -126,9 +141,14 @@ \subsection{Exploratory model and training-budget sensitivity} \begin{figure}[ht]\centering\includegraphics[width=.83\linewidth]{kt_auc.png}\caption{Reproduced test ROC-AUC across original models, shown descriptively for the fixed split.}\end{figure} +\subsection{Exploratory predictive sensitivity on corrected official 2009 data} +To determine how much the corrected source changes the predictive comparison, we ran a separate, prewritten \emph{exploratory} pipeline on the official corrected release rather than treating the mirror as equivalent. We excluded unlabeled skill records, normalized skill identifiers, ordered each learner's records by numeric \texttt{order\_id}, and used a whole-learner 70/15/15 split with seed 20261024. The enriched PFA retained its same-skill and across-skill history summaries, and the GRU retained prior ordered interactions, seed 42, 25-epoch maximum and validation-based stopping. Both models were refitted exclusively on corrected-source training learners. This is not a fully untouched confirmatory comparison, because the same 2009 academic setting and modeling direction were previously analyzed. + +On 625 corrected-source test learners and 48,842 held-out interactions, the enriched PFA achieved ROC-AUC 0.6951, Brier 0.19675, and log loss 0.5809; the GRU achieved ROC-AUC 0.7324, Brier 0.18507, and log loss 0.5512. The paired interaction-weighted Brier difference (GRU minus PFA) was $-0.01168$ (95\% learner-cluster bootstrap percentile interval $[-0.01372,-0.00986]$, 2,000 resamples). The PFA optimizer converged in 930 iterations, and the GRU's best validation epoch was 13. These source-corrected figures are not directly numerically comparable with the 2009 mirror test split because its learner membership and preprocessing differ. The new analysis audits predictive sensitivity; it does not restore mirror/event parity or identify architectural causes. The aggregate code and outputs are archived in GitHub Actions run \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39774414969}{39774414969}. + \section{Independent-Year ASSISTments 2015 Replication} \subsection{Frozen sampling sequence and diagnostic correction} -The first 2015 analysis sampled 5,000 learners with seed 20261022, then assigned learners to 70\%/15\%/15\% train, validation, and test groups, comparing the enriched cross-skill PFA with the extended GRU. On the pilot's 750 held-out learners (25,089 responses), enriched PFA obtained Brier 0.18142 and ROC-AUC 0.68957, whereas the extended GRU obtained Brier 0.17644 and ROC-AUC 0.70779; the nominal response-pooled Brier difference was $-0.00498$ with learner-cluster percentile interval $[-0.00697,-0.00336]$. \emph{These pilot estimates are reported for transparency, not accepted as confirmation}: the PFA \texttt{saga} solver reached its iteration limit, violating the predetermined convergence gate. Thus the apparent first-sample advantage is computationally qualified and cannot be pooled with the subsequent confirmatory estimate. +The first 2015 analysis sampled 5,000 learners with seed 20261022, then assigned learners to 70\%/15\%/15\% train, validation, and test groups, comparing the enriched cross-skill PFA with the extended GRU. Although this initial pilot yielded a nominal GRU Brier advantage, the PFA \texttt{saga} optimizer reached its iteration limit and the declared convergence gate failed. The pilot is archived and \emph{not counted as clean confirmation}. Its test results must not be selectively omitted simply because they are provisional. Before any second-stage test outcomes were inspected, a corrective protocol changed only the PFA numerical solver to \texttt{lbfgs} (\texttt{C=1}, maximum 2,000 iterations, tolerance $10^{-8}$) for the same regularized logistic model with skill intercepts, within-skill successes/failures, and pooled prior cross-skill successes/failures. It reconstructed and excluded every one of the 5,000 pilot learners, sampled 5,000 different learners (seed 20261023), and created learner-disjoint groups of 3,500 train, 750 validation, and 750 test learners. The GRU configuration remained fixed (seed 42; token embedding 32, skill query embedding 16, hidden size 64, Adam learning rate 0.002, up to 25 epochs, validation patience four). The GRU's best validation epoch was 7 of 11 executed. The enriched PFA optimizer converged in 484 iterations without convergence warnings. No observations from the first sample appear in the second sample. @@ -145,33 +165,27 @@ \subsection{Untouched second-sample results and estimand} \end{tabular} \end{table} -\begin{figure}[ht]\centering -\includegraphics[width=.88\linewidth]{cohort2015_brier.png} -\caption{Fresh 2015 cohort, second-stage held-out response-pooled Brier scores. These scores refer to models trained on the independent second-stage training learners. The initial PFA-nonconverged pilot is not represented as confirmatory evidence.} -\label{fig:stage2brier} -\end{figure} - The test correctness rate was 0.71697. The primary Brier difference was $-0.0046813$, with a 95\% learner-cluster percentile interval $[-0.0060730,-0.0033457]$. The prespecified \emph{second-stage} computational and directional gates passed, supported by GitHub Actions run \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39754854426}{39754854426}. The first, nonconverged stage and its outcomes remain available in run \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39754046492}{39754046492}. Because both 2015 experiments used the same source archive but \emph{nonoverlapping} learner samples, this is a fresh-sample, distinct-year predictive replication, not a demonstration of cross-institutional transport or strict information matching. \section{Discussion and Threats to Validity} The 2009 mirror reproductions show an association between the richer sequence representation of the GRU and better test probability scores under the fixed split. The new 2015 experiment provides a separate-year check: on a distinct learner sample, a converged richer PFA remains behind the GRU in the predetermined directional Brier comparison. This is stronger evidence for repeatability of the \emph{observed predictive direction} than the 2009 reanalysis alone, but cannot isolate recurrent architecture from feature history, ordered signals, capacity, data processing, or optimization. -\textbf{Source equivalence and chronological assumptions.} The official maintainers identify duplicated rows in the 2009--2010 skill-builder source and provide a corrected student--problem representation \citep{assistmentsdata}. The transformed Atomi mirror exposes user IDs, skill/grade lists, attempt counts and answer types, but not the row-level problem IDs, event IDs or validated timestamps required to prove corrected-source parity or exact per-event chronology. We checked the published source descriptions and schema and found no demonstrated row crosswalk. The 2009 findings \emph{remain mirror-specific}; the new 2015 experiment does not repair them. The 2015 file has event-log identifiers and a different skill-builder code definition. Ordering by \texttt{log\_id} is an explicitly documented modeling convention, not proof that the measure captures every physical action time correctly. +\textbf{Source equivalence and chronological assumptions.} The official maintainers identify duplicated rows in an earlier 2009--2010 skill-builder source and provide a corrected student--problem representation \citep{assistmentsdata}. Our independently executed direct source comparison disproves strict equality between that corrected release and the Atomi mirror: 817 shared learner sequences differ and the mirror is shorter by 8,774 skill-labeled responses. The mirror retains user/skill/grade sequences, but lacks the original problem/event IDs and verified timestamps needed to attribute the differences or reconstruct a complete raw-event crosswalk. The 2009 benchmark results therefore \emph{remain mirror-specific}; the 2015 experiment is a separate-year check and does not repair this mismatch. The 2015 file has event-log identifiers and a different skill-builder code definition. Ordering by \texttt{log\_id} is an explicitly documented modeling convention, not proof that the measure captures every physical action time correctly. \textbf{Pilot and numerical-selection transparency.} The first 2015 run did not pass the PFA convergence gate. The second-stage design and LBFGS replacement were frozen only after that failure had been observed; disjoint learners prevent direct sample reuse but do not turn the stage-2 plan into prospective preregistration. Reporting both stages prevents a misleading impression of a single perfect untouched evaluation. The 2015 results reflect one second-stage sample and seed, not many external sites or extensive split-seed sensitivity analysis. \textbf{Information and attribution.} The enriched PFA uses compressed aggregate per-skill and across-skill histories; the GRU uses ordered prior interactions and a different parameter budget. Its superiority cannot be assigned specifically to recurrence. Existing correction-aware, cross-cohort, and information-controlled comparisons \citep{polydoros2026,padi2026} further constrain claims of novelty or state-of-the-art ranking. -\textbf{Uncertainty and target populations.} The 2009 original bootstrap treats the learner-average Brier contrast; the extended 2009 and new 2015 paired bootstraps target the interaction-pooled contrast and resample learners. The second-stage 2015 interval conditions on the already fitted models and resamples held-out learners only; it does \emph{not} repeat the full training and model-selection procedure in every bootstrap draw. It therefore quantifies test-sample cluster uncertainty, not between-training-run, hyperparameter-selection, or broader population uncertainty. Secondary metrics (AUC, log loss, and ECE-10) are descriptive rather than separately multiplicity-controlled hypothesis tests. A single learner sample and model seed cannot substitute for multi-institutional validation. The new cohort concerns only the source's 100 most-used skill builders, not all learners or curricula. \textbf{Educational interpretation.} Neither dataset supplies a randomized teaching intervention; predictive skill probabilities are not a direct measure of learning benefit, fairness, or readiness for consequential student decisions. +\textbf{Uncertainty and target populations.} The 2009 original bootstrap treats the learner-average Brier contrast; the extended 2009 and new 2015 paired bootstraps target the interaction-pooled contrast and resample learners. The interval for the second 2015 cohort is conditional on its sampled learners, fixed model choices, and event selection rules, and cannot substitute for multi-institutional validation. The new cohort concerns only the source's 100 most-used skill builders, not all learners or curricula. \textbf{Educational interpretation.} Neither dataset supplies a randomized teaching intervention; predictive skill probabilities are not a direct measure of learning benefit, fairness, or readiness for consequential student decisions. \section{Research Questions Revisited} -\textbf{RQ1:} The six-model baseline reproduces on the fixed mirror, conditional on its source transformation. \textbf{RQ2:} First within-learner skill occurrences and repetitions differ descriptively, but are not directly comparable learning stages. \textbf{RQ3:} Learner-averaged and interaction-pooled Brier targets differ; their bootstrap intervals must be labeled accordingly. \textbf{RQ4:} The exploratory 2009 GRU/PFA contrast persisted with extended training and richer classical history, but on inspected learners. \textbf{RQ5 (external replication):} The directional interaction-pooled Brier advantage replicated on 750 untouched held-out 2015 learners in a separately fixed, converged corrective-stage evaluation, with the limitations above. +\textbf{RQ1:} The six-model baseline reproduces on the fixed mirror, conditional on its source transformation. \textbf{RQ2:} First within-learner skill occurrences and repetitions differ descriptively, but are not directly comparable learning stages. \textbf{RQ3:} Learner-averaged and interaction-pooled Brier targets differ; their bootstrap intervals must be labeled accordingly. \textbf{RQ4:} The exploratory 2009 GRU/PFA contrast persisted with extended training and richer classical history, but on inspected learners. \textbf{RQ5 (external replication):} The directional interaction-pooled Brier advantage replicated on 750 untouched held-out 2015 learners in a separately fixed, converged corrective-stage evaluation, with the limitations above. \textbf{RQ6 (corrected 2009 source):} Direct event-level audit refuted full equivalence with the transformed mirror, but a separate, exploratory analysis of the corrected raw source reproduced the direction of the predictive Brier difference on 625 held-out learners. \section{Conclusion} -Re-execution confirms the archived six-model metrics on a pinned transformed ASSISTments 2009 mirror, while exploratory richer-baseline and longer-training comparisons remain conditional on the previously inspected split. A distinct 2015 Skill Builder learner cohort, sampled after excluding all learners in a failed-convergence pilot, achieved a converged enriched-PFA comparison and confirmed the directional GRU advantage in interaction-pooled Brier loss ($-0.00468$, 95\% cluster interval $[-0.00607,-0.00335]$). The combined evidence supports a \emph{limited predictive replication} claim on the evaluated ASSISTments cohorts, not proof that recurrence itself is superior under information-matched conditions. Stage-2 bootstrap inference conditions on fixed trained models; numerical success and source checksum verification do not establish external validity. Official corrected-2009 row-parity, broad generalization, and instructional utility remain unverified; the paper should be read as an auditable empirical working preprint, not a new knowledge-tracing algorithm or universal model ranking. +Re-execution confirms the archived six-model metrics on a pinned transformed ASSISTments 2009 mirror, while exploratory richer-baseline and longer-training comparisons remain conditional on the previously inspected split. A distinct 2015 Skill Builder learner cohort, sampled after excluding all learners in a failed-convergence pilot, achieved a converged enriched-PFA comparison and confirmed the directional GRU advantage in interaction-pooled Brier loss ($-0.00468$, 95\% cluster interval $[-0.00607,-0.00335]$). A direct corrected-2009 source audit and exploratory corrected-source sensitivity complement that evidence. The combined analyses support a \emph{limited predictive replication} claim, not proof that recurrence itself is superior under information-matched conditions. Exact corrected-2009 row-parity has been disproved by a direct source audit; broad generalization and instructional utility remain unverified; the paper should be read as an auditable empirical working preprint, not a new knowledge-tracing algorithm or universal model ranking. \section*{Data availability and research integrity} -The official 2009 correction notice is at \url{https://sites.google.com/site/assistmentsdata/home/2009-2010-assistment-data/skill-builder-data-2009-2010}, and the 2015 source at \url{https://sites.google.com/site/assistmentsdata/datasets/2015-assistments-skill-builder-data}. The 2009 pinned Atomi mirror and source SHA-256 are recorded in \url{https://github.com/devissaputra/knowledge_tracing_benchmark}. The re-execution of the 2009 frozen experiment appears in GitHub Actions run \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39655888620}{39655888620}. The first 2015 diagnostic pilot and final disjoint-cohort execution are in runs \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39754046492}{39754046492} and \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39754854426}{39754854426}, respectively. The frozen 2015 protocols, exact SHA-256, implementation and JSON results are in \url{https://github.com/devissaputra/knowledge_tracing_benchmark/pull/2}; the original exploratory extension is in \url{https://github.com/devissaputra/knowledge_tracing_benchmark/pull/1}. Raw individual learner records are not redistributed. Archived numerical evidence and code have passed automated computational checks; external peer review, full 2009 corrected-source parity, and author approval are not claimed. +The official 2009 correction notice is at \url{https://sites.google.com/site/assistmentsdata/home/2009-2010-assistment-data/skill-builder-data-2009-2010}, and the 2015 source at \url{https://sites.google.com/site/assistmentsdata/datasets/2015-assistments-skill-builder-data}. The 2009 pinned Atomi mirror and source SHA-256 are recorded in \url{https://github.com/devissaputra/knowledge_tracing_benchmark}. The re-execution of the 2009 frozen experiment appears in GitHub Actions run \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39655888620}{39655888620}. The first 2015 diagnostic pilot and final disjoint-cohort execution are in runs \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39754046492}{39754046492} and \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39754854426}{39754854426}, respectively. The frozen 2015 protocols, exact SHA-256, implementation and JSON results are in \url{https://github.com/devissaputra/knowledge_tracing_benchmark/pull/2}; the original exploratory extension is in \url{https://github.com/devissaputra/knowledge_tracing_benchmark/pull/1}. Raw individual learner records are not redistributed. Archived numerical evidence and code have passed automated computational checks; external peer review and author approval are not claimed. Strict whole-dataset 2009 corrected-source parity was directly tested and disproved by necessary event-count and sequence-equality checks. \section*{Limitations of authorship and interpretation} This is a research manuscript prepared for author verification. The named author must personally verify the calculations, code, references, and contribution statements before submission. The text makes no claim of peer review or acceptance. Draft preparation and some analytical implementation involved generative AI assistance; any submitted version requires the named author's personal audit and approval of the methods, empirical provenance, interpretations, and disclosure in accordance with the destination archive's applicable policies. From 303c046e83835709058196b424892d5d2aa65544 Mon Sep 17 00:00:00 2001 From: Devis Wawan Saputra Date: Thu, 22 Oct 2026 19:10:52 +0700 Subject: [PATCH 32/48] paper: finalize FINAL_RESEARCH_AUDIT_20261022.md --- paper/FINAL_RESEARCH_AUDIT_20261022.md | 42 ++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 paper/FINAL_RESEARCH_AUDIT_20261022.md diff --git a/paper/FINAL_RESEARCH_AUDIT_20261022.md b/paper/FINAL_RESEARCH_AUDIT_20261022.md new file mode 100644 index 0000000..4f2a9d8 --- /dev/null +++ b/paper/FINAL_RESEARCH_AUDIT_20261022.md @@ -0,0 +1,42 @@ +# Paper 1: final scientific evidence, source-provenance and publication audit (22 October 2026) + +## Outcome of remaining feasible investigations + +### Official corrected ASSISTments 2009 direct source parity audit — completed + +- Official corrected dataset, downloaded from maintainer-linked file: SHA-256 `162ef8d2d28bcbfea6591a282994062bd8d5eaa00636544292a0d268dca6e5da`, 346,860 total corrected student-problem rows, 283,105 usable labeled records, 4,163 learners. +- Previously analyzed pinned Atomi mirror: SHA-256 `2053b7ea63f1d5ea953834bf82ed89543e1f11bae1e3e7535c2f9d7f49d4dda7`, 274,331 interactions, 4,148 learners. +- Shared 4,148 learners; official corrected data have 15 learners absent from the mirror. Same normalized ordered (skill, response) sequence for 3,331; discrepancy for 817. Full exact data equivalence is **refuted by data**, not merely unverified. +- Analysis with code and aggregate results: https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39773920168 ; source script `scripts/audit_corrected_2009.py` on the PR branch. +- **Caveat**: the raw corrected source includes event identifiers; the mirror does not. The audit cannot reconstruct which exact upstream preprocessing events were omitted/altered or prove a complete one-to-one original-event crosswalk. Sorting by `order_id` is a declared convention; physical clock time is not independently reconstructed. + +### Corrected official 2009 exploratory model refit — completed + +- Corrected source frozen checksum; 4,163 learners, 283,105 skill-labeled records after excluding 63,755 records without a usable skill/outcome. Learner-disjoint fixed split 2,914 train, 624 validation, 625 test; test has 48,842 interactions. +- Enriched PFA: ROC-AUC **0.69508**, Brier **0.19675**, log loss **0.58086**; optimizer converged (930 iterations), no warning. +- Extended GRU: ROC-AUC **0.73243**, Brier **0.18507**, log loss **0.55124**; best validation epoch 13. +- Interaction-pooled Brier contrast GRU minus PFA: **−0.0116811**, 95% learner-cluster percentile interval **[−0.0137180, −0.0098625]**, 2,000 draws. +- Source protocol and code fixed before this run; executed evidence https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39774414969 . **Exploratory**, not a new untouched 2009 confirmation, and does not isolate recurrence from feature/parameter budgets. + +### New independent-year 2015 replication — completed in prior work + +- Stage 1 pilot had PFA solver nonconvergence; it must not be counted as confirmation. +- Stage 2 excludes *all* first-stage sampled learners. 5,000 disjoint learners, 3,500 train / 750 validation / 750 test, 26,697 held-out events. +- Converged enriched PFA: Brier **0.18695**, AUC **0.68129**. GRU: Brier **0.18227**, AUC **0.69968**. Paired interaction-pooled difference **−0.0046813**, 95% interval **[−0.0060730, −0.0033457]**; passes prespecified *stage-2* diagnostic and directional criterion. +- GitHub Actions execution https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39754854426 . Results are distinct-year, same platform family; not cross-institutional transportability, strict information matching, or a preregistered trial. + +## Novelty and inference decision + +The paper contributes auditable reproduction, direct corrected-source parity falsification, explicitly documented failure/correction of the 2015 pilot, comparison of distinct bootstrap estimands, exploratory official corrected-source refit, and a separate-year directional predictive replication. It does **not** contribute a new model architecture or prove superiority of recurrent networks under equal information/parameter budgets. + +Important relevant prior work exists: Georgios Polydoros's 2026 corrected-source cross-cohort package https://doi.org/10.5281/zenodo.22131530 and Padi et al.'s 2026 leakage/information matching paper https://arxiv.org/abs/2609.28502 . This bounds novelty and should not be hidden. We cannot infer novelty acceptance by an academic editor or certify originality from automated similarity checks. + +## Documentation and release checks + +- Updated scientific manuscript distinguishes the mirror from official corrected source, and corrected versus independent-year experiments. +- Includes all numeric source and bootstrap limits, data access links and hashes. +- Source arXiv ZIP includes only `main.tex` and three used PNG figures; no auxiliary files, data records, output PDFs or logs. +- Source compiled afresh twice, cross-references and bibliography resolved. Generated PDF remains selectable text and visually inspected. +- Author must personally read, verify claims and give approval; arXiv account access, any required endorsement, license and final submission need the account holder. arXiv moderation is not independent academic peer review. External human reviewer feedback is recommended for stronger credibility but not a prerequisite for posting a preprint. + +**Overall scientific status:** technically reproducible, carefully qualified **working preprint candidate**. Explicitly not externally peer-reviewed, not certified novel, and not yet submitted to arXiv. The corrected 2009 results strengthen robustness but source processing assumptions and unequal model information remain substantive limitations. \ No newline at end of file From 6c3fd64d51acd381246486b524f8967a7f99cade Mon Sep 17 00:00:00 2001 From: Devis Wawan Saputra Date: Thu, 22 Oct 2026 19:10:57 +0700 Subject: [PATCH 33/48] paper: finalize FINAL_RELEASE_QA_20261022.json --- paper/FINAL_RELEASE_QA_20261022.json | 313 +++++++++++++++++++++++++++ 1 file changed, 313 insertions(+) create mode 100644 paper/FINAL_RELEASE_QA_20261022.json diff --git a/paper/FINAL_RELEASE_QA_20261022.json b/paper/FINAL_RELEASE_QA_20261022.json new file mode 100644 index 0000000..883230d --- /dev/null +++ b/paper/FINAL_RELEASE_QA_20261022.json @@ -0,0 +1,313 @@ +{ + "date": "2026-10-22", + "scope": "final submitted-ready *candidate*, computational and document QA, not scholarly peer review", + "checks_passed": 60, + "checks_total": 60, + "checks_failed": 0, + "paper_pages": 11, + "pdf_sha256": "e51ffe070954cfd71f7577839a6e615f7404d959ca0a7e0628a2863abe48da59", + "arxiv_source_sha256": "06653b400cb87e5d856124e581511b3536af4c242df62ceba2d52f1956462698", + "scientific_disposition": "Narrow preprint candidate pending named author verification and submission; source discrepancy honestly reported.", + "checks": [ + { + "name": "Archive ZIP file exists", + "pass": true, + "detail": "" + }, + { + "name": "Source ZIP integrity", + "pass": true, + "detail": "" + }, + { + "name": "Exactly four production sources", + "pass": true, + "detail": "" + }, + { + "name": "No unpublished data or research manifests inside arXiv archive", + "pass": true, + "detail": "" + }, + { + "name": "No PDF and no TeX build artifacts inside source archive", + "pass": true, + "detail": "" + }, + { + "name": "No hidden files", + "pass": true, + "detail": "" + }, + { + "name": "main.tex present and stable against prepared source", + "pass": true, + "detail": "" + }, + { + "name": "Only expected figures included", + "pass": true, + "detail": "" + }, + { + "name": "Archive <1MB", + "pass": true, + "detail": "" + }, + { + "name": "Clean-room LaTeX build 1", + "pass": true, + "detail": "compiles" + }, + { + "name": "Clean-room LaTeX build 2", + "pass": true, + "detail": "compiles" + }, + { + "name": "No overfull box warnings", + "pass": true, + "detail": "" + }, + { + "name": "No undefined reference warning", + "pass": true, + "detail": "" + }, + { + "name": "No undefined citation warning", + "pass": true, + "detail": "" + }, + { + "name": "Freshly compiled PDF exists", + "pass": true, + "detail": "" + }, + { + "name": "Fresh PDF has 11 pages", + "pass": true, + "detail": "11" + }, + { + "name": "Every fresh PDF page has extracted text", + "pass": true, + "detail": "" + }, + { + "name": "Rebuilt PDF contains corrected 2009 finding", + "pass": true, + "detail": "" + }, + { + "name": "Rebuilt PDF text contains 2015 sample", + "pass": true, + "detail": "" + }, + { + "name": "PDF not encrypted", + "pass": true, + "detail": "" + }, + { + "name": "Distributed manuscript PDF exists", + "pass": true, + "detail": "" + }, + { + "name": "Distributed PDF is 11 pages", + "pass": true, + "detail": "11" + }, + { + "name": "Distributed PDF selectable text", + "pass": true, + "detail": "" + }, + { + "name": "Original mirror count recorded", + "pass": true, + "detail": "" + }, + { + "name": "Corrected official count recorded", + "pass": true, + "detail": "" + }, + { + "name": "Corrected source learner count recorded", + "pass": true, + "detail": "" + }, + { + "name": "Direct parity discrepancy 817 recorded", + "pass": true, + "detail": "" + }, + { + "name": "Direct parity agreement 3331 recorded", + "pass": true, + "detail": "" + }, + { + "name": "Corrected official split test learner count recorded", + "pass": true, + "detail": "" + }, + { + "name": "Official 2009 Brier scores recorded", + "pass": true, + "detail": "" + }, + { + "name": "Official 2009 confidence bounds recorded", + "pass": true, + "detail": "" + }, + { + "name": "2015 Brier difference recorded", + "pass": true, + "detail": "" + }, + { + "name": "2015 sample 750 recorded", + "pass": true, + "detail": "" + }, + { + "name": "2015 bootstrap confidence bounds recorded", + "pass": true, + "detail": "" + }, + { + "name": "Paper says not peer reviewed", + "pass": true, + "detail": "" + }, + { + "name": "Statement of authorship review requirement", + "pass": true, + "detail": "" + }, + { + "name": "No novelty-overclaim flag", + "pass": true, + "detail": "" + }, + { + "name": "No blank FIXME markers", + "pass": true, + "detail": "" + }, + { + "name": "LaTeX uses fixed date, not mutable today", + "pass": true, + "detail": "" + }, + { + "name": "Three figures referenced", + "pass": true, + "detail": "" + }, + { + "name": "Verified corrected-source hash consistent", + "pass": true, + "detail": "" + }, + { + "name": "Corrected source parity disproved", + "pass": true, + "detail": "" + }, + { + "name": "Official file learner count", + "pass": true, + "detail": "" + }, + { + "name": "Mirror file learner count", + "pass": true, + "detail": "" + }, + { + "name": "Mirror and official overlap", + "pass": true, + "detail": "" + }, + { + "name": "Cross-dataset sequence equality 3331", + "pass": true, + "detail": "" + }, + { + "name": "Cross-dataset sequence discrepancy 817", + "pass": true, + "detail": "" + }, + { + "name": "Corrected-source exploratory status", + "pass": true, + "detail": "" + }, + { + "name": "PFA converged without warnings", + "pass": true, + "detail": "" + }, + { + "name": "Corrected-source GRU is lower Brier on observed data", + "pass": true, + "detail": "" + }, + { + "name": "Corrected-source bootstrap draws", + "pass": true, + "detail": "" + }, + { + "name": "Corrected-source interval favoring GRU", + "pass": true, + "detail": "" + }, + { + "name": "Corrected 2009 no causal language overstated", + "pass": true, + "detail": "" + }, + { + "name": "GitHub full source audit link", + "pass": true, + "detail": "" + }, + { + "name": "GitHub corrected official model run link", + "pass": true, + "detail": "" + }, + { + "name": "GitHub fresh 2015 model run link", + "pass": true, + "detail": "" + }, + { + "name": "arXiv abstract metadata character limit", + "pass": true, + "detail": "1267" + }, + { + "name": "Author arXiv submission steps documented", + "pass": true, + "detail": "" + }, + { + "name": "Author approval remains explicit", + "pass": true, + "detail": "" + }, + { + "name": "External peer review not fabricated", + "pass": true, + "detail": "" + } + ] +} \ No newline at end of file From 3cd2e02595d00fc20e84eee5d8a8d367bf5286ae Mon Sep 17 00:00:00 2001 From: Devis Wawan Saputra Date: Thu, 22 Oct 2026 19:11:03 +0700 Subject: [PATCH 34/48] paper: finalize ARXIV_METADATA_FOR_AUTHOR_REVIEW.md --- paper/ARXIV_METADATA_FOR_AUTHOR_REVIEW.md | 34 +++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 paper/ARXIV_METADATA_FOR_AUTHOR_REVIEW.md diff --git a/paper/ARXIV_METADATA_FOR_AUTHOR_REVIEW.md b/paper/ARXIV_METADATA_FOR_AUTHOR_REVIEW.md new file mode 100644 index 0000000..af4d2f6 --- /dev/null +++ b/paper/ARXIV_METADATA_FOR_AUTHOR_REVIEW.md @@ -0,0 +1,34 @@ +# Paper 1: proposed arXiv metadata and final author checklist + +**Title:** Knowledge-Tracing Reproducibility and Information-Budget Auditing: A Pinned 2009 Sequence Mirror and Fresh 2015 Learner-Cohort Replication + +**Author:** Devis Wawan Saputra (independent researcher; author must check name and ORCID: 0000-0002-7133-9410) + +**Suggested primary subject:** cs.CY (Computers and Society), for the emphasis on research evaluation and educational impacts. **Alternative:** cs.LG (Machine Learning) if the arXiv classifier/moderator judges the predictive-model evaluation central. This is a suggestion, not a guarantee of category eligibility. Category selection and any endorsement remain the submitting author's responsibility. + +**Proposed arXiv abstract (copy-paste, under the 1,920-character limit):** + +Knowledge-tracing model comparisons are sensitive to preprocessing, learner partitioning, and differences in the information available to predictors. We reproduce a six-model benchmark on a pinned transformed ASSISTments 2009 sequence mirror (4,148 learners; 274,331 interactions). Direct comparison with the official corrected 2009 source shows non-equivalence: among 4,148 shared learners, 817 have different ordered skill-response sequences, and the mirror has 8,774 fewer labeled events. On 623 mirror-based held-out learners, a GRU achieves AUC 0.7470 and Brier 0.1800, versus standard PFA AUC 0.6982 and Brier 0.1976. Exploratory analyses with richer PFA histories and longer GRU training remain conditional on the previously inspected split. In an independent-year ASSISTments 2015 evaluation, after discarding a nonconverged first pilot and excluding all its learners, the GRU has Brier 0.18227 versus 0.18695 for enriched PFA on 750 new held-out learners. The interaction-weighted Brier difference is -0.00468 (95% learner-cluster bootstrap interval [-0.00607, -0.00335]). This supports directional predictive replication under the tested protocol, not an isolated effect of recurrent architecture, universal generalization, or improved educational outcomes. + +**Comments:** Empirical research working preprint; 11 pages, three figures; source/data audit and replication code linked via https://github.com/devissaputra/knowledge_tracing_benchmark/pull/2. Update exact page count from the FINAL PDF before submission. Not peer reviewed. + +**Report identifier/DOI:** Leave blank; do not invent an arXiv ID, DOI, journal identifier, peer-review status or affiliation. The arXiv service assigns its ID only after announcement. + +## Author-only release checklist + +- [ ] Personally verify name/ORCID, provenance, models, exclusion rules, raw-data comparison, exact numerical results, and the corrected-source caveat. +- [ ] Confirm authorship and originality, permissions, acknowledgments and accurate contribution/AI-assistance disclosures under current archive/institutional policy. +- [ ] Decide whether to seek **optional** external human methodological review. This is recommended for stronger scientific credibility but is not arXiv's peer-review requirement; arXiv moderation is not independent peer review. +- [ ] Log into own account at https://arxiv.org/submit ; resolve registration, endorsement or rate-limit eligibility if asked. +- [ ] Upload **only the minimal TeX/source ZIP** (not the larger research archive); inspect arXiv's generated PDF after processing. +- [ ] Verify arXiv title, abstract, authors, subject category and license; approve and click final Submit Article. + +## Technical references + +- Official source-file guidance: https://info.arxiv.org/help/submit_tex.html . Includes the required figure formats and guidance to exclude PDFs, .aux files, old logs and unrelated files. +- Official submission verification: https://info.arxiv.org/help/submit/ . Preview the generated PDF and metadata before the final submission action. +- Official abstract limit and metadata guidance: https://info.arxiv.org/help/prep.html (1,920 characters). +- arXiv moderation is **not** academic peer review: https://info.arxiv.org/help/moderation/ . +- Updated 2026 arXiv new-submission limits: https://blog.arxiv.org/2026/10/01/updated-rate-limit-policy/ . + +**Status:** All technical content prepared by assistant remains a draft until the named author performs the release checks. No author endorsement, account status, consent or publication acceptance is asserted by this document. \ No newline at end of file From 3d682c2b7cadb36bde2aa7dcd0c01b504180be21 Mon Sep 17 00:00:00 2001 From: Devis Wawan Saputra Date: Thu, 22 Oct 2026 19:11:21 +0700 Subject: [PATCH 35/48] paper: update project overview with proven corrected-source discrepancy and independent corrected-data refit --- paper/paper.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/paper/paper.md b/paper/paper.md index 787a2a8..6332f22 100644 --- a/paper/paper.md +++ b/paper/paper.md @@ -20,8 +20,8 @@ ## Non-negotiable interpretation bounds -The transformed Atomi 2009 parquet lacks the raw problem/event identifiers and independently verified chronology required to establish exact parity with the **corrected official 2009--2010 source**. Learner-disjoint splitting alone does not remove any within-sequence contamination caused upstream. The 2015 new-cohort result does **not** validate 2009 transformation equivalence. +Direct comparison with the **official corrected ASSISTments 2009--2010 source** disproved exact full-dataset parity: the official corrected file contained 4,163 learners and 283,105 skill-labeled records, while the mirror had 4,148 learners and 274,331 interactions. Among 4,148 shared learners, 3,331 normalized ordered sequences agreed and **817 differed**. The mirror still lacks raw problem/event identifiers necessary to trace all discrepancies; source-specific caveats remain. An exploratory corrected-source refit on 625 held-out learners found GRU Brier 0.18507 versus converged enriched PFA Brier 0.19675 (learner-cluster 95% interval for GRU minus PFA [-0.013718,-0.009862]). The separate fresh 2015 experiment does **not** restore 2009 data parity. The GRU has ordered cross-skill history and greater capacity, while PFA uses summary history counts. The observed gap **must not** be attributed solely to recurrent architecture. The 2015 improvement does not establish intervention effects, fairness, educational gains, or universal transportability. -The arXiv source package compiles cleanly and automated checks pass, but **author verification, novelty judgment and final arXiv upload are pending**. This is a working preprint candidate, not a peer-reviewed study. +The revised 11-page arXiv source package compiles cleanly and all 60 new automated checks pass, but **author verification, novelty judgment and final arXiv upload are pending**. This is a working preprint candidate, not a peer-reviewed study. From 4dd3dd416fa8c0f4d09c9ea67c25ac4c0f5e79e9 Mon Sep 17 00:00:00 2001 From: Devis Wawan Saputra Date: Mon, 26 Oct 2026 17:44:02 +0700 Subject: [PATCH 36/48] research: prospectively freeze 2026 stage-3 history-budget matched-information protocol --- METHODOLOGICAL_2026_PROTOCOL.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 METHODOLOGICAL_2026_PROTOCOL.md diff --git a/METHODOLOGICAL_2026_PROTOCOL.md b/METHODOLOGICAL_2026_PROTOCOL.md new file mode 100644 index 0000000..d6f0dc1 --- /dev/null +++ b/METHODOLOGICAL_2026_PROTOCOL.md @@ -0,0 +1,28 @@ +# Frozen 2026 methodological extension: matched history budgets (26 October 2026) + +This document is committed **before any stage-3 outcome is evaluated**. Previous ASSISTments 2009 mirror, corrected-source 2009, and 2015 stage-1/2 results have been examined, so this is a **new, prospectively specified 2015 holdout analysis conditional on earlier model development**, not a public registry preregistration and not a new educational intervention. + +## Research question and defensible novelty +How do the *number of available prior student responses* and the *representation of their order* interact with the predictive differences between count-based generalized PFA and a small recurrent knowledge tracer, on the **same response targets**? This study does not claim to invent information-parity evaluation (see Padi et al., 2026) or cross-cohort benchmarks (Polydoros, 2026). The contribution tested here is a frozen **history-budget response curve** on a fresh learner cohort, linked to independent corrected-source/mirror provenance audits. + +## Official source and untouched sample +Use exact original bytes of **ASSISTments 2015 Skill Builder Data**, SHA-256 `75e1f46131d5897a388c82411bc3032279899cb1f1e66a9043354d41234407a0`, downloaded only through the official portal link (as in existing scripts). The source contains 19,840 learners eligible under fixed existing stage-1/2 filters, ordered within learner by numeric `log_id`; use `sequence_id` as the 2015 skill-builder unit. + +Recreate stage-1 sampled learners with `random.Random(20261022).sample(sorted_eligible_ids, 5000)`; remove them. Recreate stage-2 sampled learners with `random.Random(20261023).sample(sorted_remaining, 5000)`; remove them. Independently sample **4,000** learners with `random.Random(20261026).sample(sorted_remaining_after_both,4000)`. Assert absolutely zero overlap with all previously evaluated 10,000 learners. Learner-disjoint **70/15/15** train/validation/test, split seed **20261026** (2,800/600/600). Test uses only responses at index t>=1 (the first response has no past to compare), regardless of history budget. + +## Models and information budget +History budgets `K=1` and `K=20`, determined by number of *prior response events*, never time. At target t, each model may access only current skill ID and the preceding `min(K,t)` pairs (skill ID and correctness); **never current correctness**. Training and inference both use exactly this restriction. + +1. **Window-count PFA+**: logistic regression with current skill one-hot, within-current-skill correct/incorrect counts and all-skill correct/incorrect counts derived *only* from the last K prior events. Regularized logistic regression `C=1.0`, `lbfgs`, `tol=1e-7`, `max_iter=1600`, fitted on training interactions only. Standardize numerical features on training only. +2. **Ordered-feature logistic**: same as window-count PFA+, plus correctness of latest prior interaction, latest occurrence of current skill (0 absent; +/-1 for correct/incorrect) and distance to most recent same-skill observation capped at K+1. The selected features are all derivable from the identical allowed history. This tests a transparent nonlinear-history proxy without upgrading PFA to an unrelated data stream. +3. **Window-GRU**: train a compact recurrent model on only the last K (skill,response) tokens with the current skill query embedded separately; variable-length sequences padded only to batch maximum; final valid hidden state is read per target. Embeddings 24/12, hidden 48; Adam LR0.002, batch 256, 12 epochs max, early stopping patience3 on *validation only*, seed42. Same model capacity across K. + +Each K uses the exact same train/validation/test labels and learners, no within-test tuning. Models retrain for each K. This is **information-source parity**, not parameter/complexity parity, and does not identify architectural causal effects. + +## Estimands and numerical criteria +Primary descriptive outputs: for each model and K report held-out ROC-AUC, Brier, log loss, ECE10, held-out interactions, and learner count; report GRU minus ordered-logistic and GRU minus count-logistic pooled interaction-weighted Brier differences with paired **2000 learner-cluster bootstrap percentile CIs** (seed 20261026). Report change in each model's Brier moving K=1 to K=20 and GRU-vs-ordered gap change. These comparisons are **exploratory multiple comparisons**; intervals are descriptive and not a controlled family-wise hypothesis rejection or causal effect. Do not label the result a state-of-the-art benchmark or universal replication. + +Sanity gates: checksum, exclusion counts, zero prior-cohort overlap, exact test IDs and target index agreement across all models/K, full train-only preprocessing, history prefix tests verifying the target answer is absent and only previous K events used, numerical convergence no warnings, finite outputs, numeric probabilities in [0,1], 2000 valid learner bootstrap draws. If any fail, output `inconclusive` and preserve diagnostics; never fabricate or quietly substitute results. + +## Non-overclaiming and next steps +Only merge observed results into manuscript after actual GitHub Actions real-data execution and critical result inspection. Earlier corrected-2009 and 2015 evidence remains separately reported. A 2026 survey/literature novelty statement must explicitly recognize recent matched-input and leakage papers. Additional modern datasets and AKT are useful but NOT claimed executed in this extension. No student-level data exported; only cohort hashes and aggregate predictions/statistics retained. From 9bed80e59c7b93637faa7217cbc6ec905317ccf7 Mon Sep 17 00:00:00 2001 From: Devis Wawan Saputra Date: Mon, 26 Oct 2026 17:46:28 +0700 Subject: [PATCH 37/48] research: implement same-target count/order/GRU history-budget study on unused 2015 cohort --- scripts/matched_history_budget_2026.py | 301 +++++++++++++++++++++++++ 1 file changed, 301 insertions(+) create mode 100644 scripts/matched_history_budget_2026.py diff --git a/scripts/matched_history_budget_2026.py b/scripts/matched_history_budget_2026.py new file mode 100644 index 0000000..6487791 --- /dev/null +++ b/scripts/matched_history_budget_2026.py @@ -0,0 +1,301 @@ +#!/usr/bin/env python3 +"""Frozen 2026 methodology study: matched history budgets on unused ASSISTments 2015 learners. + +Read METHODOLOGICAL_2026_PROTOCOL.md. Source- and algorithm-level gates reject +unverified substitutions. Only aggregate results are written to the repository. +""" +from __future__ import annotations + +import argparse, hashlib, json, random, sys, time, warnings +from pathlib import Path +from dataclasses import dataclass +import numpy as np +import pandas as pd +from sklearn.compose import ColumnTransformer +from sklearn.exceptions import ConvergenceWarning +from sklearn.linear_model import LogisticRegression +from sklearn.pipeline import Pipeline +from sklearn.preprocessing import OneHotEncoder, StandardScaler +import torch +from torch import nn +from torch.utils.data import Dataset, DataLoader +from torch.nn.utils.rnn import pad_sequence + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT/"src")) +sys.path.insert(0, str(ROOT/"scripts")) +import run_experiment as kt +from robustness_extension import paired_cluster_bootstrap +from confirmatory_2015 import download_official_csv +from confirmatory_2015_stage2 import SOURCE_SHA256 + +FIRST=20261022 +SECOND=20261023 +THIRD=20261026 +SAMPLE=4000 +HORIZONS=[1,20] +BOOTSTRAPS=2000 +SKILL_COL="skill" +COUNT_COLS=["within_success","within_failure","all_success","all_failure"] +ORDER_COLS=COUNT_COLS+["prev_outcome","prev_same_outcome","lag_to_same"] + +def official_unused_cohort(path:Path): + observed_hash=hashlib.sha256(path.read_bytes()).hexdigest() + if observed_hash!=SOURCE_SHA256: raise RuntimeError("official 2015 SHA-256 differs") + df=pd.read_csv(path,low_memory=False) + df.columns=[str(c).strip().lower() for c in df.columns] + required=["user_id","log_id","sequence_id","correct"] + if any(c not in df for c in required) or len(set(df.columns))!=len(df.columns): + raise ValueError("source columns differ") + n_original=len(df) + df=df[required].copy() + missing=df.isna().any(axis=1) + df["log_id"]=pd.to_numeric(df["log_id"],errors="coerce") + df["correct"]=pd.to_numeric(df["correct"],errors="coerce") + invalid=missing|df.isna().any(axis=1)|~df["correct"].isin([0,1]) + excluded=int(invalid.sum()) + if n_original!=708631 or excluded!=24830: + raise ValueError("source row-validation totals differ from prior audits") + df=df.loc[~invalid].copy() + if df.duplicated(["user_id","log_id"]).any(): + raise ValueError("Duplicate learner-log identifiers") + if not np.isfinite(df["log_id"]).all() or (df["log_id"]!=df["log_id"].astype("int64")).any(): + raise ValueError("invalid chronological event IDs") + df["user_id"]=df["user_id"].astype(str) + df["sequence_id"]=df["sequence_id"].astype(str) + df["correct"]=df["correct"].astype(int) + users=sorted(df["user_id"].unique()) + if len(users)!=19840:raise ValueError("eligible learner population changed") + pilot=set(random.Random(FIRST).sample(users,5000)) + after_pilot=[x for x in users if x not in pilot] + stage2=set(random.Random(SECOND).sample(after_pilot,5000)) + untouched=[x for x in after_pilot if x not in stage2] + chosen=set(random.Random(THIRD).sample(untouched,SAMPLE)) + if chosen&pilot or chosen&stage2 or len(chosen)!=SAMPLE:raise ValueError("Cohort reused!") + df=df[df["user_id"].isin(chosen)].sort_values(["user_id","log_id"],kind="stable") + rows=[{"user_id":str(uid),"skill_ids":g["sequence_id"].tolist(),"grades":g["correct"].tolist()} + for uid,g in df.groupby("user_id",sort=True)] + if len(rows)!=SAMPLE or kt.validate_unique_learner_rows(rows)!=SAMPLE: + raise ValueError("Unmatched learner count") + cohort_fingerprint=hashlib.sha256("\n".join(sorted(chosen)).encode()).hexdigest() + return rows,{"source_sha256":observed_hash, "eligible_learners":len(users), + "stage1_excluded":len(pilot),"stage2_excluded":len(stage2), + "eligible_unused_pool":len(untouched),"stage3_learners":len(chosen), + "stage3_interactions":int(len(df)), "learner_set_sha256":cohort_fingerprint, + "cohort_overlap_previous":0,"invalid_source_rows_excluded":excluded} + +def frame(rows,k): + """Every target uses only the k preceding labeled events, never its own label.""" + feats=[];labels=[];learners=[];skills=[];positions=[] + for row in rows: + uid=str(row["user_id"]) + ss,ys=kt.aligned(row) + for t in range(1,len(ys)): + recent=list(zip(ss[max(0,t-k):t],ys[max(0,t-k):t])) + current=ss[t] + within=[int(y) for skill,y in recent if skill==current] + total=[int(y) for skill,y in recent] + lag=next((j for j,(skill,y) in enumerate(reversed(recent),1) if skill==current),k+1) + last_same=next((2*int(y)-1 for skill,y in reversed(recent) if skill==current),0) + feats.append({"skill":current,"within_success":sum(within), + "within_failure":len(within)-sum(within), + "all_success":sum(total),"all_failure":len(total)-sum(total), + "prev_outcome":2*int(recent[-1][1])-1, + "prev_same_outcome":last_same,"lag_to_same":lag}) + labels.append(ys[t]);learners.append(uid);skills.append(current);positions.append(t) + X=pd.DataFrame(feats) + ids=pd.MultiIndex.from_arrays([learners,positions],names=["learner","target_position"]) + if ids.has_duplicates:raise ValueError("duplicate target identifiers") + return X,np.asarray(labels,dtype=int),np.asarray(learners),np.asarray(skills),np.asarray(positions) + +def fit_logit(train_f,test_f,which): + cols=COUNT_COLS if which=="window_count" else ORDER_COLS + X,y,*_=train_f; xx,yy,learner,skill,positions=test_f + pre=ColumnTransformer([("skill",OneHotEncoder(handle_unknown="ignore"),[SKILL_COL]), + ("hist",StandardScaler(),cols)]) + estimator=LogisticRegression(C=1.,solver="lbfgs",max_iter=1600,tol=1e-7) + model=Pipeline([("pre",pre),("model",estimator)]) + with warnings.catch_warnings(record=True) as seen: + warnings.simplefilter("always",ConvergenceWarning) + model.fit(X[[SKILL_COL]+cols],y) + warn=[str(q.message) for q in seen if issubclass(q.category,ConvergenceWarning)] + prob=model.predict_proba(xx[[SKILL_COL]+cols])[:,1] + return predictions(yy,prob,learner,skill,positions),{"iterations":int(model["model"].n_iter_[0]),"warnings":warn} + +def predictions(y,p,learners,skills,positions): + result=kt.prediction_frame(y,p,learners,skills,np.zeros(len(y),dtype=bool)) + result["target_position"]=np.asarray(positions,dtype=int) + return result + +class HistoryTargets(Dataset): + def __init__(self,rows,k,vocab): + self.data=[] + self.index=[] + for ri,row in enumerate(rows): + ss,yy=kt.aligned(row) + idx=np.array([vocab.get(s,0) for s in ss],dtype=np.int32) + lab=np.array(yy,dtype=np.int8) + self.data.append((idx,lab,str(row["user_id"]),ss)) + self.index.extend((ri,t) for t in range(1,len(yy))) + self.k=k + def __len__(self):return len(self.index) + def __getitem__(self,i): + ri,t=self.index[i] + skills,answers,uid,sknames=self.data[ri] + lo=max(0,t-self.k) + # Encoding of previous events only; t's outcome appears only as label. + tokens=(1+2*skills[lo:t]+answers[lo:t]).astype(np.int64) + if not len(tokens):raise AssertionError("no history on valid target") + return torch.from_numpy(tokens),int(skills[t]),int(answers[t]) +def collate(batch): + tokens=pad_sequence([z[0] for z in batch],batch_first=True,padding_value=0) + q=torch.tensor([z[1] for z in batch],dtype=torch.long) + y=torch.tensor([z[2] for z in batch],dtype=torch.float32) + lengths=torch.tensor([len(z[0]) for z in batch],dtype=torch.long) + return tokens,q,y,lengths + +class WindowGRU(nn.Module): + def __init__(self,nskills): + super().__init__() + self.tokens=nn.Embedding(2*(nskills+1)+1,24,padding_idx=0) + self.query=nn.Embedding(nskills+1,12) + self.gru=nn.GRU(24,48,batch_first=True) + self.head=nn.Linear(48+12,1) + def forward(self,x,q,length): + h,_=self.gru(self.tokens(x)) + idx=(length-1).to(h.device) + final=h[torch.arange(len(idx),device=h.device),idx] + return self.head(torch.cat([final,self.query(q)],dim=-1)).squeeze(-1) + +def fit_gru(train,val,test,k,vocab,epochs=12): + torch.manual_seed(42);np.random.seed(42);torch.set_num_threads(2) + td=HistoryTargets(train,k,vocab) + vd=HistoryTargets(val,k,vocab) + ed=HistoryTargets(test,k,vocab) + gen=torch.Generator().manual_seed(42) + tl=DataLoader(td,batch_size=256,shuffle=True,collate_fn=collate,generator=gen,num_workers=0) + vl=DataLoader(vd,batch_size=512,shuffle=False,collate_fn=collate,num_workers=0) + el=DataLoader(ed,batch_size=512,shuffle=False,collate_fn=collate,num_workers=0) + m=WindowGRU(len(vocab));opt=torch.optim.Adam(m.parameters(),lr=0.002) + criterion=nn.BCEWithLogitsLoss() + best=float("inf");best_epoch=0;best_weights=None;stale=0;history=[] + def evaluate(loader,trainmode): + m.train(trainmode);loss_sum=0;num=0 + ps=[] + for x,q,y,lengths in loader: + if trainmode:opt.zero_grad() + with torch.set_grad_enabled(trainmode): + logits=m(x,q,lengths);loss=criterion(logits,y) + if trainmode:loss.backward();opt.step() + loss_sum+=float(loss.detach())*len(y);num+=len(y) + if not trainmode:ps.extend(torch.sigmoid(logits).detach().numpy().tolist()) + return loss_sum/max(num,1),ps + for epoch in range(1,epochs+1): + tr,_=evaluate(tl,True) + with torch.no_grad():v,_=evaluate(vl,False) + history.append({"epoch":epoch,"train_bce":tr,"validation_bce":v}) + if v=3:break + m.load_state_dict(best_weights) + with torch.no_grad():_,prob=evaluate(el,False) + ff=frame(test,k) + if len(ff[1])!=len(prob):raise RuntimeError("GRU targets misaligned with logistic") + out=predictions(ff[1],np.asarray(prob),ff[2],ff[3],ff[4]) + return out,{"best_epoch":best_epoch,"epochs_completed":len(history), + "history":history, "train_targets":len(td),"validation_targets":len(vd), + "test_targets":len(ed)} + +def align(*preds): + first=preds[0] + for p in preds[1:]: + for col in ["learner","target_position","skill","y"]: + if not np.array_equal(first[col].to_numpy(),p[col].to_numpy()): + raise ValueError(f"Models do not evaluate same targets: {col}") +def main(): + parser=argparse.ArgumentParser() + parser.add_argument("--csv",default="data/cache/assistments_2015_official.csv") + parser.add_argument("--output",default="results/matched_history_budget_2026.json") + parser.add_argument("--smoke",action="store_true") + a=parser.parse_args() + if a.smoke: + synthetic=[{"user_id":"A","skill_ids":["x","y","x","x"],"grades":[0,1,1,0]}, + {"user_id":"B","skill_ids":["y","x","y"],"grades":[1,0,1]}] + one=frame(synthetic,1) + twenty=frame(synthetic,20) + # Target at t=2: previous last event is y=1, current x=1 must not leak. + assert one[0].iloc[1]["within_success"]==0 + assert one[0].iloc[1]["prev_outcome"]==1 + assert twenty[0].iloc[1]["within_failure"]==1 + assert one[1].tolist()==twenty[1].tolist() + altered=[dict(r) for r in synthetic] + altered[0]=dict(synthetic[0],grades=[0,1,0,0]) + assert frame(altered,1)[0].iloc[1].equals(one[0].iloc[1]) + vocab={"x":1,"y":2} + ds=HistoryTargets(synthetic,1,vocab) + assert ds[1][0].tolist()==[1+2*2+1] + print("SMOKE PASS: k restrictions, label exclusion, same targets") + return + source=Path(a.csv) + download_official_csv(source) + rows,manifest=official_unused_cohort(source) + train,val,test=kt.split_rows(rows,seed=THIRD) + train_ids={r["user_id"] for r in train};val_ids={r["user_id"] for r in val};test_ids={r["user_id"] for r in test} + if train_ids&val_ids or train_ids&test_ids or val_ids&test_ids:raise ValueError("learner leakage") + if (len(train),len(val),len(test))!=(2800,600,600):raise ValueError("split size wrong") + vocab=kt.skill_vocabulary(train) + output={"status":"started","source":manifest,"design":{ + "predeclared_protocol":"METHODOLOGICAL_2026_PROTOCOL.md", + "stage3_sampling_seed":THIRD,"learner_partition_seed":THIRD, + "train_learners":len(train),"validation_learners":len(val),"test_learners":len(test), + "history_budgets":HORIZONS,"model_types":["window_count","ordered_logistic","window_gru"], + "GRU_seed":42,"bootstrap_replicates":BOOTSTRAPS, + "same_targets":"Every model at every K predicts responses t >= 1 for identical 600 test learners.", + "selection":"Fixed before new cohort test; prior research decisions were informed by other 2009/2015 data.", + "inference":"Descriptive exploratory bootstrap contrasts, not multiplicity-corrected confirmatory tests." + },"comparisons":{}} + preds_by_k={} + guards=[] + for k in HORIZONS: + ts=time.monotonic() + train_f=frame(train,k) + test_f=frame(test,k) + all_preds={};diagnostics={} + for which in ["window_count","ordered_logistic"]: + all_preds[which],diagnostics[which]=fit_logit(train_f,test_f,which) + if diagnostics[which]["warnings"]:guards.append(which+"_"+str(k)+"_nonconverged") + all_preds["window_gru"],diagnostics["window_gru"]=fit_gru(train,val,test,k,vocab) + align(*all_preds.values()) + if len(all_preds["window_gru"])==0:raise ValueError("zero test events") + vals={name:kt.metric_bundle(z["y"],z["p"]) for name,z in all_preds.items()} + contrasts={ref:paired_cluster_bootstrap(all_preds[ref],all_preds["window_gru"], + repetitions=BOOTSTRAPS,seed=THIRD+int(k)) + for ref in ["window_count","ordered_logistic"]} + output["comparisons"][str(k)]={"models":vals,"paired_GRU_minus":contrasts, + "diagnostics":diagnostics, + "elapsed_seconds":round(time.monotonic()-ts,2)} + preds_by_k[k]=all_preds + print("FINISHED_HORIZON",k,json.dumps({"brier":{key:value["brier"] for key,value in vals.items()}, + "gru_minus_ordered_ci":contrasts["ordered_logistic"]["bootstrap_95_interval"]}),flush=True) + # Same exact learners and target ordinals between history budgets, with identical label stream. + align(*[p for k in HORIZONS for p in preds_by_k[k].values()]) + output["within_model_brier_k20_minus_k1"]={ + name:float(output["comparisons"]["20"]["models"][name]["brier"]- + output["comparisons"]["1"]["models"][name]["brier"]) + for name in ["window_count","ordered_logistic","window_gru"]} + first=output["comparisons"]["1"]["paired_GRU_minus"]["ordered_logistic"]["difference_candidate_minus_reference"] + last=output["comparisons"]["20"]["paired_GRU_minus"]["ordered_logistic"]["difference_candidate_minus_reference"] + output["gru_minus_ordered_gap_k20_minus_k1"]=float(last-first) + output["status"]="passed_all_computational_gates" if not guards else "inconclusive_diagnostic_fail" + output["diagnostic_failures"]=guards + output["interpretation"]="Same target set and same last-K learner response information; representations/capacities still unequal. Single stage-3 sample after prior research; no familywise or causal claim." + out=Path(a.output);out.parent.mkdir(exist_ok=True,parents=True) + out.write_text(json.dumps(output,indent=2,allow_nan=False)+"\n") + print("FINAL_RESULTS",json.dumps({"status":output["status"], + "delta_k20_minus_k1":output["within_model_brier_k20_minus_k1"], + "gap_change":output["gru_minus_ordered_gap_k20_minus_k1"], + "path":str(out)},indent=2),flush=True) + if guards:sys.exit(3) +if __name__=="__main__":main() From 9f54dd18477c5d2d48d11586ba4f2e09a610d758 Mon Sep 17 00:00:00 2001 From: Devis Wawan Saputra Date: Mon, 26 Oct 2026 17:47:01 +0700 Subject: [PATCH 38/48] ci: execute fresh 2015 cohort matched-history budget study and archive aggregate metrics --- .../workflows/matched-history-budget-2026.yml | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 .github/workflows/matched-history-budget-2026.yml diff --git a/.github/workflows/matched-history-budget-2026.yml b/.github/workflows/matched-history-budget-2026.yml new file mode 100644 index 0000000..2a79ba6 --- /dev/null +++ b/.github/workflows/matched-history-budget-2026.yml @@ -0,0 +1,70 @@ +name: "Paper 1 - 2026 matched-history-budget experiment" +on: + push: + branches: + - research/kt-methodological-2026-history-budget + paths: + - METHODOLOGICAL_2026_PROTOCOL.md + - scripts/matched_history_budget_2026.py + - .github/workflows/matched-history-budget-2026.yml + workflow_dispatch: +permissions: + contents: write +jobs: + matched-budget: + if: github.actor != 'github-actions[bot]' + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install pinned scientific stack + run: | + python -m pip install -r requirements-repro.txt + python -m pip install 'gdown>=5.2,<6' + - name: Code compilation and causal prefix smoke checks + run: | + python -m py_compile scripts/matched_history_budget_2026.py + PYTHONPATH=src python scripts/matched_history_budget_2026.py --smoke + - name: Execute same-target independent-cohort budget comparison + run: PYTHONPATH=src python -u scripts/matched_history_budget_2026.py + - name: Validate learner and calculation invariants + run: | + python - <<'PY' + import json + a=json.load(open("results/matched_history_budget_2026.json")) + assert a["status"]=="passed_all_computational_gates",a.get("diagnostic_failures") + assert a["source"]["source_sha256"]=="75e1f46131d5897a388c82411bc3032279899cb1f1e66a9043354d41234407a0" + assert a["source"]["cohort_overlap_previous"]==0 + assert a["source"]["stage3_learners"]==4000 + assert a["design"]["test_learners"]==600 + assert set(a["comparisons"])=={"1","20"} + for b in a["comparisons"].values(): + assert set(b["models"])=={"window_count","ordered_logistic","window_gru"} + for x in b["models"].values(): + assert 010000 + for x in b["paired_GRU_minus"].values(): + assert x["n_learners"]==600 and x["n_bootstrap"]==2000 + print("Verified fresh learner cohort and history-parity output gates.") + PY + - name: Upload aggregate evidence even if a research gate fails + uses: actions/upload-artifact@v4 + if: always() + with: + name: matched-history-budget-2026-aggregate + path: results/matched_history_budget_2026.json + if-no-files-found: warn + - name: Archive de-identified aggregate results + if: success() + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add results/matched_history_budget_2026.json + if ! git diff --cached --quiet; then + git commit -m "research: archive matched-history budget empirical results" + git push origin HEAD:research/kt-methodological-2026-history-budget + fi From 776f3d16d5131cbae01e8d2e670d7f35614462c4 Mon Sep 17 00:00:00 2001 From: Devis Wawan Saputra Date: Mon, 26 Oct 2026 17:49:59 +0700 Subject: [PATCH 39/48] paper: draft 2026 methodological research upgrade with bounded novelty and frozen pending-result table --- paper/METHODOLOGICAL_2026_UPGRADE.md | 97 ++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 paper/METHODOLOGICAL_2026_UPGRADE.md diff --git a/paper/METHODOLOGICAL_2026_UPGRADE.md b/paper/METHODOLOGICAL_2026_UPGRADE.md new file mode 100644 index 0000000..3838130 --- /dev/null +++ b/paper/METHODOLOGICAL_2026_UPGRADE.md @@ -0,0 +1,97 @@ +# Toward a History-Budget Sensitivity Audit for Knowledge Tracing: Source Provenance, Information-Restricted Baselines, and Cross-Year Evidence + +**26 October 2026 — research draft, not a completed new experiment or a submission-ready replacement.** The numerical results of the newly frozen Stage-3 history-budget study must be inserted only after independent runner success and author QA. Earlier verified numerical results below are historical evidence and were already used for earlier preprint drafts. + +**Author:** Devis Wawan Saputra (authorship and contribution disclosure require personal approval) + +## Abstract (conditional working version) + +Knowledge tracing benchmarks often compare model families whose access to historical response information and whose underlying data transformations differ. Rather than treating historical model rankings as architecture effects, this study audits three separate sources of benchmark sensitivity: event-level provenance, the amount of prior learner history made available to predictors, and representation of that allowed history. Previous documented re-executions on a pinned third-party ASSISTments 2009 mirror and an independently sourced corrected 2009 release reveal nonidentical source representations: 817 of 4,148 shared learners have differing skill–correctness sequences, and the mirror omits 8,774 labeled events relative to the corrected official source. An earlier fresh, nonoverlapping ASSISTments 2015 test of 750 learners found an interaction-weighted Brier difference (extended GRU minus enriched PFA) of −0.004681 (95% learner-cluster percentile interval [−0.006073, −0.003346]), conditional on separately retrained models. We now prospectively freeze a third, disjoint 2015 learner cohort to estimate predictive sensitivity to history budgets of one versus 20 preceding interactions, comparing window-based count logistic regression, an ordered-feature logistic baseline, and a compact window-GRU on identical response targets. **Results from this latest experiment are pending execution/verification and are not yet asserted here.** The study is a test of benchmark interpretation, not a novel model architecture, causal educational intervention, or generalization guarantee. + +## 1. Introduction and specific research contribution + +Claims that sequential architectures outperform interpretable baselines are sensitive to the availability and representation of input data. Reproducible benchmarking therefore requires (a) identifying the exact event source; (b) fixing the observable learner history permitted at prediction time; (c) matching target events, learner splits, preprocessing scope, and uncertainty estimands; and (d) distinguishing computational/predictive results from causal pedagogical value. + +**Research questions:** + +- **RQ1 — provenance:** Can the common transformed ASSISTments 2009 mirror stand in for the source-corrected official 2009 Skill Builder data without changing its labeled event population? Earlier checksum-verified direct audits show **no**. +- **RQ2 — history budget:** Holding cohort, splits, test targets and model capacity fixed *within model family*, how does restricting the previous observable response stream to K=1 versus K=20 alter test Brier score and discrimination? +- **RQ3 — order representation:** At the same history budget and on exactly the same target responses, what is the descriptive predictive gap between a logistic model with summary counts, a logistic model augmented with derivable order proxies, and a small recurrent history encoder? +- **RQ4 — continuity across years:** Do earlier corrected-2009 and fresh 2015 direction-specific results survive careful qualification when placed alongside a new 2015 window study? This is comparison of the published scopes, **not** frozen-weight transfer nor a controlled cross-year experimental effect. + +The study's proposed incremental contribution is an **explicit two-axis source/history-window audit**, not the first proposal of input-matched knowledge tracing. + +## 2. Literature positioning (necessary non-overclaiming) + +- Polydoros (2026), *Cross-Cohort Knowledge-Tracing Benchmark*, Zenodo v2.0.1 (https://doi.org/10.5281/zenodo.22131530), compares PFA, BKT, DKT and SAKT on corrected ASSISTments 2009 and 2015. Hence neither cross-year comparison nor student-level cluster uncertainty is new here. +- Padi et al. (2026), *Stable and Faithful Explanations for Knowledge Tracing*, arXiv:2609.28502, explicitly studies information-matched baselines and duplicated KC records. Hence information parity and detecting leakage are not new constructs here. +- Badran and Preisach (2026), *Enhancing Knowledge Tracing through Leakage-Free and Recency-Aware Embeddings*, SN Computer Science 7:774, DOI 10.1007/s42979-026-05344-3, addresses multi-KC leakage and recency. This paper does **not** claim first detection of KC leakage, nor does it introduce their recency encoding. +- Gervet et al. (2020), *When Is Deep Learning the Best Approach to Knowledge Tracing?*, Journal of Educational Data Mining 12(3), demonstrates model-dependent comparisons on KT benchmarks. +- Piech et al. (2015), *Deep Knowledge Tracing*, NeurIPS 28, introduces neural sequential models and is background, not a method newly invented here. +- Eedi NeurIPS 2020 challenge (Wang et al., 2020, arXiv:2007.12061) publishes a larger, newer-source benchmark under CC BY-NC-ND 4.0; no experiment on Eedi is claimed in the current work. + +**What remains distinct and testable:** An explicitly pre-specified new learner subset with same-target restriction to previous 1 versus 20 response events and three representations, interpreted alongside a concrete **different-source 2009 count/sequence discrepancy**. Novelty is bounded; review by a subject specialist is necessary. + +## 3. Sources and prior verified findings + +### 3.1 Corrected 2009 vs transformed mirror + +Pinned third-party mirror SHA-256: `2053b7ea63f1d5ea953834bf82ed89543e1f11bae1e3e7535c2f9d7f49d4dda7`. Corrected official 2009 source SHA-256: `162ef8d2d28bcbfea6591a282994062bd8d5eaa00636544292a0d268dca6e5da`. The mirror includes 4,148 learner sequences and 274,331 labeled events; corrected official source includes 4,163 learners and 283,105 labeled events; 817/4,148 common learners differ in at least the count or sequence of skill–response pairs. These facts **reject exact source parity** and explain why old mirror-based test metrics cannot be silently labeled "corrected official 2009" results. + +### 3.2 Earlier predictive reanalyses + +Corrected official 2009 exploratory reanalysis: enriched PFA AUC 0.69508/Brier 0.196755 versus extended GRU AUC 0.73243/Brier 0.185074; 625 held-out learners/48,842 responses, with GRU minus PFA paired pooled Brier −0.011681 (2,000 learner-cluster bootstrap 95% interval [−0.013718, −0.009862]). This is **exploratory**, and its split/representation is not held fixed against the mirror benchmark; no model ranking change is causally attributed to source correction. + +2015 independent-year Stage 2: exclude 5,000 previously tested learners before selecting 5,000 new learners. Enriched PFA AUC 0.681286/Brier 0.186954 and GRU AUC 0.699683/Brier 0.182273 on 750 test learners/26,697 events, Brier delta −0.004681 [−0.006073, −0.003346]. Stage-1 result was discarded as confirmatory evidence after baseline numerical nonconvergence. Different-year models were retrained, not transported unchanged. + +## 4. New window-budget experiment — frozen before fresh data evaluation + +Complete operational specification and decision gates are in [METHODOLOGICAL_2026_PROTOCOL.md](../METHODOLOGICAL_2026_PROTOCOL.md); executable implementation is [scripts/matched_history_budget_2026.py](../scripts/matched_history_budget_2026.py). + +Official ASSISTments 2015 CSV SHA-256 `75e1f46131d5897a388c82411bc3032279899cb1f1e66a9043354d41234407a0`. Exclude all Stage-1 and Stage-2 learners (10,000 IDs), sample a new 4,000 from the remainder with seed 20261026, and split by learner with the same seed 2,800/600/600 for train/validation/test. Predictions for every model and history budget concern exactly the event targets at t>=1; the current response is never an input. + +- **K=1:** the single immediately preceding observed (skill, response) pair and current skill ID. +- **K=20:** at most 20 immediately preceding observed (skill, response) pairs and current skill ID. +- **Window-count logistic (PFA+):** current skill, previous same-skill success/failure counts and all-skill success/failure counts within the K-window. +- **Ordered-feature logistic:** same features, plus previous response outcome, most recent same-skill response sign and lag since that same-skill observation (all derivable from the same window). +- **Window-GRU:** a compact recurrent model reading only K preceding event tokens, current skill as prediction query, with train-only fitting and validation-only early stopping. + +All models share the **observable input stream**, learner populations, and test events; architectures, feature engineering and parameter counts differ. Numeric classifiers must converge and GRU training must complete under identical frozen hyperparameters across budgets. + +### Predeclared descriptive primary table — do not fill until real results exist + +| History budget | Model | Test Brier | Test ROC-AUC | Test log loss | Learners / test events | +|---|---|---:|---:|---:|---| +| K=1 | Window-count PFA+ | **PENDING** | **PENDING** | **PENDING** | 600 / PENDING | +| K=1 | Ordered-feature logistic | **PENDING** | **PENDING** | **PENDING** | 600 / PENDING | +| K=1 | Window-GRU | **PENDING** | **PENDING** | **PENDING** | 600 / PENDING | +| K=20 | Window-count PFA+ | **PENDING** | **PENDING** | **PENDING** | 600 / PENDING | +| K=20 | Ordered-feature logistic | **PENDING** | **PENDING** | **PENDING** | 600 / PENDING | +| K=20 | Window-GRU | **PENDING** | **PENDING** | **PENDING** | 600 / PENDING | + +Matched comparison: GRU Brier minus ordered-feature logistic Brier at each K, paired learner-cluster bootstrap (2,000 repetitions); positive/negative differences interpreted descriptively, and compared across budgets. All repeated choices are labeled exploratory with no family-wise multiplicity guarantee. + +## 5. Validity threats and interpretation bounds + +- New sample comes from the *same historical platform/year* as Stage 1 and 2; no modern-school deployment or external ecological validity follows. +- Split-seed and model-hyperparameter sensitivity are not completely assessed by a single stage-3 split and GRU seed. +- Ordered-feature logistic transforms historical signals explicitly, whereas recurrent models learn their own representations; equal input *availability* is not equal expressive capacity or optimization. +- Sequence labeling is based on recorded `log_id`, not on proof of clinical/educational learning states. Learner-based partitions prevent train–test ID overlap but do not rule out all upstream duplicated-question artifacts. +- Correction of 2009 data is a provenance result. A direct *paired* source-effect estimate would require aligned exact learner-event IDs and targets, unavailable from the transformed mirror. +- Bootstrap CIs are conditional on analyzed learners/model fits and not prospective causal intervals; model selection, multiple comparisons and the reviewed historical corpus limit confirmation language. +- Eedi and attention-model benchmarks are promising additional contexts but have **not** been executed and must not appear as study results. +- Disclosure of substantial generative AI involvement in coding/manuscript drafting is required by author policy; author must independently check data, code, citations and final interpretation. + +## 6. Conditional scientific conclusion + +**Until Stage-3 completes:** Earlier corrected-source and independent-year historical evidence shows reproducible predictive advantages of the tested GRU over the specific summary PFA baseline, but cannot attribute them to recurrence itself. The newly specified history-budget experiment is not claimed as executed, favorable, significant or publication-ready yet. + +**After execution:** The conclusion must be driven by observed changes and confidence intervals; an absence of predicted differences, reversal, or numerical failure is a reportable result rather than grounds for secretly retuning and replacing the study. + +## Links and reproducibility + +- [Official ASSISTments 2009 and corrected file](https://sites.google.com/site/assistmentsdata/home/2009-2010-assistment-data/skill-builder-data-2009-2010). +- [Official ASSISTments 2015](https://sites.google.com/site/assistmentsdata/datasets/2015-assistments-skill-builder-data). +- [Prior corrected source reanalysis](../results/corrected_2009_sensitivity.json). +- [Prior independent-year 2015 result](../results/confirmatory_assist2015_stage2.json). +- New Stage-3 results: [planned fixed output path](../results/matched_history_budget_2026.json), only valid when source file is actually created by the successful real-data pipeline. From bc89bbb7982de79b9541a1a4b3883ce94bbf5f1e Mon Sep 17 00:00:00 2001 From: Devis Wawan Saputra Date: Mon, 26 Oct 2026 17:51:38 +0700 Subject: [PATCH 40/48] ci: distinguish split-level 600 learners from 582 evaluable with a past event --- .github/workflows/matched-history-budget-2026.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/matched-history-budget-2026.yml b/.github/workflows/matched-history-budget-2026.yml index 2a79ba6..e3f39df 100644 --- a/.github/workflows/matched-history-budget-2026.yml +++ b/.github/workflows/matched-history-budget-2026.yml @@ -48,7 +48,8 @@ jobs: for x in b["models"].values(): assert 010000 for x in b["paired_GRU_minus"].values(): - assert x["n_learners"]==600 and x["n_bootstrap"]==2000 + assert 400 <= x["n_learners"] <= 600 and x["n_bootstrap"]==2000 + assert x["n_interactions"]==b["models"]["window_gru"]["n"] print("Verified fresh learner cohort and history-parity output gates.") PY - name: Upload aggregate evidence even if a research gate fails From 26ae8e0e2e8c4146c414973dab2882f2b0661aae Mon Sep 17 00:00:00 2001 From: Devis Wawan Saputra Date: Mon, 26 Oct 2026 17:52:06 +0700 Subject: [PATCH 41/48] protocol: disclose 18 single-event test learners and correction of release-only QA assertion --- METHODOLOGICAL_2026_PROTOCOL.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/METHODOLOGICAL_2026_PROTOCOL.md b/METHODOLOGICAL_2026_PROTOCOL.md index d6f0dc1..973af25 100644 --- a/METHODOLOGICAL_2026_PROTOCOL.md +++ b/METHODOLOGICAL_2026_PROTOCOL.md @@ -26,3 +26,7 @@ Sanity gates: checksum, exclusion counts, zero prior-cohort overlap, exact test ## Non-overclaiming and next steps Only merge observed results into manuscript after actual GitHub Actions real-data execution and critical result inspection. Earlier corrected-2009 and 2015 evidence remains separately reported. A 2026 survey/literature novelty statement must explicitly recognize recent matched-input and leakage papers. Additional modern datasets and AKT are useful but NOT claimed executed in this extension. No student-level data exported; only cohort hashes and aggregate predictions/statistics retained. + +## Post-execution correction to a software assertion (26 October 2026; no model/test retuning) + +The predeclared test split comprises **600 distinct learners**, but 18 test learners have no response at index t>=1 and therefore generate no history-eligible targets. **582 learners / 20,592 interactions** are evaluable on the common response target set. The first Actions run 40310088864 successfully executed both K budgets and wrote aggregate metrics (`status: passed_all_computational_gates`) but the downstream release assertion incorrectly demanded 600 **evaluated** learners. The workflow assertion was changed to verify **up to 600** evaluation learners, requiring more than 400, and exact event counts across paired models. There was no change to sampling, models, features, targets, seeds, or inferential analysis. The initial diagnostic artifact is retained. This correction and the initially failed pipeline must be disclosed as technical QA history; the new clean workflow run is the final computational release gate. From 9c20400c382b2667043e9a5ee325070cffcf6ead Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 26 Oct 2026 10:54:35 +0000 Subject: [PATCH 42/48] research: archive matched-history budget empirical results --- results/matched_history_budget_2026.json | 291 +++++++++++++++++++++++ 1 file changed, 291 insertions(+) create mode 100644 results/matched_history_budget_2026.json diff --git a/results/matched_history_budget_2026.json b/results/matched_history_budget_2026.json new file mode 100644 index 0000000..89401fe --- /dev/null +++ b/results/matched_history_budget_2026.json @@ -0,0 +1,291 @@ +{ + "status": "passed_all_computational_gates", + "source": { + "source_sha256": "75e1f46131d5897a388c82411bc3032279899cb1f1e66a9043354d41234407a0", + "eligible_learners": 19840, + "stage1_excluded": 5000, + "stage2_excluded": 5000, + "eligible_unused_pool": 9840, + "stage3_learners": 4000, + "stage3_interactions": 138129, + "learner_set_sha256": "0d9bef7d4d9d9ce4064eb63a04e7bba488ae72cd919bd7730bccd9bdecbb19f1", + "cohort_overlap_previous": 0, + "invalid_source_rows_excluded": 24830 + }, + "design": { + "predeclared_protocol": "METHODOLOGICAL_2026_PROTOCOL.md", + "stage3_sampling_seed": 20261026, + "learner_partition_seed": 20261026, + "train_learners": 2800, + "validation_learners": 600, + "test_learners": 600, + "history_budgets": [ + 1, + 20 + ], + "model_types": [ + "window_count", + "ordered_logistic", + "window_gru" + ], + "GRU_seed": 42, + "bootstrap_replicates": 2000, + "same_targets": "Every model at every K predicts responses t >= 1 for identical 600 test learners.", + "selection": "Fixed before new cohort test; prior research decisions were informed by other 2009/2015 data.", + "inference": "Descriptive exploratory bootstrap contrasts, not multiplicity-corrected confirmatory tests." + }, + "comparisons": { + "1": { + "models": { + "window_count": { + "n": 20592, + "positive_rate": 0.7369366744366744, + "brier": 0.17788199203683247, + "log_loss": 0.5345889370393874, + "average_precision": 0.8482492230416429, + "ece_10": 0.009320407240567558, + "roc_auc": 0.6844395899296832 + }, + "ordered_logistic": { + "n": 20592, + "positive_rate": 0.7369366744366744, + "brier": 0.1778819783584268, + "log_loss": 0.5345889271360941, + "average_precision": 0.8482469144579249, + "ece_10": 0.009320949287155366, + "roc_auc": 0.6844363662020748 + }, + "window_gru": { + "n": 20592, + "positive_rate": 0.7369366744366744, + "brier": 0.18000310877303297, + "log_loss": 0.5406576923145966, + "average_precision": 0.838027003779582, + "ece_10": 0.005780779843442258, + "roc_auc": 0.6709091368043554 + } + }, + "paired_GRU_minus": { + "window_count": { + "difference_candidate_minus_reference": 0.0021211167362004853, + "bootstrap_95_interval": [ + 0.0011984039269019259, + 0.003013353764954072 + ], + "resampling_unit": "learner", + "estimand_unit": "interaction", + "n_learners": 582, + "n_interactions": 20592, + "n_bootstrap": 2000 + }, + "ordered_logistic": { + "difference_candidate_minus_reference": 0.002121130414606117, + "bootstrap_95_interval": [ + 0.0011983198118476887, + 0.003013421140517525 + ], + "resampling_unit": "learner", + "estimand_unit": "interaction", + "n_learners": 582, + "n_interactions": 20592, + "n_bootstrap": 2000 + } + }, + "diagnostics": { + "window_count": { + "iterations": 295, + "warnings": [] + }, + "ordered_logistic": { + "iterations": 367, + "warnings": [] + }, + "window_gru": { + "best_epoch": 6, + "epochs_completed": 9, + "history": [ + { + "epoch": 1, + "train_bce": 0.5700001594834665, + "validation_bce": 0.5546365849513333 + }, + { + "epoch": 2, + "train_bce": 0.5496423426455948, + "validation_bce": 0.5503481250851094 + }, + { + "epoch": 3, + "train_bce": 0.545433577422607, + "validation_bce": 0.5471623890125249 + }, + { + "epoch": 4, + "train_bce": 0.5431627397913593, + "validation_bce": 0.5474760850748747 + }, + { + "epoch": 5, + "train_bce": 0.5419399417416683, + "validation_bce": 0.5466526810471217 + }, + { + "epoch": 6, + "train_bce": 0.5412652839078146, + "validation_bce": 0.5449585088872168 + }, + { + "epoch": 7, + "train_bce": 0.5408221157525078, + "validation_bce": 0.5450970664384037 + }, + { + "epoch": 8, + "train_bce": 0.5403140336999217, + "validation_bce": 0.545603285447066 + }, + { + "epoch": 9, + "train_bce": 0.5400628717056964, + "validation_bce": 0.5456161411414004 + } + ], + "train_targets": 93426, + "validation_targets": 20111, + "test_targets": 20592 + } + }, + "elapsed_seconds": 14.7 + }, + "20": { + "models": { + "window_count": { + "n": 20592, + "positive_rate": 0.7369366744366744, + "brier": 0.17667353711786812, + "log_loss": 0.5315903261834504, + "average_precision": 0.8522443340072219, + "ece_10": 0.006126123934703186, + "roc_auc": 0.6910582372474963 + }, + "ordered_logistic": { + "n": 20592, + "positive_rate": 0.7369366744366744, + "brier": 0.17388534919067833, + "log_loss": 0.5240946395813895, + "average_precision": 0.8610186198519205, + "ece_10": 0.0072511782088527195, + "roc_auc": 0.7050441361763368 + }, + "window_gru": { + "n": 20592, + "positive_rate": 0.7369366744366744, + "brier": 0.17514020613173276, + "log_loss": 0.5271526751223963, + "average_precision": 0.8589845012762509, + "ece_10": 0.011658007349593165, + "roc_auc": 0.6993609963629661 + } + }, + "paired_GRU_minus": { + "window_count": { + "difference_candidate_minus_reference": -0.0015333309861353841, + "bootstrap_95_interval": [ + -0.0027605673893992813, + -0.0002792867452439877 + ], + "resampling_unit": "learner", + "estimand_unit": "interaction", + "n_learners": 582, + "n_interactions": 20592, + "n_bootstrap": 2000 + }, + "ordered_logistic": { + "difference_candidate_minus_reference": 0.0012548569410544136, + "bootstrap_95_interval": [ + 7.536414859135726e-05, + 0.002425946875021449 + ], + "resampling_unit": "learner", + "estimand_unit": "interaction", + "n_learners": 582, + "n_interactions": 20592, + "n_bootstrap": 2000 + } + }, + "diagnostics": { + "window_count": { + "iterations": 299, + "warnings": [] + }, + "ordered_logistic": { + "iterations": 334, + "warnings": [] + }, + "window_gru": { + "best_epoch": 6, + "epochs_completed": 9, + "history": [ + { + "epoch": 1, + "train_bce": 0.562216157968126, + "validation_bce": 0.5460942462327485 + }, + { + "epoch": 2, + "train_bce": 0.5360034013242106, + "validation_bce": 0.53683507956098 + }, + { + "epoch": 3, + "train_bce": 0.5258215163299819, + "validation_bce": 0.5301942893069468 + }, + { + "epoch": 4, + "train_bce": 0.5190114440188723, + "validation_bce": 0.5288743185004735 + }, + { + "epoch": 5, + "train_bce": 0.5149992852793813, + "validation_bce": 0.5286781249708553 + }, + { + "epoch": 6, + "train_bce": 0.5117375197071786, + "validation_bce": 0.5277301228198755 + }, + { + "epoch": 7, + "train_bce": 0.5087343534964336, + "validation_bce": 0.5304507112539802 + }, + { + "epoch": 8, + "train_bce": 0.5056064738932391, + "validation_bce": 0.5304346994831836 + }, + { + "epoch": 9, + "train_bce": 0.5025589439044289, + "validation_bce": 0.5334981878134205 + } + ], + "train_targets": 93426, + "validation_targets": 20111, + "test_targets": 20592 + } + }, + "elapsed_seconds": 36.92 + } + }, + "within_model_brier_k20_minus_k1": { + "window_count": -0.0012084549189643468, + "ordered_logistic": -0.003996629167748483, + "window_gru": -0.00486290264130021 + }, + "gru_minus_ordered_gap_k20_minus_k1": -0.0008662734735517034, + "diagnostic_failures": [], + "interpretation": "Same target set and same last-K learner response information; representations/capacities still unequal. Single stage-3 sample after prior research; no familywise or causal claim." +} From c87230e4efdc30247bbde743d559c06a0d48fe23 Mon Sep 17 00:00:00 2001 From: Devis Wawan Saputra Date: Mon, 26 Oct 2026 18:01:13 +0700 Subject: [PATCH 43/48] paper: integrate verified matched-history experiment into 2026 methodological preprint --- paper/main_methodological_2026.tex | 261 +++++++++++++++++++++++++++++ 1 file changed, 261 insertions(+) create mode 100644 paper/main_methodological_2026.tex diff --git a/paper/main_methodological_2026.tex b/paper/main_methodological_2026.tex new file mode 100644 index 0000000..e142ab6 --- /dev/null +++ b/paper/main_methodological_2026.tex @@ -0,0 +1,261 @@ +\documentclass[11pt,a4paper]{article} +\usepackage[margin=0.84in]{geometry} +\usepackage[T1]{fontenc} +\usepackage[utf8]{inputenc} +\usepackage{lmodern,microtype} +\usepackage{amsmath,amssymb,booktabs,array,graphicx,caption,subcaption} +\usepackage[colorlinks=true,urlcolor=blue,citecolor=blue,linkcolor=blue]{hyperref} +\usepackage{natbib} +\sloppy\emergencystretch=4em +\setlength{\parskip}{0.14em}\renewcommand{\baselinestretch}{0.96} +\setlength{\parindent}{0pt} +\title{When Knowledge-Tracing Rankings Depend on Available History\\\large A Source-Correction and Matched-Information Audit on ASSISTments 2009--2015} +\author{Devis Wawan Saputra\\\small Independent Researcher, Indonesia\\\small ORCID: \href{https://orcid.org/0000-0002-7133-9410}{0000-0002-7133-9410}} +\date{26 October 2026\\\small Revised empirical methodological manuscript; not peer reviewed} +\begin{document} +\maketitle +\begin{abstract} +Predictive comparisons between transparent and recurrent knowledge-tracing models can reflect unequal access to response history, transformations of historical datasets, or feature representation rather than recurrent architecture alone. We combine corrected-source provenance checks with a frozen, matched-history-budget test on independently sampled ASSISTments 2015 learners. A direct audit rejects exact equivalence between a frequently used transformed ASSISTments 2009 mirror (4,148 learners; 274,331 labeled responses) and the corrected official release: 817 shared learners have different normalized skill--response sequences. Earlier re-executions and a new 2015 cohort had favored GRU over count-based PFA, but supplied different history representations. For a new 2015 experiment, we excluded all 10,000 learners in the previous two evaluations and sampled 4,000 untouched learners. On the same 20,592 held-out next-response targets from 582 evaluable learners, we compared window-count logistic regression, an order-aware logistic baseline, and a window-GRU, each restricted to only the preceding one or 20 labeled interactions and the current skill identifier. With a one-event history, Brier scores were 0.17788, 0.17788, and 0.18000, respectively; with 20 events, they were 0.17667, 0.17389, and 0.17514. The GRU-minus-order-aware Brier difference was +0.00212 (95\% learner-cluster interval [+0.00120,+0.00301]) for one event and +0.00125 ([+0.00008,+0.00243]) for 20. These descriptive comparisons reverse the apparent recurrent advantage seen against less expressive history summaries, while longer history improves all three models. The evidence identifies sensitivity to observable-history budget and model specification, \emph{not} the causal contribution of recurrence, modern classroom effectiveness, or universal superiority. The 2015 window analysis uses one split and neural seed and should be replicated on additional domains. +\end{abstract} +\textbf{Keywords:} AI in education; knowledge tracing; ASSISTments; leakage-aware evaluation; BKT; PFA; recurrent networks; information parity; history window; model ranking sensitivity; reproducibility. + +\section{Introduction} +Knowledge tracing predicts a learner's subsequent response from observed practice history. The probabilistic outputs can be used to rank cases for inspection and to support adaptive content, but predictive validity must not be conflated with instructional value. A model that has learned response regularities on a historical tutoring platform may still fail under new curricula, item policies, or student populations. + +A second challenge concerns experimental comparison. A random row split can let responses from the same learner appear in both model development and evaluation. Such a split evaluates interpolation within known learner histories rather than the intended generalization to unseen learners. Another challenge is information symmetry: a recurrent model may use prior outcomes from other skills while a classical per-skill model sees only within-skill history. Predictive differences then reflect both representation and architecture. + +Rather than introduce a new model, this study audits the empirical scope of a controlled learner-disjoint benchmark, explicitly recording probabilistic performance, slice results, calibration, and uncertainty limits. It answers: (RQ1) How do conventional baselines compare on a frozen learner-disjoint partition? (RQ2) Does prediction differ between a learner's first observed and later encounters with a skill? (RQ3) What do the resampling intervals establish under a fixed learner split? (RQ4) Do extended training and a richer transparent baseline materially change the results, and what remains untested? (RQ5) Does the direction of the predictive difference replicate on an independent-year cohort? (RQ6) Holding the learner sample, response targets and observable history restricted, does representation of that history change the apparent advantage of recurrence, and how do predictions change between K=1 and K=20 prior events? + +\section{Related Work and Positioning} +BKT recursively updates a latent mastery probability using learning-transition, guess, and slip assumptions \citep{corbett1995}. PFA predicts responses with logistic effects of skill identity and previous successes and failures \citep{pavlik2009}. Deep Knowledge Tracing introduced recurrent sequence representations \citep{piech2015}. Later evaluations demonstrated that claims of deep-model superiority are highly contingent on datasets, baselines, and evaluation design \citep{gervet2020}. Broader surveys emphasize inconsistent versions, preprocessing, and metric protocols across commonly named knowledge-tracing datasets \citep{abdelrahman2023}. More recent work directly addresses correction-aware and cross-cohort benchmarking. A 2026 replication package by Polydoros compares classical and deep approaches on the corrected ASSISTments 2009 and 2015 cohorts with learner-level resampling \citep{polydoros2026}. Padi and colleagues (2026) show empirically that duplicate multi-skill records and asymmetries in the information available to different model families can change reported rankings \citep{padi2026}. Badran and Preisach (2026) independently study leakage-free embeddings and skill-specific recency, establishing that these methodological ideas already appear in contemporary work \citep{badran2026}. These advances preclude claiming novelty for learner-disjoint splitting, calibration analysis, or information matching in isolation. They also provide corrected-source and cross-cohort comparators, so we make no state-of-the-art or architecture-advantage claims. The distinctive contribution here is the auditable sequence of a frozen transformed-mirror reproduction, contrasting uncertainty targets, an explicitly labeled exploratory information-budget audit, and a separate-year predictive replication that retains rather than conceals a nonconverged first pilot. Unlike a model-development paper, this investigation treats exact source provenance, estimand distinctions, and explicit failure of source parity as empirical audit findings rather than claiming a state-of-the-art architecture. + +\section{Study Contribution and Evidence Hierarchy} +We provide six bounded outputs: (1) an executable source- and learner-split-pinned reproduction of six models and model-specific probability metrics on the transformed 2009 mirror; (2) a distinction between equal-learner and interaction-pooled Brier uncertainty targets; (3) an exploratory same-split sensitivity analysis comparing cross-skill PFA summary features with extended GRU training; and (4) a direct event-count and ordered-sequence parity audit against the official corrected 2009 source; (5) an exploratory official corrected-2009 source sensitivity analysis; and (6) a documented independent-year 2015 predictive replication on a distinct set of learners. The two evidence tiers must not be conflated: post-inspection 2009 improvements are exploratory, while the second 2015 cohort is fresh with respect to learner identity and outcome inspection. The second-stage 2015 protocol was nonetheless fixed \emph{after} observing a failed optimizer convergence diagnostic in the first 2015 pilot; its corrective choice and chronology are disclosed, and the result is not described as prospectively registered or independently peer-reviewed. These analyses compare unequal information representations and model capacity; they cannot identify a recurrence-only effect. (7) We add a newly frozen third ASSISTments 2015 learner sample, disjoint from both earlier samples, comparing three predictors at two \emph{matched available history} budgets. Unlike the earlier pilot and exploratory comparisons, these model/feature windows were fixed before evaluating the new learner cohort. The upgrade tests the dependency of model ranking on representation and history horizon; it does not invent information-parity benchmarking. + +\section{Data and Evaluation Protocol} +\subsection{Dataset origin and observed differences from the corrected source} +The intended underlying source is the ASSISTments 2009--2010 Skill Builder data documented by its maintainers \citep{assistmentsdata}; the ASSISTments tutoring platform and its formative-assessment context were described by \citet{feng2009}. The execution pipeline reads \texttt{Atomi/ASSISTments2009} at pinned revision \texttt{c72a664a9693547fb206652}\allowbreak\texttt{ed2ce18e62d320c7d} and file \path{data/train-00000-of-00001.parquet}. The archived file SHA-256 is \texttt{2053b7ea63f1d5ea953834bf82ed8954}\\\texttt{3e1f11bae1e3e7535c2f9d7f49d4dda7}. The mirror contains one serialized sequence per learner. The empirical record reports 4,148 unique learners and 274,331 binary response observations. + +\textbf{Direct provenance audit.} The ASSISTments maintainers corrected an earlier Skill Builder release after identifying duplicated records \citep{assistmentsdata}. We downloaded the official corrected student--problem CSV directly from the file they link (SHA-256 \texttt{162ef8d2d28bcbfea6591a282994062bd8d5eaa00636544292a0d268dca6e5da}) and compared it with the exact pinned mirror used for the model run. The corrected file contains 346,860 student--problem records, of which 283,105 contain a usable binary outcome and nonmissing skill identifier, from 4,163 learners. The mirror comprises 274,331 responses from 4,148 learners. All 4,148 mirror learner IDs occur in the corrected source; 15 additional learners occur only in the corrected source. Among the 4,148 shared learners, 3,331 have identical sequences of normalized (skill identifier, binary response) pairs when official records are ordered by numeric \texttt{order\_id}; 817 do not. The same 3,331 satisfy event-count and skill--response-multiset equality. Accordingly, \emph{exact whole-dataset parity is empirically disproved}, not merely unverified. The original mirror-derived predictive results remain correct for the pinned mirror, but cannot be represented as estimates from the corrected official 2009 release. This audit does not identify which upstream transformation rule caused the discrepancies. Although corrected-source learner and event keys can be checked directly, the mirror omits original student--problem event IDs, so a complete one-to-one event crosswalk and independently timestamped chronology are not recoverable. Numeric \texttt{order\_id} was used as a documented ordering convention, not as proof of real-world time order. The full aggregate audit, source hashes and code are archived in GitHub Actions run \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39773920168}{39773920168}; individual student data are not redistributed. + +\subsection{Officially linked 2015 independent-year source} +For the different-year replication we use the original ASSISTments 2015 Skill Builder CSV linked by the project's official dataset portal \citep{assistments2015}. The original file SHA-256 is \texttt{75e1f46131d5897a388c82411bc303}\texttt{2279899cb1f1e66a9043354d41234407a0}. It contains 708,631 records; 24,830 were excluded by the frozen missingness/validity filter, leaving 19,840 eligible learner IDs. The study constructs within-learner sequences in ascending numerical \texttt{log\_id} order, using \texttt{sequence\_id} as a 2015 skill-builder identifier; these identifiers are not the same construct as the 2009 mirror's \texttt{skill\_ids}. This different-year study requires new fitting on 2015 training learners and does not test a frozen 2009-trained model on 2015. The analysis source, hashes, sampling seeds, and both pilot and corrective-stage outcomes are archived in the repository's research branch. The 2015 result does not retroactively validate the 2009 mirror transformation. + +\subsection{Partition unit, leakage checks, and model selection} +Learners are shuffled at seed 42 and divided into training (2,903), validation (622), and test (623) partitions. The runner first rejects duplicated \texttt{user\_id} sequence rows, then forbids learner overlap by partitioning whole serialized rows. It reports 194,951 training responses and 41,228 test responses. This check establishes no repeated \emph{learner identifier} across partitions, but does not verify the identity or chronological ordering of individual raw student--problem events. The validation set determines the BKT parameter choice and neural stopping epoch. Global and skill priors, PFA coefficients, and recurrent embeddings are learned from the training partition only. Every stateful model emits its prediction \emph{before} observing the target response. + +\subsection{Model specifications} +\textbf{Global prior.} The binary response probability is the aggregate training correctness fraction. \textbf{Skill prior.} The response probability is the training correctness frequency for the queried skill, with a global fallback. \textbf{Fixed BKT.} Initial mastery, learning, guess, and slip probabilities are respectively 0.20, 0.15, 0.20, and 0.10. The prediction for a current response uses the previous posterior mastery state: +\begin{equation} +P(Y_t=1\mid H_{t-1})=m_{t-1}(1-s)+(1-m_{t-1})g. +\end{equation} +\textbf{Validation-tuned BKT.} The four parameters are selected from a fixed grid using validation log loss; the archived selection is initial mastery 0.40, learning 0.15, guess 0.30, and slip 0.20. \textbf{PFA.} A regularized logistic model uses a skill intercept and preceding learner--skill success/failure counts. \textbf{GRU.} A 64-dimensional recurrent hidden state processes previous response-conditioned skill tokens and is queried with the next skill embedding; seed-42 predictions for each learner's first response fall back to a training skill/global prior. Seeds 13, 42, and 73 were run for at most eight training epochs with validation-loss monitoring. + +\textbf{Comparability caveat.} PFA counts only prior outcomes on the queried skill; the GRU can exploit the complete previous cross-skill sequence. The observed gap does not isolate the effect of nonlinearity or recurrence. The executed extension adds a PFA comparator with prior cross-skill success/failure counts, and longer GRU training. Because this comparator is still a compact summary rather than a full ordered-history model, the test does not isolate the effect of model architecture. + +\subsection{Evaluation, uncertainty, and estimands} +ROC-AUC and average precision quantify discrimination and ranking. Brier score and log loss quantify probabilistic accuracy. Expected calibration error (ECE-10) compares confidence and observed response frequencies in ten probability bins, but is sample/bin-sensitive. A learner-first skill occurrence means that skill key has not appeared previously \emph{for that learner}; it is not necessarily unseen in training. + +There are two distinct aggregation targets in the existing report. The primary table pools response-level losses, giving interaction-weighted estimates: +\begin{equation} +\widehat{B}_{\mathrm{event}}(f)=\frac{1}{N}\sum_{i=1}^{n}\sum_{t=1}^{T_i}(y_{it}-\hat p_{it}^{(f)})^2,\quad N=\sum_i T_i. +\end{equation} +The archived cluster bootstrap instead averages per-learner loss differences: +\begin{equation} +\widehat{\Delta}_{\mathrm{learner}}=\frac{1}{n}\sum_i\frac{1}{T_i}\sum_t\left[(y_{it}-\hat p_{it}^{(f)})^2-(y_{it}-\hat p_{it}^{(g)})^2\right]. +\end{equation} +That is a defensible \emph{different} estimand. It must not be interpreted as an uncertainty interval for the event-pooled Brier difference. A paired learner-cluster bootstrap for the event-weighted difference resamples learners and recomputes the ratio of summed losses to summed response counts in each replicate. The executed supplement implements this correction, with 2,000 learner-resampling draws per comparison. Each draw recomputes the ratio of the sampled cluster loss sums to the sampled cluster response counts; the resulting percentile intervals are conditional on this one previously examined test partition. + +\section{Reproduction and Exploratory Robustness Results} +\subsection{Overall held-out comparisons} +Table~\ref{tab:results} reports the results from a new full execution of the original study on the pinned real-data mirror. Reproduced original values agree with the archived record to within $4\times10^{-8}$ for ROC-AUC and much more closely for Brier score. +\begin{table}[ht] +\centering\small +\caption{Reproduced results on 41,228 responses from 623 learner-disjoint test learners (seed-42 original GRU). Lower Brier, log loss, and ECE are better.} +\label{tab:results} +\begin{tabular}{lrrrrr}\toprule +Model & ROC-AUC & AP & Brier & Log loss & ECE-10\\\midrule +Global prior & 0.5000 & 0.6747 & 0.2197 & 0.6313 & 0.0160\\ +Skill prior & 0.6234 & 0.7669 & 0.2097 & 0.6096 & 0.0145\\ +Fixed BKT & 0.6746 & 0.7806 & 0.2187 & 0.6360 & 0.1226\\ +Validation-tuned BKT & 0.6722 & 0.7770 & 0.2024 & 0.5928 & 0.0279\\ +PFA logistic & 0.6982 & 0.8074 & 0.1976 & 0.5840 & 0.0413\\ +GRU (seed 42) & \textbf{0.7470} & \textbf{0.8439} & \textbf{0.1800} & \textbf{0.5382} & \textbf{0.0133}\\\bottomrule +\end{tabular} +\end{table} +The recorded GRU-minus-PFA Brier difference is approximately $-0.01765$ (an 8.9\% relative reduction with PFA as denominator), and ROC-AUC difference is $+0.04878$. These are \emph{descriptive differences} on one test partition. A separate \emph{exploratory} paired uncertainty calculation for the later strengthened comparison is reported below. In particular, the fixed-parameter BKT achieves higher AUC than the training-skill prior while having worse Brier score, illustrating why ranking and probability quality must be considered separately. + +\begin{figure}[ht]\centering\includegraphics[width=.83\linewidth]{kt_brier.png}\caption{Reproduced response-pooled Brier scores for the original six benchmark models on the pinned real-data mirror.}\end{figure} + +\subsection{First skill occurrence versus repeated occurrences} +\begin{table}[ht]\centering\small +\caption{Recorded PFA and seed-42 GRU performance by within-learner skill occurrence.}\begin{tabular}{lrrrrr}\toprule +Slice & N & PFA AUC & GRU AUC & PFA Brier & GRU Brier\\\midrule +First observed skill & 6,193 & 0.6499 & 0.7224 & 0.2244 & 0.2094\\ +Repeated skill & 35,035 & 0.7068 & 0.7548 & 0.1929 & 0.1747\\\bottomrule +\end{tabular}\end{table} +The GRU has lower recorded Brier scores in both slices. However, this does not imply validated mastery estimation or a benefit for instructional decisions. Difficulty mix and number of available previous responses vary by slice. + +\subsection{Random-seed variability and archived learner uncertainty} +Across the three prespecified GRU seeds the mean ROC-AUC is 0.74708 (sample SD 0.00083) and mean Brier score is 0.17975 (sample SD 0.00019). These quantify three optimization restarts on the same data split, not independent replications or evidence of robustness across populations. + +The original run reproduces a \emph{mean per-learner} Brier difference between the seed-42 GRU and skill prior of $-0.02888$, with 95\% learner-block percentile interval $[-0.03439,-0.02365]$ for 623 learners. This interval must be linked to the learner-averaged quantity given above and cannot be relabelled as the pooled response-level comparison. The new cluster bootstrap for the interaction-pooled GRU--PFA comparison is presented separately below; its estimand must not be conflated with the original equal-learner-weighted interval. + +\subsection{Independent execution reproducibility check} +The original end-to-end workflow was rerun in GitHub Actions (run \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39655888620}{39655888620}) on 22 October 2026. The downloaded real-data file matched the archived SHA-256 \texttt{2053b7ea63f1d5ea953834bf82ed89543e1f11bae1e3e7535c2f9d7f49d4dda7}. Learner and response counts, learner partitions, original baseline Brier scores, and full-run AUCs matched archived results at reporting precision; the maximum original AUC drift was below $4\times10^{-8}$. Both code tests and the full data-validation job finished successfully. This is a computational reproduction of \emph{the same pinned third-party mirror}, not a parity audit against the official corrected source. + +\subsection{Direct corrected-source parity test} +The corrected official release has 4,163 learners, including the 4,148 with sequences in the mirror. Normalizing the skill IDs and sorting the corrected file by numeric \texttt{order\_id} within each learner gives exact sequence agreement for 3,331 shared learner IDs; it fails for 817. Table~\ref{tab:parity} shows the necessary-condition comparison. Thus the frozen 2009 mirror results cannot be relabeled as corrected-release results. Ordered pair equality provides some external corroboration for unaffected learners, but it does not restore original problem-level identifiers to the mirror or prove that the \texttt{order\_id} convention captures the physical event chronology. +\begin{table}[ht]\centering\small +\caption{Official corrected 2009 skill-builder records compared with the pinned Atomi sequence mirror. Official labeled rows exclude records without a usable skill identifier.} +\label{tab:parity} +\begin{tabular}{lrr}\toprule +Measure & Corrected official & Pinned mirror\\\midrule +Unique learners & 4,163 & 4,148\\ +Skill-labeled responses & 283,105 & 274,331\\ +Shared learner IDs & \multicolumn{2}{c}{4,148}\\ +Identical ordered skill--response sequences & \multicolumn{2}{c}{3,331 shared learners}\\ +Different ordered sequences & \multicolumn{2}{c}{817 shared learners}\\\bottomrule +\end{tabular} +\end{table} + +\subsection{Exploratory model and training-budget sensitivity} +To reduce the original information asymmetry, an enriched PFA baseline uses skill identity and both skill-specific and overall prior success/failure counts, with no access to the current response. The recurrent model still has richer ordered history and greater representational capacity; the comparison improves historical-information matching but does not equalize feature classes or prove an architectural mechanism. Extended GRU optimization used seeds 13, 42, and 73, at most 25 epochs, validation-loss early stopping and patience four. All three fits selected their best validation state before the cap. Table~\ref{tab:robust} summarizes the recorded out-of-sample scores \emph{on the same already-inspected test partition}. + +\begin{table}[ht]\centering\small +\caption{Exploratory robustness results from real-data reruns. All rows use the same 623 test learners (41,228 responses); GRU settings are exploratory revisions after earlier test inspection.} +\label{tab:robust} +\begin{tabular}{lrrrr}\toprule +Model & ROC-AUC & Brier & Log loss & Best epoch\\\midrule +Standard PFA & 0.6982 & 0.1976 & 0.5840 & --\\ +PFA + cross-skill counts & 0.7135 & 0.1929 & 0.5709 & --\\ +GRU (seed 13, extended) & 0.7471 & 0.1793 & 0.5367 & 13\\ +GRU (seed 42, extended) & 0.7491 & 0.1791 & 0.5357 & 13\\ +GRU (seed 73, extended) & 0.7498 & 0.1789 & 0.5358 & 11\\\bottomrule +\end{tabular} +\end{table} + +Relative to the stronger PFA, the extended seed-42 GRU has a response-pooled Brier improvement of $-0.013874$ (paired learner-cluster 95\% percentile interval $[-0.017231,-0.011049]$). Results for seeds 13 and 73 are $-0.013583$ [$-0.017004,-0.010911$] and $-0.014020$ [$-0.017418,-0.011123$], respectively. These bootstrap intervals (2,000 resamples of learners) describe conditional uncertainty \emph{on the fixed test partition after exploratory model changes}; they should not be interpreted as confirmatory evidence from an untouched test sample. The stronger PFA itself improves Brier score relative to the original PFA by $-0.004680$ [$-0.005900,-0.003537$], underscoring the effect of history-feature availability. + +\begin{figure}[ht]\centering\includegraphics[width=.90\linewidth]{robust_comparison.png}\caption{Exploratory robustness: real-data ROC-AUC and Brier score for PFA variants and three extended GRU seeds. This is a same-split sensitivity comparison, not an independent validation sample.}\end{figure} + +\begin{figure}[ht]\centering\includegraphics[width=.83\linewidth]{kt_auc.png}\caption{Reproduced test ROC-AUC across original models, shown descriptively for the fixed split.}\end{figure} + +\subsection{Exploratory predictive sensitivity on corrected official 2009 data} +To determine how much the corrected source changes the predictive comparison, we ran a separate, prewritten \emph{exploratory} pipeline on the official corrected release rather than treating the mirror as equivalent. We excluded unlabeled skill records, normalized skill identifiers, ordered each learner's records by numeric \texttt{order\_id}, and used a whole-learner 70/15/15 split with seed 20261024. The enriched PFA retained its same-skill and across-skill history summaries, and the GRU retained prior ordered interactions, seed 42, 25-epoch maximum and validation-based stopping. Both models were refitted exclusively on corrected-source training learners. This is not a fully untouched confirmatory comparison, because the same 2009 academic setting and modeling direction were previously analyzed. + +On 625 corrected-source test learners and 48,842 held-out interactions, the enriched PFA achieved ROC-AUC 0.6951, Brier 0.19675, and log loss 0.5809; the GRU achieved ROC-AUC 0.7324, Brier 0.18507, and log loss 0.5512. The paired interaction-weighted Brier difference (GRU minus PFA) was $-0.01168$ (95\% learner-cluster bootstrap percentile interval $[-0.01372,-0.00986]$, 2,000 resamples). The PFA optimizer converged in 930 iterations, and the GRU's best validation epoch was 13. These source-corrected figures are not directly numerically comparable with the 2009 mirror test split because its learner membership and preprocessing differ. The new analysis audits predictive sensitivity; it does not restore mirror/event parity or identify architectural causes. The aggregate code and outputs are archived in GitHub Actions run \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39774414969}{39774414969}. + +\section{Independent-Year ASSISTments 2015 Replication} +\subsection{Frozen sampling sequence and diagnostic correction} +The first 2015 analysis sampled 5,000 learners with seed 20261022, then assigned learners to 70\%/15\%/15\% train, validation, and test groups, comparing the enriched cross-skill PFA with the extended GRU. Although this initial pilot yielded a nominal GRU Brier advantage, the PFA \texttt{saga} optimizer reached its iteration limit and the declared convergence gate failed. The pilot is archived and \emph{not counted as clean confirmation}. Its test results must not be selectively omitted simply because they are provisional. + +Before any second-stage test outcomes were inspected, a corrective protocol changed only the PFA numerical solver to \texttt{lbfgs} (\texttt{C=1}, maximum 2,000 iterations, tolerance $10^{-8}$) for the same regularized logistic model with skill intercepts, within-skill successes/failures, and pooled prior cross-skill successes/failures. It reconstructed and excluded every one of the 5,000 pilot learners, sampled 5,000 different learners (seed 20261023), and created learner-disjoint groups of 3,500 train, 750 validation, and 750 test learners. The GRU configuration remained fixed (seed 42; token embedding 32, skill query embedding 16, hidden size 64, Adam learning rate 0.002, up to 25 epochs, validation patience four). The GRU's best validation epoch was 7 of 11 executed. The enriched PFA optimizer converged in 484 iterations without convergence warnings. No observations from the first sample appear in the second sample. + +\subsection{Untouched second-sample results and estimand} +The primary estimand is the interaction-pooled paired Brier loss difference, GRU minus enriched PFA, on the \emph{second} sample's final holdout. Two thousand learner-cluster bootstrap replicates (seed 20261023) resample whole learners and preserve the pooled-interaction weighting. A directional replication was counted only if the upper 95\% percentile interval endpoint was below zero and all provenance, disjointness, and convergence checks passed. This is a one-sided directional assessment using a displayed two-sided interval, not a preregistered general benchmark claim. + +\begin{table}[ht]\centering\small +\caption{ASSISTments 2015 independent learner-cohort test (750 learners, 26,697 test responses). Higher AUC and lower Brier, log loss and ECE-10 are favorable. The second sample excludes all 5,000 learners from the first diagnostic pilot.} +\label{tab:cohort2015} +\begin{tabular}{lrrrr}\toprule +Model & ROC-AUC & Brier & Log loss & ECE-10\\\midrule +Enriched PFA (converged) & 0.68129 & 0.18695 & 0.55668 & 0.01694\\ +Extended GRU (seed 42) & 0.69968 & 0.18227 & 0.54346 & 0.00820\\\bottomrule +\end{tabular} +\end{table} + +The test correctness rate was 0.71697. The primary Brier difference was $-0.0046813$, with a 95\% learner-cluster percentile interval $[-0.0060730,-0.0033457]$. The prespecified \emph{second-stage} computational and directional gates passed, supported by GitHub Actions run \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39754854426}{39754854426}. The first, nonconverged stage and its outcomes remain available in run \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39754046492}{39754046492}. Because both 2015 experiments used the same source archive but \emph{nonoverlapping} learner samples, this is a fresh-sample, distinct-year predictive replication, not a demonstration of cross-institutional transport or strict information matching. + + +\section{2026 Matched-History-Budget Study on a New ASSISTments 2015 Cohort} +\subsection{Frozen stage-3 design and disjoint history budgets} +After completing the previous 2015 pilot and disjoint second-stage replication, we froze a separate methodological protocol \emph{before} evaluating a third learner sample (see \href{https://github.com/devissaputra/knowledge_tracing_benchmark/blob/research/kt-methodological-2026-history-budget/METHODOLOGICAL_2026_PROTOCOL.md}{frozen protocol}). The official 2015 source hash remained \texttt{75e1f46131d5897a388c82411bc3032279899cb1f1e66a9043354d41234407a0}. Using the original 19,840 eligible learner IDs and the published stage-1 and stage-2 random-seed sampling algorithms, we excluded \emph{all 10,000 learners} selected earlier. From the 9,840 remaining learners, we sampled a third 4,000 with seed 20261026, yielding 138,129 interactions. The learner-disjoint train/validation/test split contained 2,800/600/600 learners (the same seed). + +Only responses after a learner's first recorded event are eligible targets, ensuring every evaluated response has at least one observed prior event. Consequently 18 of the 600 test learners contributed no history-eligible target, leaving \textbf{582 evaluable learners and 20,592 common next-response targets}. This distinction required correction of a downstream workflow assertion after the first computational run, with no changes to the model, split, data, features, training seeds or bootstrap. The initial failed release gate and corrected gate are preserved in the public workflow record. + +For an event at position $t\geq1$, the permissible input comprises its current skill identifier and only the immediately preceding $\min(K,t)$ observed skill--correctness pairs, for a history budget $K\in\{1,20\}$. The current response is never an input. Count-based and neural models use the \emph{same observable information stream and identical response targets}. Training, validation and test learners remain disjoint; supervised fitting and standardization use training data only. Predictions on held-out learners are sequential online predictions, permitted to condition on \emph{earlier, already observed} responses within that learner. Such history access is not test-label leakage for the current event. + +\subsection{Three fixed representations and descriptive estimands} +\textbf{Window-count PFA+:} Regularized logistic regression ($C=1$) with current-skill indicators and previous correct/incorrect counts for the queried skill and across all skills, computed strictly inside the last-$K$ window. \textbf{Order-aware logistic:} The identical count features plus the most recent prior correctness value, the latest result for the current skill (or absence flag) and lag to that skill within the window, all derivable from the permitted events. Both use LBFGS, standardized numeric features, 1,600-iteration ceiling and recorded convergence warnings. \textbf{Window-GRU:} Last-$K$ skill/outcome tokens enter a 48-unit hidden-state GRU with a separate 12-dimensional current-skill query embedding; token embedding 24, Adam learning rate 0.002, batch 256, up to 12 epochs, patience 3 on validation, seed 42. It sees no additional response history beyond the allowed K. Parameter counts and inductive biases remain unequal, so any gap cannot identify the isolated architecture effect. + +Primary descriptive metrics are test Brier score, AUC, log loss and ECE-10. The principal model contrasts are paired \emph{interaction-weighted} GRU-minus-logistic Brier differences, with 2,000 learner-cluster percentile bootstrap resamples. These intervals describe sampling uncertainty conditional on one model fit and test split, without corrections for multiple models or budgets. + +\subsection{Real-data results: order-aware logistic reverses the previous model ranking} +\begin{table}[t] +\centering\small +\caption{Same-target Stage-3 test performance at fixed prior-interaction budgets; $n=20{,}592$ responses from 582 evaluable learners at \emph{every} row. Lower Brier/log loss/ECE and higher AUC are better.} +\label{tab:budgetstudy} +\begin{tabular}{r l r r r r} +\toprule +$K$ & Model & Brier & AUC & Log loss & ECE-10\\ +\midrule +1 & Window-count PFA+ & 0.17788 & 0.68444 & 0.53459 & 0.00932 \\ +1 & Order-aware logistic & 0.17788 & 0.68444 & 0.53459 & 0.00932 \\ +1 & Window-GRU & 0.18000 & 0.67091 & 0.54066 & 0.00578 \\ +20 & Window-count PFA+ & 0.17667 & 0.69106 & 0.53159 & 0.00613 \\ +20 & Order-aware logistic & 0.17389 & 0.70504 & 0.52409 & 0.00725 \\ +20 & Window-GRU & 0.17514 & 0.69936 & 0.52715 & 0.01166 \\ +\bottomrule +\end{tabular} +\end{table} +\begin{figure}[t] +\centering\includegraphics[width=.88\linewidth]{history_budget_brier.png} +\caption{Frozen Stage-3 budget sweep with shared train/validation/test learner partitions and exactly matching held-out prediction events. Error intervals for paired model differences are reported in text, not on the points.} +\label{fig:historybudget} +\end{figure} + +The best single model at both budgets was the \emph{order-aware logistic} model. With only one previous response, count and order-aware models are numerically nearly identical, as expected: the single response already determines the recent-history summary. At 20 prior events, explicit order features improve the logistic Brier score from 0.17667 to 0.17389; the corresponding GRU is 0.17514. Moving from $K=1$ to $K=20$, Brier improves by \textbf{0.00121} for window-count, \textbf{0.00400} for order-aware logistic and \textbf{0.00486} for GRU. The original GRU advantage over standard PFA thus does not universally carry over when a transparent ordered-history comparator is given the same observable response window. + +At $K=1$, GRU minus order-aware logistic had pooled Brier difference $+0.002121$ with a 95\% paired learner-cluster bootstrap interval $[+0.001198,+0.003013]$. Relative to window-count PFA+, the corresponding GRU difference was $+0.002121$, interval $[+0.001198,+0.003013]$. + +At $K=20$, GRU minus order-aware logistic had pooled Brier difference $+0.001255$ with a 95\% paired learner-cluster bootstrap interval $[+0.000075,+0.002426]$. Relative to window-count PFA+, the corresponding GRU difference was $-0.001533$, interval $[-0.002761,-0.000279]$. + + + +For the new $K=20$ analysis, all logistic solvers reached numerical convergence without warnings, and the GRU's best validation epoch was 6 (both budgets). The preprocessing hash, nonoverlapping cohort, same-label target identity and numerical sanity gates passed. No family-wise significance claim or empirical generalization beyond the study's single held-out source is made. In particular, these confidence intervals are descriptive, not evidence that ordering \emph{causes} improved learning. + +\section{Discussion and Threats to Validity} + \textbf{The novel constrained-window result limits the earlier model ranking.} On the newly untouched 2015 cohort, both $K=1$ and $K=20$ evaluations favored an order-aware, interpretable logistic predictor in Brier score over the tested GRU. This contrasts with the earlier, non-information-matched enriched-PFA-versus-GRU comparison, but the new design does not prove that any one architecture is intrinsically better. Conditional on this fixed 2015 sample, performance depends on both the history budget and its representation, and the transparent model benefits particularly from explicit within-window recency and response-order proxies. Its own feature choice confers an inductive advantage, just as the recurrent model's greater capacity can do. A properly optimized AKT baseline, more neural seeds, extra split seeds, and different-year/institutional contexts are needed to assess robustness; none are falsely claimed here. + +The 2009 mirror reproductions show an association between the richer sequence representation of the GRU and better test probability scores under the fixed split. The new 2015 experiment provides a separate-year check: on a distinct learner sample, a converged richer PFA remains behind the GRU in the predetermined directional Brier comparison. This is stronger evidence for repeatability of the \emph{observed predictive direction} than the 2009 reanalysis alone, but cannot isolate recurrent architecture from feature history, ordered signals, capacity, data processing, or optimization. + +\textbf{Source equivalence and chronological assumptions.} The official maintainers identify duplicated rows in an earlier 2009--2010 skill-builder source and provide a corrected student--problem representation \citep{assistmentsdata}. Our independently executed direct source comparison disproves strict equality between that corrected release and the Atomi mirror: 817 shared learner sequences differ and the mirror is shorter by 8,774 skill-labeled responses. The mirror retains user/skill/grade sequences, but lacks the original problem/event IDs and verified timestamps needed to attribute the differences or reconstruct a complete raw-event crosswalk. The 2009 benchmark results therefore \emph{remain mirror-specific}; the 2015 experiment is a separate-year check and does not repair this mismatch. The 2015 file has event-log identifiers and a different skill-builder code definition. Ordering by \texttt{log\_id} is an explicitly documented modeling convention, not proof that the measure captures every physical action time correctly. + +\textbf{Pilot and numerical-selection transparency.} The first 2015 run did not pass the PFA convergence gate. The second-stage design and LBFGS replacement were frozen only after that failure had been observed; disjoint learners prevent direct sample reuse but do not turn the stage-2 plan into prospective preregistration. Reporting both stages prevents a misleading impression of a single perfect untouched evaluation. The 2015 results reflect one second-stage sample and seed, not many external sites or extensive split-seed sensitivity analysis. + +\textbf{Information and attribution.} The enriched PFA uses compressed aggregate per-skill and across-skill histories; the GRU uses ordered prior interactions and a different parameter budget. Its superiority cannot be assigned specifically to recurrence. Existing correction-aware, cross-cohort, and information-controlled comparisons \citep{polydoros2026,padi2026} further constrain claims of novelty or state-of-the-art ranking. + +\textbf{Uncertainty and target populations.} The 2009 original bootstrap treats the learner-average Brier contrast; the extended 2009 and new 2015 paired bootstraps target the interaction-pooled contrast and resample learners. The interval for the second 2015 cohort is conditional on its sampled learners, fixed model choices, and event selection rules, and cannot substitute for multi-institutional validation. The new cohort concerns only the source's 100 most-used skill builders, not all learners or curricula. \textbf{Educational interpretation.} Neither dataset supplies a randomized teaching intervention; predictive skill probabilities are not a direct measure of learning benefit, fairness, or readiness for consequential student decisions. + +\section{Research Questions Revisited} + extbf{RQ6 (matched history budget):} The previously observed neural Brier advantage does not persist against the order-aware logistic comparator on the third 2015 learner cohort. At both one and 20 prior interactions, this transparent predictor scores a lower Brier loss on identical held-out targets, while all models improve with 20 events. This is a narrow, source- and protocol-conditional model-ranking observation. + +\textbf{RQ1:} The six-model baseline reproduces on the fixed mirror, conditional on its source transformation. \textbf{RQ2:} First within-learner skill occurrences and repetitions differ descriptively, but are not directly comparable learning stages. \textbf{RQ3:} Learner-averaged and interaction-pooled Brier targets differ; their bootstrap intervals must be labeled accordingly. \textbf{RQ4:} The exploratory 2009 GRU/PFA contrast persisted with extended training and richer classical history, but on inspected learners. \textbf{RQ5 (external replication):} The directional interaction-pooled Brier advantage replicated on 750 untouched held-out 2015 learners in a separately fixed, converged corrective-stage evaluation, with the limitations above. \textbf{RQ6 (corrected 2009 source):} Direct event-level audit refuted full equivalence with the transformed mirror, but a separate, exploratory analysis of the corrected raw source reproduced the direction of the predictive Brier difference on 625 held-out learners. + +\section{Conclusion} +The empirical contribution of this study is not a new KT architecture or a claim that old GRU models dominate PFA. Instead, it exposes three measurable evaluation dependencies: (i) the common 2009 transformed mirror is not exactly identical to the corrected official event source; (ii) a recurrent predictor can appear stronger when compared with a baseline supplied with a coarser summary of learner history; and (iii) on a new, explicitly disjoint 2015 sample, holding available history and prediction targets fixed permits an interpretable order-aware logistic model to outperform a GRU in probability accuracy at both one and 20 previous responses. Increasing available history improved all three tested representations. The disagreement with earlier non-information-matched comparisons supports \emph{conditional, representation-aware benchmarking} rather than universal model rankings. The study does not establish effects on education, prove recurrence is useless, or claim to invent information parity or historical source correction. Multiple split seeds, neural seeds, modern baselines and additional contemporary datasets remain important follow-ups before stronger generalization claims. + +\section*{Data availability and research integrity} +The official 2009 correction notice is at \url{https://sites.google.com/site/assistmentsdata/home/2009-2010-assistment-data/skill-builder-data-2009-2010}, and the 2015 source at \url{https://sites.google.com/site/assistmentsdata/datasets/2015-assistments-skill-builder-data}. The 2009 pinned Atomi mirror and source SHA-256 are recorded in \url{https://github.com/devissaputra/knowledge_tracing_benchmark}. The re-execution of the 2009 frozen experiment appears in GitHub Actions run \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39655888620}{39655888620}. The first 2015 diagnostic pilot and final disjoint-cohort execution are in runs \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39754046492}{39754046492} and \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39754854426}{39754854426}, respectively. The frozen 2015 protocols, exact SHA-256, implementation and JSON results are in \url{https://github.com/devissaputra/knowledge_tracing_benchmark/pull/2}; the original exploratory extension is in \url{https://github.com/devissaputra/knowledge_tracing_benchmark/pull/1}. Raw individual learner records are not redistributed. The new Stage-3 results were independently executed in GitHub Actions with strict source/cohort checks and de-identified aggregate evidence; one release assertion about split learners versus evaluable learners was corrected without model retuning. External peer review and author approval are not claimed. Full Stage-3 evidence: \url{https://github.com/devissaputra/knowledge_tracing_benchmark/tree/research/kt-methodological-2026-history-budget/results}. Strict whole-dataset 2009 corrected-source parity was directly tested and disproved by necessary event-count and sequence-equality checks. + +\section*{Limitations of authorship and interpretation} +This is a research manuscript prepared for author verification. The named author must personally verify the calculations, code, references, and contribution statements before submission. The text makes no claim of peer review or acceptance. Draft preparation and some analytical implementation involved generative AI assistance; any submitted version requires the named author's personal audit and approval of the methods, empirical provenance, interpretations, and disclosure in accordance with the destination archive's applicable policies. + +\begingroup\small +\setlength{\itemsep}{0pt} +\begin{thebibliography}{99} +\bibitem[Abdelrahman et al.(2023)]{abdelrahman2023} Abdelrahman, G., Wang, Q., and Nunes, B. (2023). Knowledge tracing: A survey. \emph{ACM Computing Surveys}, 55(11), Article 224. \url{https://doi.org/10.1145/3569576}. +\bibitem[Corbett and Anderson(1995)]{corbett1995} Corbett, A. T. and Anderson, J. R. (1995). Knowledge tracing: Modeling the acquisition of procedural knowledge. \emph{User Modeling and User-Adapted Interaction}, 4, 253--278. \url{https://doi.org/10.1007/BF01099821}. +\bibitem[Feng et al.(2009)]{feng2009} Feng, M., Heffernan, N. T., and Koedinger, K. R. (2009). Addressing the assessment challenge with an online system that tutors as it assesses. \emph{User Modeling and User-Adapted Interaction}, 19(3), 243--266. \url{https://doi.org/10.1007/s11257-009-9063-7}. +\bibitem[Gervet et al.(2020)]{gervet2020} Gervet, T., Koedinger, K., Schneider, J., and Mitchell, T. (2020). When is deep learning the best approach to knowledge tracing? \emph{Journal of Educational Data Mining}, 12(3), 31--54. \url{https://doi.org/10.5281/zenodo.4143614}. +\bibitem[Polydoros(2026)]{polydoros2026} Polydoros, G. (2026). Cross-Cohort Knowledge-Tracing Benchmark: ASSISTments 2009--2010 and 2015 Replication Package (v2.0.1, corrected statistical reporting). Zenodo software. \url{https://doi.org/10.5281/zenodo.22131530}. +\bibitem[Badran and Preisach(2026)]{badran2026} Badran, Y. and Preisach, C. (2026). Enhancing knowledge tracing through leakage-free and recency-aware embeddings. \emph{SN Computer Science}, 7, article 774. \url{https://doi.org/10.1007/s42979-026-05344-3}. +\bibitem[Padi et al.(2026)]{padi2026} Padi, P., Morampudi, A., Irrinki, U. S. G., and Kakitapelli, P. K. D. (2026). Stable and faithful explanations for knowledge tracing. arXiv:2609.28502. \url{https://arxiv.org/abs/2609.28502}. +\bibitem[Pavlik et al.(2009)]{pavlik2009} Pavlik, P. I., Cen, H., and Koedinger, K. R. (2009). Performance factors analysis: A new alternative to knowledge tracing. \emph{Proceedings of AIED}, 531--538. \url{https://doi.org/10.3233/978-1-60750-028-5-531}. +\bibitem[Piech et al.(2015)]{piech2015} Piech, C., Bassen, J., Huang, J., Ganguli, S., Sahami, M., Guibas, L., and Sohl-Dickstein, J. (2015). Deep knowledge tracing. \emph{NeurIPS 28}. \url{https://arxiv.org/abs/1506.05908}. +\bibitem[ASSISTmentsData(2015)]{assistments2015} ASSISTmentsData (2015). 2015 ASSISTments Skill Builder Data, original dataset portal (100 skill builders). \url{https://sites.google.com/site/assistmentsdata/datasets/2015-assistments-skill-builder-data}. +\bibitem[ASSISTmentsData(n.d.)]{assistmentsdata} ASSISTmentsData (n.d.). Skill-builder data 2009--2010, correction notice and official data. \url{https://sites.google.com/site/assistmentsdata/home/2009-2010-assistment-data/skill-builder-data-2009-2010}. +\end{thebibliography} +\endgroup +\end{document} \ No newline at end of file From 06f0f37020f4f9cb81ce8cc9a3dac735a60d3ef6 Mon Sep 17 00:00:00 2001 From: Devis Wawan Saputra Date: Mon, 26 Oct 2026 18:01:17 +0700 Subject: [PATCH 44/48] paper: archive numerical and clean-room manuscript QA 41 checks --- paper/METHODOLOGICAL_2026_FINAL_QA.json | 181 ++++++++++++++++++++++++ 1 file changed, 181 insertions(+) create mode 100644 paper/METHODOLOGICAL_2026_FINAL_QA.json diff --git a/paper/METHODOLOGICAL_2026_FINAL_QA.json b/paper/METHODOLOGICAL_2026_FINAL_QA.json new file mode 100644 index 0000000..d5c43ba --- /dev/null +++ b/paper/METHODOLOGICAL_2026_FINAL_QA.json @@ -0,0 +1,181 @@ +{ + "date": "2026-10-26", + "scope": "Internal computation/transcription/document QA, not outside peer review", + "checks_total": 41, + "passed": 41, + "failed": 0, + "checks": [ + { + "name": "Official 2015 CSV SHA-256 confirmed", + "passed": true + }, + { + "name": "Fresh cohort has no overlap with first/second sample", + "passed": true + }, + { + "name": "Correct stage3 learner splits", + "passed": true + }, + { + "name": "Result scientifically completed", + "passed": true + }, + { + "name": "Both history budgets present", + "passed": true + }, + { + "name": "Same 20592 targets across both budgets", + "passed": true + }, + { + "name": "582 evaluable history-eligible test learners", + "passed": true + }, + { + "name": "2k learner cluster resamples", + "passed": true + }, + { + "name": "No logistic convergence warnings", + "passed": true + }, + { + "name": "All six models report bounded metrics", + "passed": true + }, + { + "name": "Manuscript Brier 1 window_count", + "passed": true + }, + { + "name": "Manuscript AUC 1 window_count", + "passed": true + }, + { + "name": "Manuscript Brier 1 ordered_logistic", + "passed": true + }, + { + "name": "Manuscript AUC 1 ordered_logistic", + "passed": true + }, + { + "name": "Manuscript Brier 1 window_gru", + "passed": true + }, + { + "name": "Manuscript AUC 1 window_gru", + "passed": true + }, + { + "name": "Positive GRU-minus-ordered contrast for K=1", + "passed": true + }, + { + "name": "Paired interval excludes zero positive at K=1", + "passed": true + }, + { + "name": "Manuscript Brier 20 window_count", + "passed": true + }, + { + "name": "Manuscript AUC 20 window_count", + "passed": true + }, + { + "name": "Manuscript Brier 20 ordered_logistic", + "passed": true + }, + { + "name": "Manuscript AUC 20 ordered_logistic", + "passed": true + }, + { + "name": "Manuscript Brier 20 window_gru", + "passed": true + }, + { + "name": "Manuscript AUC 20 window_gru", + "passed": true + }, + { + "name": "Positive GRU-minus-ordered contrast for K=20", + "passed": true + }, + { + "name": "Paired interval excludes zero positive at K=20", + "passed": true + }, + { + "name": "New figure exists", + "passed": true + }, + { + "name": "Multiple comparison caveat", + "passed": true + }, + { + "name": "Late arxiv file includes AI-assistance caveat", + "passed": true + }, + { + "name": "No novel architect invention claim", + "passed": true + }, + { + "name": "Inherited figure kt_auc.png present", + "passed": true + }, + { + "name": "Inherited figure kt_brier.png present", + "passed": true + }, + { + "name": "Inherited figure robust_comparison.png present", + "passed": true + }, + { + "name": "arXiv archive ZIP integrity", + "passed": true + }, + { + "name": "Clean-room pdflatex pass 1", + "passed": true + }, + { + "name": "Clean-room pdflatex pass 2", + "passed": true + }, + { + "name": "Final PDF contains 20,592 evaluation count", + "passed": true + }, + { + "name": "Final PDF contains new method section", + "passed": true + }, + { + "name": "No LaTeX overfull warning", + "passed": true + }, + { + "name": "No unresolved citations", + "passed": true + }, + { + "name": "Final manuscript PDF hash matches clean-room compiled PDF", + "passed": true + } + ], + "source_sha256": "75e1f46131d5897a388c82411bc3032279899cb1f1e66a9043354d41234407a0", + "stage3_results_sha256": "e9a8cba68c2cb5b945d3a286aa27c0bfdc24b8841cd5052badc5ea4ac2e7381d", + "arxiv_source_sha256": "6d15e720f8bb613e40d9f4c1e6e4f9118d162092125dc749122f92cc2edcb6ff", + "pdf_sha256": "b0783067f84ac3c5b1f6b478f673e59465d6ecc2655833a0443831af4546609a", + "author_approval": false, + "external_peer_review": false, + "modern_independent_source_validation": false, + "scientific_caveat": "Single stage-3 split and GRU seed; matched input-stream availability does not match parameter count or optimization; descriptive bootstrap comparisons and 2009 source audit only" +} \ No newline at end of file From 136696a004c5eb2d8476182241381197e20b1e2c Mon Sep 17 00:00:00 2001 From: Devis Wawan Saputra Date: Mon, 26 Oct 2026 18:02:12 +0700 Subject: [PATCH 45/48] figures: regenerate matched-history budget Brier figure from verified cohort output --- paper/generate_figures.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/paper/generate_figures.py b/paper/generate_figures.py index 565d1c3..93ec4e1 100644 --- a/paper/generate_figures.py +++ b/paper/generate_figures.py @@ -65,4 +65,24 @@ def bars(items,metric,title,filename,ylim=None): bars([('Enriched PFA (converged)',ms['enriched_pfa']),('Extended GRU',ms['extended_gru'])], 'brier','Fresh ASSISTments 2015: 750 learners; 26,697 held-out responses', 'cohort2015_brier.png',ylim=(0,.23)) -print('Generated four manuscript figures from executed JSON.') \ No newline at end of file +print('Generated four manuscript figures from executed JSON.') + +# 2026 matched-history-budget figure, recomputed from executed aggregate evidence. +budget=read("results/matched_history_budget_2026.json") +assert budget["status"]=="passed_all_computational_gates" +assert budget["source"]["cohort_overlap_previous"]==0 +fig,ax=plt.subplots(figsize=(7.25,3.35)) +for name,label in [("window_count","Window-count PFA+"),("ordered_logistic","Order-aware logistic"),("window_gru","Window-GRU")]: + y=[budget["comparisons"][str(k)]["models"][name]["brier"] for k in (1,20)] + ax.plot([1,20],y,marker="o",linewidth=1.8,markersize=6,label=label) + for k,v in zip((1,20),y): + ax.annotate(f"{v:.4f}",(k,v),xytext=(3,7 if name!="window_count" else -14),textcoords="offset points",fontsize=7) +ax.set_xticks([1,20],["1 prior event","20 prior events"]) +ax.set_ylabel("Interaction-pooled test Brier score (lower better)") +ax.set_xlabel("Information budget per next-response prediction") +ax.grid(axis="y",alpha=.27) +ax.legend(loc="upper right",fontsize=8,frameon=False) +fig.tight_layout() +fig.savefig(OUT/"history_budget_brier.png",dpi=190) +plt.close(fig) +print("Generated new 2026 matched-budget figure.") From 901ba2de98c8cb9d22f94a5d34ebe640e7350c53 Mon Sep 17 00:00:00 2001 From: Devis Wawan Saputra Date: Mon, 26 Oct 2026 18:03:41 +0700 Subject: [PATCH 46/48] paper: document actual fresh-cohort matched-window findings, corrected QA and limitations --- paper/METHODOLOGICAL_2026_EXECUTED_RESULTS.md | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 paper/METHODOLOGICAL_2026_EXECUTED_RESULTS.md diff --git a/paper/METHODOLOGICAL_2026_EXECUTED_RESULTS.md b/paper/METHODOLOGICAL_2026_EXECUTED_RESULTS.md new file mode 100644 index 0000000..5ae140e --- /dev/null +++ b/paper/METHODOLOGICAL_2026_EXECUTED_RESULTS.md @@ -0,0 +1,41 @@ +# Executed 2026 matched-history-budget results — 26 October 2026 + +**Scientific outcome:** An interpretable order-aware logistic regression scored better Brier loss than the compact GRU on the same next-response targets under both fixed observable history budgets. The earlier GRU-over-PFA result does not generalize to every classical baseline under matched input stream availability. + +The official 2015 source SHA-256 was verified; stage 3 excluded the full 5,000-learner stage-1 sample and full 5,000-learner stage-2 sample. Of 19,840 source-eligible students, 9,840 were eligible for further sampling and 4,000 were selected for stage 3 (138,129 source events). Learner splits: 2,800 train / 600 validation / 600 test. **582** test learners produced **20,592** eligible targets at t>=1. The other 18 test learners had no post-first response and were properly excluded from next-response metrics. + +## Actual same-target results + +| Previous history K | Model | Brier (lower) | ROC-AUC (higher) | Log loss | ECE-10 | +|---|---|---:|---:|---:|---:| +| 1 | Count-based PFA+ | 0.177882 | 0.684440 | 0.534589 | 0.009320 | +| 1 | Order-aware logistic | **0.177882** | 0.684436 | 0.534589 | 0.009321 | +| 1 | GRU | 0.180003 | 0.670909 | 0.540658 | 0.005781 | +| 20 | Count-based PFA+ | 0.176674 | 0.691058 | 0.531590 | 0.006126 | +| 20 | Order-aware logistic | **0.173885** | 0.705044 | 0.524095 | 0.007251 | +| 20 | GRU | 0.175140 | 0.699361 | 0.527153 | 0.011658 | + +Paired learner-cluster bootstrap, 2,000 draws: **GRU minus order-aware Brier K=1 +0.002121** (95% interval [0.001198,0.003013]), **K=20 +0.001255** ([0.000075,0.002426]). GRU-minus-count at K=20 is **−0.001533**, interval [−0.002761, −0.000279], demonstrating the difference that comparator choice makes. + +Improvements in Brier from K=1 to K=20: count-based 0.001208, order-aware 0.003997, GRU 0.004863. Interpret intervals descriptively: fixed data split/model fits, one GRU seed, no multiple-comparison correction. + +## Audited execution status + +- New protocol committed **before evaluating stage-3 cohort**: [frozen protocol](../METHODOLOGICAL_2026_PROTOCOL.md) +- [Python implementation](../scripts/matched_history_budget_2026.py) +- [Real-data, checksum-verified and learner-disjoint aggregate JSON](../results/matched_history_budget_2026.json) +- [Successful GitHub Actions run 40310733288](https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/40310733288). +- [Final 13-page author-review manuscript](main_methodological_2026.tex). +- [41 internal manuscript and numerical QA checks](METHODOLOGICAL_2026_FINAL_QA.json). +- [Figure generator](generate_figures.py). +- User-facing final PDF, minimal arXiv ZIP, and complete evidence package are preserved in ChatGPT Library folder `/Paper1_Methodological_2026`. + +### Transparency on prior release assertion +The initial run 40310088864 completed all model training, both history horizons and aggregate output. Its release gate nevertheless failed on a mistaken assertion requiring 600 **evaluated** learners, confusing split members with learners having t>=1 targets. The corrected check permits 582 observed evaluable test learners and verifies exactly equal event counts across models, without changing any data, model, tuning, or observed result. The final run succeeds. + +### Contribution boundaries +The study is **not** the first knowledge-tracing information-parity comparison, not the first source correction study and not a model architecture innovation. Polydoros (2026, DOI 10.5281/zenodo.22131530), Padi et al. (2026, arXiv:2609.28502), and Badran & Preisach (2026, DOI 10.1007/s42979-026-05344-3) are material prior work. The incremental contribution is a frozen, newly nonoverlapping 2015 cohort with explicitly constrained K=1 versus K=20 history budgets and order-feature comparator, interpreted alongside independently verified 2009 source mismatch. + +There is no basis for causal effect on educational outcomes, current schooling deployment, universal superiority of logistic models, or cross-year frozen-weight transfer. Next strengthening steps would be independent contemporary datasets, multiple splits/seeds and attention baselines; **none are claimed completed**. + +**Status:** Audited working preprint for named human author's scientific approval, not externally peer reviewed and not submitted to arXiv. From 28817c0333d0fba1274d558a8a577b2215840a8f Mon Sep 17 00:00:00 2001 From: Devis Wawan Saputra Date: Mon, 26 Oct 2026 18:03:46 +0700 Subject: [PATCH 47/48] paper: mark pending-results draft as archived, point to executed manuscript --- paper/METHODOLOGICAL_2026_UPGRADE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paper/METHODOLOGICAL_2026_UPGRADE.md b/paper/METHODOLOGICAL_2026_UPGRADE.md index 3838130..9231195 100644 --- a/paper/METHODOLOGICAL_2026_UPGRADE.md +++ b/paper/METHODOLOGICAL_2026_UPGRADE.md @@ -1,4 +1,4 @@ -# Toward a History-Budget Sensitivity Audit for Knowledge Tracing: Source Provenance, Information-Restricted Baselines, and Cross-Year Evidence +> **ARCHIVED DRAFT NOTICE (26 October 2026):** This file preserves the design-stage manuscript with PENDING fields. Its outcomes are now executed and documented in [METHODOLOGICAL_2026_EXECUTED_RESULTS.md](METHODOLOGICAL_2026_EXECUTED_RESULTS.md) and the complete [main_methodological_2026.tex](main_methodological_2026.tex). Do not submit this archived draft as the final paper.\n\n# Toward a History-Budget Sensitivity Audit for Knowledge Tracing: Source Provenance, Information-Restricted Baselines, and Cross-Year Evidence **26 October 2026 — research draft, not a completed new experiment or a submission-ready replacement.** The numerical results of the newly frozen Stage-3 history-budget study must be inserted only after independent runner success and author QA. Earlier verified numerical results below are historical evidence and were already used for earlier preprint drafts. From a8a53cc22afceb482f301b609c6156dca11e1722 Mon Sep 17 00:00:00 2001 From: Devis Wawan Saputra Date: Mon, 26 Oct 2026 20:32:44 +0700 Subject: [PATCH 48/48] paper: final 2026 author-review QA, verified DOI corrections and natural academic rewrite --- paper/main_methodological_2026.tex | 56 +++++++++++++++--------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/paper/main_methodological_2026.tex b/paper/main_methodological_2026.tex index e142ab6..f5695d0 100644 --- a/paper/main_methodological_2026.tex +++ b/paper/main_methodological_2026.tex @@ -1,36 +1,42 @@ \documentclass[11pt,a4paper]{article} -\usepackage[margin=0.84in]{geometry} +\usepackage[margin=0.74in]{geometry} \usepackage[T1]{fontenc} \usepackage[utf8]{inputenc} \usepackage{lmodern,microtype} \usepackage{amsmath,amssymb,booktabs,array,graphicx,caption,subcaption} \usepackage[colorlinks=true,urlcolor=blue,citecolor=blue,linkcolor=blue]{hyperref} \usepackage{natbib} +\hypersetup{pdftitle={When Knowledge-Tracing Rankings Depend on Available History},pdfauthor={Devis Wawan Saputra}} \sloppy\emergencystretch=4em \setlength{\parskip}{0.14em}\renewcommand{\baselinestretch}{0.96} \setlength{\parindent}{0pt} \title{When Knowledge-Tracing Rankings Depend on Available History\\\large A Source-Correction and Matched-Information Audit on ASSISTments 2009--2015} \author{Devis Wawan Saputra\\\small Independent Researcher, Indonesia\\\small ORCID: \href{https://orcid.org/0000-0002-7133-9410}{0000-0002-7133-9410}} -\date{26 October 2026\\\small Revised empirical methodological manuscript; not peer reviewed} +\date{26 October 2026} \begin{document} \maketitle \begin{abstract} -Predictive comparisons between transparent and recurrent knowledge-tracing models can reflect unequal access to response history, transformations of historical datasets, or feature representation rather than recurrent architecture alone. We combine corrected-source provenance checks with a frozen, matched-history-budget test on independently sampled ASSISTments 2015 learners. A direct audit rejects exact equivalence between a frequently used transformed ASSISTments 2009 mirror (4,148 learners; 274,331 labeled responses) and the corrected official release: 817 shared learners have different normalized skill--response sequences. Earlier re-executions and a new 2015 cohort had favored GRU over count-based PFA, but supplied different history representations. For a new 2015 experiment, we excluded all 10,000 learners in the previous two evaluations and sampled 4,000 untouched learners. On the same 20,592 held-out next-response targets from 582 evaluable learners, we compared window-count logistic regression, an order-aware logistic baseline, and a window-GRU, each restricted to only the preceding one or 20 labeled interactions and the current skill identifier. With a one-event history, Brier scores were 0.17788, 0.17788, and 0.18000, respectively; with 20 events, they were 0.17667, 0.17389, and 0.17514. The GRU-minus-order-aware Brier difference was +0.00212 (95\% learner-cluster interval [+0.00120,+0.00301]) for one event and +0.00125 ([+0.00008,+0.00243]) for 20. These descriptive comparisons reverse the apparent recurrent advantage seen against less expressive history summaries, while longer history improves all three models. The evidence identifies sensitivity to observable-history budget and model specification, \emph{not} the causal contribution of recurrence, modern classroom effectiveness, or universal superiority. The 2015 window analysis uses one split and neural seed and should be replicated on additional domains. +Knowledge-tracing benchmarks often compare models that have different access to a learner's previous responses. We examine how data provenance, history length, and representation affect those comparisons. First, we compare a pinned ASSISTments 2009 sequence mirror against the corrected official release. Of 4,148 learners represented in both, 817 have different ordered skill--response sequences; the mirror also contains fewer labeled events. Second, we retain earlier learner-disjoint model evaluations on the 2009 and 2015 datasets, distinguishing exploratory results from a corrective 2015 replication. Finally, we evaluate a new, nonoverlapping sample of 4,000 ASSISTments 2015 learners using three models constrained to the same one-event or 20-event prior-response window. On 20,592 test responses from 582 learners, the 20-event Brier scores were 0.17667 for count-based logistic regression, 0.17389 for an order-aware logistic model, and 0.17514 for a GRU. The GRU-minus-order-aware difference was 0.00125 (95\% learner-cluster bootstrap interval [0.00008, 0.00243]); at one prior event, the corresponding difference was 0.00212 [0.00120, 0.00301]. Earlier comparisons had favored the GRU against simpler history summaries. The newer analysis instead favors a logistic comparator that represents order using the same permitted event stream. These results concern predictive performance on particular historical cohorts and model settings. They neither isolate an architectural effect nor establish generalization to contemporary classrooms. + \end{abstract} \textbf{Keywords:} AI in education; knowledge tracing; ASSISTments; leakage-aware evaluation; BKT; PFA; recurrent networks; information parity; history window; model ranking sensitivity; reproducibility. \section{Introduction} -Knowledge tracing predicts a learner's subsequent response from observed practice history. The probabilistic outputs can be used to rank cases for inspection and to support adaptive content, but predictive validity must not be conflated with instructional value. A model that has learned response regularities on a historical tutoring platform may still fail under new curricula, item policies, or student populations. +Knowledge tracing predicts a student's next response from earlier practice. Binary correctness histories allow comparisons among simple frequency estimates, probabilistic mastery models, logistic models, and neural sequence encoders. Such comparisons are useful only if the underlying event records and the information supplied to each predictor are described accurately. -A second challenge concerns experimental comparison. A random row split can let responses from the same learner appear in both model development and evaluation. Such a split evaluates interpolation within known learner histories rather than the intended generalization to unseen learners. Another challenge is information symmetry: a recurrent model may use prior outcomes from other skills while a classical per-skill model sees only within-skill history. Predictive differences then reflect both representation and architecture. +Two problems recur in published benchmarks. Datasets distributed under the same name may reflect different filtering and serialization decisions. Moreover, a recurrent predictor may see answers on other skills and their order, while a conventional performance-factors model sees counts for the queried skill. A performance difference in that setting combines model specification with differences in available history. It cannot be attributed to recurrence alone. -Rather than introduce a new model, this study audits the empirical scope of a controlled learner-disjoint benchmark, explicitly recording probabilistic performance, slice results, calibration, and uncertainty limits. It answers: (RQ1) How do conventional baselines compare on a frozen learner-disjoint partition? (RQ2) Does prediction differ between a learner's first observed and later encounters with a skill? (RQ3) What do the resampling intervals establish under a fixed learner split? (RQ4) Do extended training and a richer transparent baseline materially change the results, and what remains untested? (RQ5) Does the direction of the predictive difference replicate on an independent-year cohort? (RQ6) Holding the learner sample, response targets and observable history restricted, does representation of that history change the apparent advantage of recurrence, and how do predictions change between K=1 and K=20 prior events? +We investigate these problems using the ASSISTments 2009--2010 and 2015 Skill Builder releases. We first document what changes between a widely used transformed 2009 mirror and the source-corrected official records. We then retain the earlier held-out results, including a failed optimization diagnostic and its independently sampled follow-up. The main new experiment restricts all models to the same prior-response window and scores their predictions on precisely the same held-out response events. We ask whether the GRU advantage reported against a count-based baseline persists after adding explicit order information to an interpretable logistic model, and whether that comparison changes between windows of one and 20 previous events. \section{Related Work and Positioning} -BKT recursively updates a latent mastery probability using learning-transition, guess, and slip assumptions \citep{corbett1995}. PFA predicts responses with logistic effects of skill identity and previous successes and failures \citep{pavlik2009}. Deep Knowledge Tracing introduced recurrent sequence representations \citep{piech2015}. Later evaluations demonstrated that claims of deep-model superiority are highly contingent on datasets, baselines, and evaluation design \citep{gervet2020}. Broader surveys emphasize inconsistent versions, preprocessing, and metric protocols across commonly named knowledge-tracing datasets \citep{abdelrahman2023}. More recent work directly addresses correction-aware and cross-cohort benchmarking. A 2026 replication package by Polydoros compares classical and deep approaches on the corrected ASSISTments 2009 and 2015 cohorts with learner-level resampling \citep{polydoros2026}. Padi and colleagues (2026) show empirically that duplicate multi-skill records and asymmetries in the information available to different model families can change reported rankings \citep{padi2026}. Badran and Preisach (2026) independently study leakage-free embeddings and skill-specific recency, establishing that these methodological ideas already appear in contemporary work \citep{badran2026}. These advances preclude claiming novelty for learner-disjoint splitting, calibration analysis, or information matching in isolation. They also provide corrected-source and cross-cohort comparators, so we make no state-of-the-art or architecture-advantage claims. The distinctive contribution here is the auditable sequence of a frozen transformed-mirror reproduction, contrasting uncertainty targets, an explicitly labeled exploratory information-budget audit, and a separate-year predictive replication that retains rather than conceals a nonconverged first pilot. Unlike a model-development paper, this investigation treats exact source provenance, estimand distinctions, and explicit failure of source parity as empirical audit findings rather than claiming a state-of-the-art architecture. +Bayesian Knowledge Tracing (BKT) represents learning through an evolving latent mastery probability \citep{corbett1995}. Performance Factors Analysis (PFA) uses previous successes and failures as predictors in a logistic model \citep{pavlik2009}; Deep Knowledge Tracing introduced recurrent sequence encoders \citep{piech2015}. Later studies and surveys found that comparisons vary with datasets, splits, preprocessing, and baseline implementation \citep{gervet2020,abdelrahman2023}. + +This is an active methodological literature. A 2026 cross-cohort replication examines BKT, PFA, and neural methods using corrected ASSISTments releases \citep{polydoros2026}. Padi and colleagues test information-matched prediction and identify leakage associated with duplicated skill records \citep{padi2026}. Badran and Preisach examine leakage-free and recency-aware sequence representations \citep{badran2026}. Thus, correcting historical data and equalizing input information are not novel ideas by themselves. Our narrower test is a frozen history-window comparison on a new 2015 learner sample, reported alongside a concrete check of the 2009 mirror's provenance. We do not claim a new knowledge-tracing architecture or a comprehensive ranking of existing models. + +\section{Study Design and Evidence} +The results come from three related but distinct analyses. The original 2009 study reproduces six models on a fixed learner-level split of a transformed mirror. Additional 2009 analyses vary PFA features and GRU training; since their test set had already been inspected, these are exploratory. A separate audit compares the mirror with the corrected official source and refits PFA and GRU to that source. -\section{Study Contribution and Evidence Hierarchy} -We provide six bounded outputs: (1) an executable source- and learner-split-pinned reproduction of six models and model-specific probability metrics on the transformed 2009 mirror; (2) a distinction between equal-learner and interaction-pooled Brier uncertainty targets; (3) an exploratory same-split sensitivity analysis comparing cross-skill PFA summary features with extended GRU training; and (4) a direct event-count and ordered-sequence parity audit against the official corrected 2009 source; (5) an exploratory official corrected-2009 source sensitivity analysis; and (6) a documented independent-year 2015 predictive replication on a distinct set of learners. The two evidence tiers must not be conflated: post-inspection 2009 improvements are exploratory, while the second 2015 cohort is fresh with respect to learner identity and outcome inspection. The second-stage 2015 protocol was nonetheless fixed \emph{after} observing a failed optimizer convergence diagnostic in the first 2015 pilot; its corrective choice and chronology are disclosed, and the result is not described as prospectively registered or independently peer-reviewed. These analyses compare unequal information representations and model capacity; they cannot identify a recurrence-only effect. (7) We add a newly frozen third ASSISTments 2015 learner sample, disjoint from both earlier samples, comparing three predictors at two \emph{matched available history} budgets. Unlike the earlier pilot and exploratory comparisons, these model/feature windows were fixed before evaluating the new learner cohort. The upgrade tests the dependency of model ranking on representation and history horizon; it does not invent information-parity benchmarking. +The 2015 study proceeded in stages. A first 5,000-learner sample produced a nonconverged PFA fit. A second, nonoverlapping 5,000-learner sample used a corrected solver and provided a new held-out comparison. After those results were available, we fixed the protocol for a third sample of 4,000 learners, excluding everyone from the first two samples. That third analysis, comparing one- and 20-event history windows, is the present paper's central methodological experiment. Its design was recorded before examining the third sample's results, although the choice of research question followed earlier analyses and is not a public preregistration. \section{Data and Evaluation Protocol} \subsection{Dataset origin and observed differences from the corrected source} @@ -66,7 +72,7 @@ \subsection{Evaluation, uncertainty, and estimands} \end{equation} That is a defensible \emph{different} estimand. It must not be interpreted as an uncertainty interval for the event-pooled Brier difference. A paired learner-cluster bootstrap for the event-weighted difference resamples learners and recomputes the ratio of summed losses to summed response counts in each replicate. The executed supplement implements this correction, with 2,000 learner-resampling draws per comparison. Each draw recomputes the ratio of the sampled cluster loss sums to the sampled cluster response counts; the resulting percentile intervals are conditional on this one previously examined test partition. -\section{Reproduction and Exploratory Robustness Results} +\section{2009 Benchmark and Source Sensitivity} \subsection{Overall held-out comparisons} Table~\ref{tab:results} reports the results from a new full execution of the original study on the pinned real-data mirror. Reproduced original values agree with the archived record to within $4\times10^{-8}$ for ROC-AUC and much more closely for Brier score. \begin{table}[ht] @@ -205,7 +211,7 @@ \subsection{Real-data results: order-aware logistic reverses the previous model \label{fig:historybudget} \end{figure} -The best single model at both budgets was the \emph{order-aware logistic} model. With only one previous response, count and order-aware models are numerically nearly identical, as expected: the single response already determines the recent-history summary. At 20 prior events, explicit order features improve the logistic Brier score from 0.17667 to 0.17389; the corresponding GRU is 0.17514. Moving from $K=1$ to $K=20$, Brier improves by \textbf{0.00121} for window-count, \textbf{0.00400} for order-aware logistic and \textbf{0.00486} for GRU. The original GRU advantage over standard PFA thus does not universally carry over when a transparent ordered-history comparator is given the same observable response window. +The order-aware logistic model had the lowest Brier score at both history budgets. With only one previous response, count and order-aware models are numerically nearly identical, as expected: the single response already determines the recent-history summary. At 20 prior events, explicit order features improve the logistic Brier score from 0.17667 to 0.17389; the corresponding GRU is 0.17514. Moving from $K=1$ to $K=20$, Brier improves by \textbf{0.00121} for window-count, \textbf{0.00400} for order-aware logistic and \textbf{0.00486} for GRU. The original GRU advantage over standard PFA thus does not universally carry over when a transparent ordered-history comparator is given the same observable response window. At $K=1$, GRU minus order-aware logistic had pooled Brier difference $+0.002121$ with a 95\% paired learner-cluster bootstrap interval $[+0.001198,+0.003013]$. Relative to window-count PFA+, the corresponding GRU difference was $+0.002121$, interval $[+0.001198,+0.003013]$. @@ -213,34 +219,28 @@ \subsection{Real-data results: order-aware logistic reverses the previous model -For the new $K=20$ analysis, all logistic solvers reached numerical convergence without warnings, and the GRU's best validation epoch was 6 (both budgets). The preprocessing hash, nonoverlapping cohort, same-label target identity and numerical sanity gates passed. No family-wise significance claim or empirical generalization beyond the study's single held-out source is made. In particular, these confidence intervals are descriptive, not evidence that ordering \emph{causes} improved learning. +For both history budgets, the logistic solvers reported no convergence warnings. The GRU selected epoch 6 using validation loss. The preprocessing hash, nonoverlapping cohort, same-label target identity and numerical sanity gates passed. No family-wise significance claim or empirical generalization beyond the study's single held-out source is made. In particular, these confidence intervals are descriptive, not evidence that ordering \emph{causes} improved learning. \section{Discussion and Threats to Validity} - \textbf{The novel constrained-window result limits the earlier model ranking.} On the newly untouched 2015 cohort, both $K=1$ and $K=20$ evaluations favored an order-aware, interpretable logistic predictor in Brier score over the tested GRU. This contrasts with the earlier, non-information-matched enriched-PFA-versus-GRU comparison, but the new design does not prove that any one architecture is intrinsically better. Conditional on this fixed 2015 sample, performance depends on both the history budget and its representation, and the transparent model benefits particularly from explicit within-window recency and response-order proxies. Its own feature choice confers an inductive advantage, just as the recurrent model's greater capacity can do. A properly optimized AKT baseline, more neural seeds, extra split seeds, and different-year/institutional contexts are needed to assess robustness; none are falsely claimed here. - -The 2009 mirror reproductions show an association between the richer sequence representation of the GRU and better test probability scores under the fixed split. The new 2015 experiment provides a separate-year check: on a distinct learner sample, a converged richer PFA remains behind the GRU in the predetermined directional Brier comparison. This is stronger evidence for repeatability of the \emph{observed predictive direction} than the 2009 reanalysis alone, but cannot isolate recurrent architecture from feature history, ordered signals, capacity, data processing, or optimization. - -\textbf{Source equivalence and chronological assumptions.} The official maintainers identify duplicated rows in an earlier 2009--2010 skill-builder source and provide a corrected student--problem representation \citep{assistmentsdata}. Our independently executed direct source comparison disproves strict equality between that corrected release and the Atomi mirror: 817 shared learner sequences differ and the mirror is shorter by 8,774 skill-labeled responses. The mirror retains user/skill/grade sequences, but lacks the original problem/event IDs and verified timestamps needed to attribute the differences or reconstruct a complete raw-event crosswalk. The 2009 benchmark results therefore \emph{remain mirror-specific}; the 2015 experiment is a separate-year check and does not repair this mismatch. The 2015 file has event-log identifiers and a different skill-builder code definition. Ordering by \texttt{log\_id} is an explicitly documented modeling convention, not proof that the measure captures every physical action time correctly. - -\textbf{Pilot and numerical-selection transparency.} The first 2015 run did not pass the PFA convergence gate. The second-stage design and LBFGS replacement were frozen only after that failure had been observed; disjoint learners prevent direct sample reuse but do not turn the stage-2 plan into prospective preregistration. Reporting both stages prevents a misleading impression of a single perfect untouched evaluation. The 2015 results reflect one second-stage sample and seed, not many external sites or extensive split-seed sensitivity analysis. +\textbf{Model ranking depends on the comparator.} On the third 2015 cohort, adding order-related features to logistic regression reduced Brier loss from 0.17667 to 0.17389 at $K=20$, below the GRU's 0.17514. At $K=1$, the two logistic specifications are nearly identical, and both score below the GRU. This does not show that logistic regression is intrinsically more accurate than recurrent models. Their feature engineering, parameter counts, optimization, and inductive biases differ. It does show that a GRU advantage against a coarse count baseline need not survive comparison with a stronger, transparent representation of the same history. -\textbf{Information and attribution.} The enriched PFA uses compressed aggregate per-skill and across-skill histories; the GRU uses ordered prior interactions and a different parameter budget. Its superiority cannot be assigned specifically to recurrence. Existing correction-aware, cross-cohort, and information-controlled comparisons \citep{polydoros2026,padi2026} further constrain claims of novelty or state-of-the-art ranking. +\textbf{Dataset identity matters.} The 2009 transformed mirror contains fewer labeled records than the corrected official source, and 817 shared learner sequences differ. Models trained and evaluated on the mirror therefore describe that specific transformation. The separate corrected-source results are not a paired estimate of the effect of preprocessing: their splits and target events differ. The mirror lacks original problem/event IDs needed for an exact row-level crosswalk. -\textbf{Uncertainty and target populations.} The 2009 original bootstrap treats the learner-average Brier contrast; the extended 2009 and new 2015 paired bootstraps target the interaction-pooled contrast and resample learners. The interval for the second 2015 cohort is conditional on its sampled learners, fixed model choices, and event selection rules, and cannot substitute for multi-institutional validation. The new cohort concerns only the source's 100 most-used skill builders, not all learners or curricula. \textbf{Educational interpretation.} Neither dataset supplies a randomized teaching intervention; predictive skill probabilities are not a direct measure of learning benefit, fairness, or readiness for consequential student decisions. +\textbf{Inference remains conditional.} The 2009 exploratory revisions were made after inspecting the original holdout. In the first 2015 pilot, the PFA optimizer failed to converge; a revised solver was used on a disjoint second sample. The third protocol was fixed before its new learners were evaluated, but it was designed in light of the earlier experiments. Bootstrap intervals resample learners within the evaluated test split; they do not incorporate alternative training splits, tuning choices, multiple-testing adjustments, or shifts to other institutions. -\section{Research Questions Revisited} - extbf{RQ6 (matched history budget):} The previously observed neural Brier advantage does not persist against the order-aware logistic comparator on the third 2015 learner cohort. At both one and 20 prior interactions, this transparent predictor scores a lower Brier loss on identical held-out targets, while all models improve with 20 events. This is a narrow, source- and protocol-conditional model-ranking observation. +The 2015 tests also use \texttt{sequence\_id} as the unit being predicted and order records by \texttt{log\_id}. Those coding and ordering decisions may not capture every aspect of the original learning activity. The matched-window experiment uses one split, one GRU seed, and two history lengths. Repeating it with additional seeds, modern attention-based baselines, and a genuinely different source would show whether the ranking is stable. Finally, accurate prediction of historical correctness does not establish that using any of these models improves instruction, fairness, or learning outcomes. -\textbf{RQ1:} The six-model baseline reproduces on the fixed mirror, conditional on its source transformation. \textbf{RQ2:} First within-learner skill occurrences and repetitions differ descriptively, but are not directly comparable learning stages. \textbf{RQ3:} Learner-averaged and interaction-pooled Brier targets differ; their bootstrap intervals must be labeled accordingly. \textbf{RQ4:} The exploratory 2009 GRU/PFA contrast persisted with extended training and richer classical history, but on inspected learners. \textbf{RQ5 (external replication):} The directional interaction-pooled Brier advantage replicated on 750 untouched held-out 2015 learners in a separately fixed, converged corrective-stage evaluation, with the limitations above. \textbf{RQ6 (corrected 2009 source):} Direct event-level audit refuted full equivalence with the transformed mirror, but a separate, exploratory analysis of the corrected raw source reproduced the direction of the predictive Brier difference on 625 held-out learners. +\section{Summary of Findings} +The source audit rejects exact parity between the 2009 mirror and corrected release. The original six-model results reproduce on that mirror, but exploratory changes to model features or training cannot be treated as new confirmatory tests. The second 2015 sample reproduced the direction of the GRU--enriched-PFA Brier difference, conditional on the revised numerical solver and different model inputs. The third sample gives the clearest answer to the new question: with identical response targets and the same permitted event stream, an order-aware logistic model outperforms the tested GRU at both history lengths in Brier score. The contrast is small, especially at 20 events, and is specific to the implemented models and sample. \section{Conclusion} -The empirical contribution of this study is not a new KT architecture or a claim that old GRU models dominate PFA. Instead, it exposes three measurable evaluation dependencies: (i) the common 2009 transformed mirror is not exactly identical to the corrected official event source; (ii) a recurrent predictor can appear stronger when compared with a baseline supplied with a coarser summary of learner history; and (iii) on a new, explicitly disjoint 2015 sample, holding available history and prediction targets fixed permits an interpretable order-aware logistic model to outperform a GRU in probability accuracy at both one and 20 previous responses. Increasing available history improved all three tested representations. The disagreement with earlier non-information-matched comparisons supports \emph{conditional, representation-aware benchmarking} rather than universal model rankings. The study does not establish effects on education, prove recurrence is useless, or claim to invent information parity or historical source correction. Multiple split seeds, neural seeds, modern baselines and additional contemporary datasets remain important follow-ups before stronger generalization claims. +The transformed ASSISTments 2009 mirror is not equivalent to the corrected official source. The earlier GRU advantage on historical learner cohorts also depends on its comparison model: in a new 2015 evaluation with identical response targets, the order-aware logistic predictor achieved lower Brier loss than a compact GRU when both were given one or 20 preceding interactions. Allowing 20 previous responses improved all three tested models relative to a one-response window. The appropriate conclusion is that knowledge-tracing rankings are sensitive to source construction, history length, and representation. Stability across seeds, other datasets, and stronger neural baselines remains untested. \section*{Data availability and research integrity} -The official 2009 correction notice is at \url{https://sites.google.com/site/assistmentsdata/home/2009-2010-assistment-data/skill-builder-data-2009-2010}, and the 2015 source at \url{https://sites.google.com/site/assistmentsdata/datasets/2015-assistments-skill-builder-data}. The 2009 pinned Atomi mirror and source SHA-256 are recorded in \url{https://github.com/devissaputra/knowledge_tracing_benchmark}. The re-execution of the 2009 frozen experiment appears in GitHub Actions run \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39655888620}{39655888620}. The first 2015 diagnostic pilot and final disjoint-cohort execution are in runs \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39754046492}{39754046492} and \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/39754854426}{39754854426}, respectively. The frozen 2015 protocols, exact SHA-256, implementation and JSON results are in \url{https://github.com/devissaputra/knowledge_tracing_benchmark/pull/2}; the original exploratory extension is in \url{https://github.com/devissaputra/knowledge_tracing_benchmark/pull/1}. Raw individual learner records are not redistributed. The new Stage-3 results were independently executed in GitHub Actions with strict source/cohort checks and de-identified aggregate evidence; one release assertion about split learners versus evaluable learners was corrected without model retuning. External peer review and author approval are not claimed. Full Stage-3 evidence: \url{https://github.com/devissaputra/knowledge_tracing_benchmark/tree/research/kt-methodological-2026-history-budget/results}. Strict whole-dataset 2009 corrected-source parity was directly tested and disproved by necessary event-count and sequence-equality checks. +The 2009 corrected-source data and 2015 Skill Builder source are available from the ASSISTments data portal \citep{assistmentsdata,assistments2015}. Source checksums, preprocessing and model code, learner-sampling protocols, and aggregate results are available at \url{https://github.com/devissaputra/knowledge_tracing_benchmark}. The initial mirror reproduction, corrected-source audit, 2015 pilot and second sample, and third-sample analysis were executed in separate recorded GitHub Actions runs. The successful third-sample execution is \href{https://github.com/devissaputra/knowledge_tracing_benchmark/actions/runs/40310733288}{run 40310733288}; its first execution required correction of a release assertion that confused all test-split learners with evaluable learners. No model settings, targets, or cohort selections changed as part of that correction. Student-level source records are not redistributed. Code and manuscript revisions remain in \href{https://github.com/devissaputra/knowledge_tracing_benchmark/pull/3}{the research branch (draft pull request)}. -\section*{Limitations of authorship and interpretation} -This is a research manuscript prepared for author verification. The named author must personally verify the calculations, code, references, and contribution statements before submission. The text makes no claim of peer review or acceptance. Draft preparation and some analytical implementation involved generative AI assistance; any submitted version requires the named author's personal audit and approval of the methods, empirical provenance, interpretations, and disclosure in accordance with the destination archive's applicable policies. +\section*{Use of computational and writing tools} +Generative AI tools assisted with analysis-code development, research documentation, editing, and technical quality checks. The reported numerical results are linked to executed code and aggregate evidence. The named human author remains responsible for personally checking the methods, sources, computations, and claims before releasing this manuscript. \begingroup\small \setlength{\itemsep}{0pt}