diff --git a/act/main.py b/act/main.py index f5e07b3..4d8a08f 100644 --- a/act/main.py +++ b/act/main.py @@ -40,6 +40,11 @@ def _daysago_type(value: str) -> int: def _token_type(value: str) -> str: + # Allow empty for optional --targeting-token; --token stays required=True. + return value + + +def _required_token_type(value: str) -> str: if not value.strip(): raise argparse.ArgumentTypeError("token must be a non-empty string") return value @@ -69,9 +74,22 @@ def build_parser() -> argparse.ArgumentParser: ) parser.add_argument( "--token", - type=_token_type, + type=_required_token_type, required=True, - help="RedCap API token (required, non-empty)", + help=( + "Observational REDCap API token (BOOST_TOKEN) for reports 43327/41693" + ), + ) + parser.add_argument( + "--targeting-token", + type=_token_type, + default="", + dest="targeting_token", + help=( + "Intervention/Targeting REDCap API token for report 53846 " + "(defaults to TARGETING_TOKEN or DEBRIEF_FITNESS_TOKEN env). " + "Pass empty string to skip INT report." + ), ) parser.add_argument( "--daysago", @@ -106,12 +124,24 @@ def main(argv: list[str] | None = None) -> int: _configure_logging() args = build_parser().parse_args(argv) + targeting_token = ( + (args.targeting_token or "").strip() + or os.environ.get("TARGETING_TOKEN", "").strip() + or os.environ.get("DEBRIEF_FITNESS_TOKEN", "").strip() + ) + if not targeting_token: + logging.warning( + "No Targeting token (--targeting-token / TARGETING_TOKEN); " + "INT report 53846 will be skipped" + ) + p = Pipe( token=args.token, daysago=args.daysago, system=args.system, rebuild_manifest_only=args.rebuild_manifest_only, reconcile_manifest_only=args.reconcile_manifest_only, + targeting_token=targeting_token or None, ) try: diff --git a/act/tests/test_pipeline_smoke.py b/act/tests/test_pipeline_smoke.py index d5d9dab..a348bfa 100644 --- a/act/tests/test_pipeline_smoke.py +++ b/act/tests/test_pipeline_smoke.py @@ -252,6 +252,7 @@ def __init__( system, rebuild_manifest_only=False, reconcile_manifest_only=False, + targeting_token=None, ): call_state["pipe_args"] = { "token": token, @@ -259,6 +260,7 @@ def __init__( "system": system, "rebuild_manifest_only": rebuild_manifest_only, "reconcile_manifest_only": reconcile_manifest_only, + "targeting_token": targeting_token, } def run_pipe(self): @@ -287,6 +289,8 @@ def plot_session(self): _install_module(monkeypatch, "act.utils.pipe", pipe_mod) _install_module(monkeypatch, "act.utils.group", group_mod) + monkeypatch.setenv("TARGETING_TOKEN", "") + monkeypatch.setenv("DEBRIEF_FITNESS_TOKEN", "") monkeypatch.setattr( sys, "argv", @@ -312,6 +316,7 @@ def plot_session(self): "system": "local", "rebuild_manifest_only": False, "reconcile_manifest_only": False, + "targeting_token": None, } assert call_state["run_pipe"] == 1 assert call_state["group_systems"] == ["local", "person", "local", "session"] @@ -328,6 +333,7 @@ def __init__( system, rebuild_manifest_only=False, reconcile_manifest_only=False, + targeting_token=None, ): call_state["pipe_args"] = { "token": token, @@ -335,6 +341,7 @@ def __init__( "system": system, "rebuild_manifest_only": rebuild_manifest_only, "reconcile_manifest_only": reconcile_manifest_only, + "targeting_token": targeting_token, } def run_pipe(self): @@ -363,6 +370,8 @@ def plot_session(self): _install_module(monkeypatch, "act.utils.pipe", pipe_mod) _install_module(monkeypatch, "act.utils.group", group_mod) + monkeypatch.setenv("TARGETING_TOKEN", "") + monkeypatch.setenv("DEBRIEF_FITNESS_TOKEN", "") monkeypatch.setattr( sys, "argv", @@ -388,6 +397,7 @@ def plot_session(self): "system": "local", "rebuild_manifest_only": True, "reconcile_manifest_only": False, + "targeting_token": None, } assert call_state["run_pipe"] == 1 assert call_state["group_inits"] == 0 @@ -404,6 +414,7 @@ def __init__( system, rebuild_manifest_only=False, reconcile_manifest_only=False, + targeting_token=None, ): call_state["pipe_args"] = { "token": token, @@ -411,6 +422,7 @@ def __init__( "system": system, "rebuild_manifest_only": rebuild_manifest_only, "reconcile_manifest_only": reconcile_manifest_only, + "targeting_token": targeting_token, } def run_pipe(self): @@ -442,6 +454,8 @@ def __init__(self, system): _install_module(monkeypatch, "act.utils.pipe", pipe_mod) _install_module(monkeypatch, "act.utils.group", group_mod) + monkeypatch.setenv("TARGETING_TOKEN", "") + monkeypatch.setenv("DEBRIEF_FITNESS_TOKEN", "") monkeypatch.setattr( sys, "argv", @@ -467,6 +481,7 @@ def __init__(self, system): "system": "local", "rebuild_manifest_only": False, "reconcile_manifest_only": True, + "targeting_token": None, } assert call_state["run_pipe"] == 1 assert call_state["group_inits"] == 0 @@ -481,6 +496,7 @@ def __init__( system, rebuild_manifest_only=False, reconcile_manifest_only=False, + targeting_token=None, ): pass @@ -542,6 +558,7 @@ def __init__( system, rebuild_manifest_only=False, reconcile_manifest_only=False, + targeting_token=None, ): pass diff --git a/act/tests/test_report_merge.py b/act/tests/test_report_merge.py new file mode 100644 index 0000000..08b8463 --- /dev/null +++ b/act/tests/test_report_merge.py @@ -0,0 +1,52 @@ +"""Unit tests for multi-report boost↔lab merge (no network). + +Loads comparison_utils by path so conftest's ID_COMPARISONS stub does not apply. +""" +from __future__ import annotations + +import importlib.util +from pathlib import Path + +import pandas as pd + +_PATH = Path(__file__).resolve().parents[1] / "utils" / "comparison_utils.py" +_SPEC = importlib.util.spec_from_file_location("comparison_utils_real", _PATH) +_MOD = importlib.util.module_from_spec(_SPEC) +assert _SPEC.loader is not None +_SPEC.loader.exec_module(_MOD) +_merge = _MOD.ID_COMPARISONS._merge_report_frames + + +def test_merge_later_source_overrides_lab_conflict(): + a = pd.DataFrame( + [ + {"boost_id": "8080", "lab_id": "9999", "_source": "obs:43327"}, + {"boost_id": "7222", "lab_id": "1417", "_source": "obs:43327"}, + ] + ) + b = pd.DataFrame( + [ + {"boost_id": "8080", "lab_id": "1501", "_source": "int:53846"}, + ] + ) + merged = _merge([a, b]) + by_boost = {str(r.boost_id): str(r.lab_id) for r in merged.itertuples()} + assert by_boost["8080"] == "1501" + assert by_boost["7222"] == "1417" + + +def test_merge_skips_empty_lab(): + a = pd.DataFrame( + [ + {"boost_id": "9002", "lab_id": "", "_source": "int:53846"}, + {"boost_id": "8078", "lab_id": "1433", "_source": "int:53846"}, + ] + ) + merged = _merge([a]) + assert list(merged["boost_id"]) == ["8078"] + + +def test_normalize_id_cell_strips_float_dot_zero(): + assert _MOD._normalize_id_cell(8078.0) == "8078" + assert _MOD._normalize_id_cell("nan") == "" + assert _MOD._normalize_id_cell("7178, 8066") == "7178, 8066" diff --git a/act/utils/comparison_utils.py b/act/utils/comparison_utils.py index 538cdf6..f38a5e3 100644 --- a/act/utils/comparison_utils.py +++ b/act/utils/comparison_utils.py @@ -8,15 +8,54 @@ logger = logging.getLogger(__name__) +# OBS project (BOOST_TOKEN): legacy 43327 kept; 41693 = Names & ID Numbers. +# Targeting / INT (TARGETING_TOKEN): 53846 = boost↔lab for 8xxx/9xxx. +# Later entries override earlier on the same boost_id when lab_id differs. +DEFAULT_REPORT_SOURCES = ( + ("obs", 43327), + ("obs", 41693), + ("int", 53846), +) + + +def _normalize_id_cell(value) -> str: + """Normalize REDCap id cells to clean strings (avoid '8078.0' / 'nan').""" + if value is None or (isinstance(value, float) and pd.isna(value)): + return "" + try: + if pd.isna(value): + return "" + except (TypeError, ValueError): + pass + text = str(value).strip() + if not text or text.lower() in {"nan", "none"}: + return "" + # Combined dual-enrollment cells stay as-is (Save expands them later). + if "," in text or ";" in text: + return text + try: + return str(int(float(text))) + except ValueError: + return text + class ID_COMPARISONS: - def __init__(self, rdss_dir, token, daysago=None) -> None: + def __init__( + self, + rdss_dir, + token, + daysago=None, + targeting_token=None, + report_sources=None, + ) -> None: self.token = token + self.targeting_token = (targeting_token or "").strip() or None self.rdss_dir = os.fspath(rdss_dir) if rdss_dir is not None else None if not self.rdss_dir: raise ValueError("RDSS directory is required to compare IDs.") self.daysago = daysago + self.report_sources = tuple(report_sources or DEFAULT_REPORT_SOURCES) def compare_ids(self): """ @@ -72,31 +111,127 @@ def compare_ids(self): return {"matches": result, "duplicates": duplicates_dict} + def _token_for(self, project: str) -> str | None: + if project == "obs": + tok = (self.token or "").strip() + return tok or None + if project == "int": + return self.targeting_token + raise ValueError(f"Unknown report project key: {project}") + + def _fetch_report_csv(self, project: str, report_id: int) -> pd.DataFrame | None: + """Fetch one REDCap report. Soft-fail (None) on missing token / API error.""" + token = self._token_for(project) + label = f"{project}:{report_id}" + if not token: + logger.warning( + "Skipping REDCap report %s — no token configured for project=%s", + label, + project, + ) + return None + + url = "https://redcap.icts.uiowa.edu/redcap/api/" + data = { + "token": token, + "content": "report", + "report_id": report_id, + "format": "csv", + } + try: + r = requests.post(url, data=data, timeout=180) + except requests.RequestException as exc: + logger.error("REDCap request failed for %s: %s", label, exc) + return None + + body = (r.text or "").strip() + if r.status_code != 200 or body.startswith("ERROR"): + logger.warning( + "REDCap report %s unavailable (status=%s): %s", + label, + r.status_code, + body[:200].replace("\n", " "), + ) + return None + + df = pd.read_csv(StringIO(r.text)) + if "boost_id" not in df.columns or "lab_id" not in df.columns: + logger.warning( + "REDCap report %s missing boost_id/lab_id columns: %s", + label, + list(df.columns), + ) + return None + + slim = df[["boost_id", "lab_id"]].copy() + slim["boost_id"] = slim["boost_id"].map(_normalize_id_cell) + slim["lab_id"] = slim["lab_id"].map(_normalize_id_cell) + slim = slim[(slim["boost_id"] != "") & (slim["lab_id"] != "")] + slim["_source"] = label + logger.info( + "REDCap report %s loaded: %s rows with boost_id+lab_id", + label, + len(slim), + ) + return slim.reset_index(drop=True) + + @staticmethod + def _merge_report_frames(frames: list[pd.DataFrame]) -> pd.DataFrame: + """Concatenate reports; later sources win on boost_id lab conflicts.""" + if not frames: + return pd.DataFrame(columns=["boost_id", "lab_id", "_source"]) + + merged: dict[str, tuple[str, str]] = {} + for frame in frames: + for _, row in frame.iterrows(): + boost = _normalize_id_cell(row["boost_id"]) + lab = _normalize_id_cell(row["lab_id"]) + src = str(row.get("_source", "")) + if not boost or not lab: + continue + if boost in merged and merged[boost][0] != lab: + logger.warning( + "boost_id %s lab conflict: %s (%s) overridden by %s (%s)", + boost, + merged[boost][0], + merged[boost][1], + lab, + src, + ) + merged[boost] = (lab, src) + + rows = [ + {"boost_id": boost, "lab_id": lab, "_source": src} + for boost, (lab, src) in merged.items() + ] + return pd.DataFrame(rows) + def _return_report(self): """ - pulls the id report from the rdss via redcap api. - reads the report as a dataframe. - checks for boost_ids that are associated with multiple lab_ids, logs a critical error, - and removes these rows from the dataframe. - separates duplicate rows (based on any column) from the cleaned data. + Pull boost_id↔lab_id maps from configured REDCap reports (OBS + Targeting), + merge them, then apply the existing duplicate / dual-enrollment filters. returns: df_cleaned: dataframe with duplicates removed and problematic boost_ids excluded duplicate_rows: dataframe of duplicate rows """ - url = "https://redcap.icts.uiowa.edu/redcap/api/" - data = { - "token": self.token, - "content": "report", - "report_id": 43327, - "format": "csv", - } - r = requests.post(url, data=data) - if r.status_code != 200: - print(f"error! status code is {r.status_code}") + frames: list[pd.DataFrame] = [] + for project, report_id in self.report_sources: + frame = self._fetch_report_csv(project, int(report_id)) + if frame is not None and not frame.empty: + frames.append(frame) + + df = self._merge_report_frames(frames) + if df.empty: + print( + "error! no usable REDCap boost_id/lab_id rows from any configured report " + f"({', '.join(f'{p}:{r}' for p, r in self.report_sources)})" + ) sys.exit(1) - df = pd.read_csv(StringIO(r.text)) + # Drop helper column before downstream logic that expects lab_id/boost_id only. + if "_source" in df.columns: + df = df.drop(columns=["_source"]) # identify boost_ids associated with multiple lab_ids. boost_id_counts = df.groupby("boost_id")["lab_id"].nunique() diff --git a/act/utils/pipe.py b/act/utils/pipe.py index 5abbffc..4dc33f0 100644 --- a/act/utils/pipe.py +++ b/act/utils/pipe.py @@ -56,10 +56,12 @@ def __init__( system="vosslnx", rebuild_manifest_only=False, reconcile_manifest_only=False, + targeting_token=None, ): # ensure class attrs are set for everyone (Pipe.INT_DIR etc.) type(self).configure(system) self.token = token + self.targeting_token = targeting_token self.daysago = daysago self.system = system self.rebuild_manifest_only = rebuild_manifest_only @@ -73,6 +75,7 @@ def run_pipe(self): token=self.token, daysago=self.daysago, symlink=False, + targeting_token=self.targeting_token, ) try: diff --git a/act/utils/save.py b/act/utils/save.py index 85bf866..c406909 100644 --- a/act/utils/save.py +++ b/act/utils/save.py @@ -22,6 +22,7 @@ def __init__( daysago=None, symlink=True, manifest_path="res/data.json", + targeting_token=None, ): if not rdssdir: raise ValueError( @@ -29,7 +30,10 @@ def __init__( ) results = ID_COMPARISONS( - rdss_dir=rdssdir, token=token, daysago=daysago + rdss_dir=rdssdir, + token=token, + daysago=daysago, + targeting_token=targeting_token, ).compare_ids() self.matches = results["matches"] self.matches.pop("6022, 7143", None) @@ -40,6 +44,7 @@ def __init__( self.OBS_DIR = obsdir self.RDSS_DIR = rdssdir self.token = token + self.targeting_token = targeting_token self.daysago = daysago self.symlink = symlink self.manifest_path = manifest_path @@ -280,6 +285,7 @@ def _fetch_redcap_subject_lab_rows(self): rdss_dir=rdss_dir, token=token, daysago=daysago, + targeting_token=getattr(self, "targeting_token", None), )._return_report() return report_df diff --git a/docs/cli.md b/docs/cli.md index 5637ccf..0c50795 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -43,7 +43,7 @@ How it is used: - Passed from `act.main` into `Pipe(token=...)`. - Passed from `Pipe` into `Save(token=...)`. -- Used by `ID_COMPARISONS._return_report()` to call the REDCap API and export report `43327`. +- Used by `ID_COMPARISONS._return_report()` for Observational reports `43327` and `41693`. Intervention report `53846` uses `--targeting-token` / `TARGETING_TOKEN`. Operational notes: diff --git a/docs/system.md b/docs/system.md index ef1e5d7..5471379 100644 --- a/docs/system.md +++ b/docs/system.md @@ -15,7 +15,7 @@ Primary entrypoint: `python -m act.main`. ### In Scope -- Pulling subject mappings from REDCap report `43327`. +- Pulling subject mappings from REDCap reports `43327` + `41693` (Observational, `BOOST_TOKEN`) and `53846` (Targeting/Intervention, `TARGETING_TOKEN`); merge by `boost_id` (later reports override lab conflicts). Folder routing still by boost band: `6001–7999` → OBS, `≥8000` → INT. - Reading RDSS filenames and extracting metadata (`labID`, acquisition date, filename). - Matching BOOST IDs to RDSS lab IDs and handling duplicates. - Copying canonical accelerometer files to LSS study/session destinations.