Feature reset

.dockerignore

@@ -0,0 +1,68 @@
+# Git
+.git/
+.gitignore
+
+# Python cache
+__pycache__/
+*.py[cod]
+*.pyo
+*.pyd
+*.so
+*.egg
+*.egg-info/
+dist/
+build/
+
+# Virtual environments
+.venv/
+venv/
+env/
+ENV/
+
+# Jupyter
+.ipynb_checkpoints/
+*.ipynb
+
+# ML artifacts
+*.pkl
+*.pickle
+*.joblib
+*.h5
+*.hdf5
+*.pt
+*.pth
+*.ckpt
+*.safetensors
+models/
+checkpoints/
+runs/
+
+# Data (large files not needed at build time)
+*.csv.gz
+*.parquet
+*.feather
+data/
+datasets/
+raw/
+
+# Outputs
+output/
+outputs/
+results/
+logs/
+*.log
+
+# IDE / OS
+.idea/
+.vscode/
+.DS_Store
+*.swp
+*.swo
+
+# Environment variables
+.env
+.env.*
+
+# Docs
+README.md
+LICENSE

.gitignore

@@ -0,0 +1,103 @@
+# Python
+__pycache__/
+*.py[cod]
+*.pyo
+*.pyd
+*.so
+*.egg
+*.egg-info/
+dist/
+build/
+.eggs/
+wheels/
+
+# Virtual environments
+.venv/
+venv/
+env/
+ENV/
+
+# Jupyter
+.ipynb_checkpoints/
+*.ipynb
+
+# Local dev working dirs (build_dev_subset.py outputs)
+dev_subset/
+dev_model/
+dev_outputs/
+
+# ML artifacts
+*.pkl
+*.pickle
+*.joblib
+*.h5
+*.hdf5
+*.pt
+*.pth
+*.ckpt
+*.safetensors
+models/
+checkpoints/
+runs/
+
+# Data
+*.csv.gz
+*.parquet
+*.feather
+data/
+datasets/
+raw/
+
+# Outputs
+output/
+outputs/
+results/
+logs/
+*.log
+
+
+# macOS
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# IDE
+.idea/
+.vscode/
+*.swp
+*.swo
+
+# Environment variables
+.env
+.env.*
+
+# pytest
+.pytest_cache/
+.coverage
+htmlcov/
+
+
+#measurment artifacts
+*.edf
+*.sav
+full_model/
+full_outputs/
+docker_model/
+dev_model/
+dev_outputs/
+dev_model_v2/
+dev_outputs_v2/
+features/FEATURES.md
+CLAUDE.md
+build_dev_subset.py
+learning_log.md
+learning_log_p2.md
+eda_outputs/
+docker_outputs/
+
+#files
+phase1_eda.py
+/loso_cv.py
+archive_outputs/
+eda/
+/tools

extract.py

@@ -0,0 +1,51 @@
+import sys
+import numpy as np
+
+# Adjust these paths to match your repo layout
+sys.path.insert(0, 'tools')   # so `from loso_cv import extract_all_features` resolves
+sys.path.insert(0, '.')       # repo root, for helper_code/features imports loso_cv.py needs
+
+from loso_cv import extract_all_features
+
+DATA_PATH = '/Users/briandillon/.cache/kagglehub/datasets/physionet/physionetchallenge2026data/versions/7'
+
+# Ground truth: loso_cv.py's own extraction (same function that produced every logged result)
+X_truth, y_truth, ages_truth, sites_truth, ids_truth = extract_all_features(DATA_PATH, verbose=True)
+
+# Cache: build_features_cache.py's output
+d = np.load('tools/small_features.npz', allow_pickle=True)
+X_cache, y_cache, ids_cache = d['X'], d['y'], d['patient_id']
+
+print(f"Ground truth: {X_truth.shape}, cache: {X_cache.shape}")
+
+# Align by patient_id — order may differ between the two extraction runs
+truth_lookup = {pid: i for i, pid in enumerate(ids_truth)}
+cache_lookup = {pid: i for i, pid in enumerate(ids_cache)}
+
+common_ids = sorted(set(truth_lookup) & set(cache_lookup))
+print(f"Common patient IDs: {len(common_ids)} (expect 1103)")
+
+max_diffs = []
+mismatched_patients = []
+
+for pid in common_ids:
+    row_truth = X_truth[truth_lookup[pid]]
+    row_cache = X_cache[cache_lookup[pid]]
+    # NaN-safe comparison — both matrices use NaN for missing CAISR data
+    diff = np.abs(np.nan_to_num(row_truth, nan=-999) - np.nan_to_num(row_cache, nan=-999))
+    max_diff = diff.max()
+    max_diffs.append(max_diff)
+    if max_diff > 1e-4:
+        mismatched_patients.append((pid, max_diff, np.argmax(diff)))
+
+max_diffs = np.array(max_diffs)
+print(f"\nMax abs diff across all patients/features: {max_diffs.max():.6f}")
+print(f"Patients with any mismatch (>1e-4): {len(mismatched_patients)}")
+
+if mismatched_patients:
+    print("\nFirst 5 mismatches (patient_id, max_diff, feature_column_index):")
+    for m in mismatched_patients[:5]:
+        print(" ", m)
+else:
+    print("\nCLEAN — every patient's 48-length vector matches exactly between "
+          "loso_cv.py's extraction and build_features_cache.py's cache.")

features/__init__.py

@@ -0,0 +1,87 @@
+# features/__init__.py
+# Shared constants for the Narnia_ML feature pipeline.
+# Update these whenever a feature module adds or removes features.
+#
+# Entry 2 (2026-07-01): Dropped absolute Hjorth (site-confounded per LOSO).
+# Added within-recording ratio features (site-stable by construction).
+# Feature count: 82 → 48.
+#
+# Entry 4 (2026-07-02): Added 2 age-residualized features (CA_rate,
+# EEG_var_REM_Wake) via features/age_residuals.py — see that module's
+# header for the age-residualized EDA rationale. These are appended by an
+# AgeResidualizer pipeline step AFTER extraction/hstack, not by a feature
+# module called from team_code.py's extraction block, so they live at the
+# end of the vector rather than inside the demo/base/enriched/ratio blocks.
+# Feature count: 48 → 50.
+#
+# NEW candidate, not yet validated (2026-07-20): Added 3 stage-conditional
+# limb movement features (Limb_REM, Limb_NREM, Limb_REM_NREM_ratio) via
+# features/caisr_enriched.py — same enrichment pattern already proven for
+# AHI (REM_AHI/NREM_AHI/REM_NREM_AHI_ratio), applied to Limb_idx instead.
+# Appended at the END of the enriched block (after N3_entropy, before the
+# ratio block starts) — inserted, not appended to the whole vector, so
+# downstream index constants that are defined relative to IDX_RATIO_START
+# (e.g. IDX_EEG_VAR_REM_WAKE below) auto-update correctly; nothing already
+# shipped needs a manual index bump. Feature count: 50 → 53 (raw 48 → 51,
+# plus the same 2 age-residual features as before). Requires regenerating
+# any --features-cache built before this change — the raw hstack length
+# changed from 48 to 51.
+
+
+N_DEMOGRAPHIC_FEATURES     = 10   # features/demographic.py
+N_CAISR_BASE_FEATURES      = 12   # features/caisr_base.py
+N_CAISR_ENRICHED_FEATURES  = 14   # features/caisr_enriched.py (was 11 — +3 limb features, 2026-07-20)
+N_RATIO_FEATURES           = 15   # features/physiological_ratios.py
+N_AGE_RESIDUAL_FEATURES    = 2    # features/age_residuals.py (Entry 4, pipeline-appended)
+
+N_PHYSIOLOGICAL_FEATURES  = 0
+N_ALGORITHMIC_FEATURES    = N_CAISR_BASE_FEATURES + N_CAISR_ENRICHED_FEATURES  # 23
+
+N_TOTAL_FEATURES = (
+    N_DEMOGRAPHIC_FEATURES +
+    N_CAISR_BASE_FEATURES +
+    N_CAISR_ENRICHED_FEATURES +
+    N_RATIO_FEATURES +
+    N_AGE_RESIDUAL_FEATURES
+)  # 50
+
+IDX_DEMO_START      = 0
+IDX_DEMO_END        = N_DEMOGRAPHIC_FEATURES                              # 10
+IDX_BASE_START      = IDX_DEMO_END
+IDX_BASE_END        = IDX_BASE_START + N_CAISR_BASE_FEATURES             # 22
+IDX_ENRICHED_START  = IDX_BASE_END
+IDX_ENRICHED_END    = IDX_ENRICHED_START + N_CAISR_ENRICHED_FEATURES     # 33
+IDX_RATIO_START     = IDX_ENRICHED_END
+IDX_RATIO_END       = IDX_RATIO_START + N_RATIO_FEATURES                 # 48
+IDX_AGE_RESID_START = IDX_RATIO_END
+IDX_AGE_RESID_END   = IDX_AGE_RESID_START + N_AGE_RESIDUAL_FEATURES      # 50
+
+IDX_AGE              = IDX_DEMO_START               # Age is demo feature 0
+IDX_CA_RATE          = IDX_ENRICHED_START + 1        # CA_rate is caisr_enriched feature 1
+IDX_EEG_VAR_REM_WAKE = IDX_RATIO_START + 11          # EEG_var_REM_Wake is ratio feature 11
+
+# Human-readable names for the 48-length PRE-AgeResidualizer vector, index-aligned.
+# Pulled directly from FEATURES.md's Entry 2 (48-feature) index table — real
+# names, not placeholders. Used by reg_sweep.py to label coefficients / top
+# features so sweep output is directly readable against your own registry.
+FEATURE_NAMES_48 = [
+    'Age', 'Sex_F', 'Sex_M', 'Sex_Unk', 'Race_Asian', 'Race_Black',
+    'Race_Other', 'Race_Unavailable', 'Race_White', 'BMI',
+    'AHI_total', 'Arousal_idx', 'Limb_idx', 'Wake_pct', 'N1_pct', 'N2_pct',
+    'N3_pct', 'REM_pct', 'Sleep_eff', 'Prob_W', 'Prob_N3', 'Prob_arous',
+    'OA_rate', 'CA_rate', 'HY_rate', 'RERA_rate', 'CA_total_ratio',
+    'REM_AHI', 'NREM_AHI', 'REM_NREM_AHI_ratio', 'N3_gradient',
+    'Spont_arousal_idx', 'N3_entropy',
+    'Limb_REM', 'Limb_NREM', 'Limb_REM_NREM_ratio',
+    'EEG_std_N3_Wake', 'EEG_mav_N3_Wake', 'EEG_zcr_N3_Wake', 'EEG_rms_N3_Wake',
+    'EEG_var_N3_Wake', 'EEG_mob_N3_Wake', 'EEG_cplx_N3_Wake',
+    'EEG_std_REM_Wake', 'EEG_mav_REM_Wake', 'EEG_zcr_REM_Wake', 'EEG_rms_REM_Wake',
+    'EEG_var_REM_Wake', 'EEG_mob_REM_Wake', 'EEG_cplx_REM_Wake',
+    'Chin_mav_REM_NREM_atonia',
+]
+assert len(FEATURE_NAMES_48) == IDX_RATIO_END, \
+    f"FEATURE_NAMES_48 length {len(FEATURE_NAMES_48)} != {IDX_RATIO_END}"
+assert FEATURE_NAMES_48[IDX_CA_RATE] == 'CA_rate'
+assert FEATURE_NAMES_48[IDX_EEG_VAR_REM_WAKE] == 'EEG_var_REM_Wake'
+
+FEATURE_NAMES_50 = FEATURE_NAMES_48 + ['CA_rate_age_residual', 'EEG_var_REM_Wake_age_residual']

features/age_residuals.py

@@ -0,0 +1,71 @@
+# features/age_residuals.py
+# Age-residualized features — Entry 4
+#
+# Origin: 2026-07-02 age-residualized EDA (see learning_log.md). CA_rate and
+# EEG_var_REM_Wake were previously written off as uninformative in the raw
+# Phase 1 EDA (AUROC ~0.52, BH-FDR p_adj > 0.7 both) — but regressing Age out
+# of every feature and re-testing the residual showed both carry real signal
+# that raw testing couldn't see:
+#
+#     Feature            Raw AUROC / p_adj      Residual AUROC / p_adj
+#     CA_rate             0.520 / 0.805           0.622 / 0.0075
+#     EEG_var_REM_Wake    0.520 / 0.808           0.618 / 0.0098
+#
+# Age is available at real inference time (unlike the ICD-subtype features
+# ruled out earlier for exactly that reason) — this is feature engineering,
+# not leakage.
+#
+# Fitting discipline: identical to SimpleImputer. The linear age-regression
+# coefficient for each source feature is fit ONCE on training data (inside
+# AgeResidualizer.fit(), called by Pipeline.fit()) and frozen. transform()
+# only ever applies the already-fitted coefficients — it must never re-fit,
+# including at inference time on a single patient.
+#
+# Source features (already present earlier in the vector — see
+# features/__init__.py for the index constants used to locate them):
+#   Age                 index 0                 (features/demographic.py)
+#   CA_rate             IDX_ENRICHED_START + 1  (features/caisr_enriched.py)
+#   EEG_var_REM_Wake    IDX_RATIO_START + 11    (features/physiological_ratios.py)
+#
+# These 2 new features are APPENDED at the end of the vector (indices 48-49)
+# rather than inserted mid-vector, so the existing demo/base/enriched/ratio
+# index constants don't shift. Total feature count: 48 -> 50.
+
+import numpy as np
+from sklearn.base import BaseEstimator, TransformerMixin
+
+N_AGE_RESIDUAL_FEATURES = 2
+_MIN_FIT_ROWS = 10
+
+
+class AgeResidualizer(BaseEstimator, TransformerMixin):
+    def __init__(self, age_idx, ca_rate_idx, eeg_var_rem_wake_idx):
+        self.age_idx = age_idx
+        self.ca_rate_idx = ca_rate_idx
+        self.eeg_var_rem_wake_idx = eeg_var_rem_wake_idx
+
+    @staticmethod
+    def _fit_one(age, feature):
+        mask = ~(np.isnan(age) | np.isnan(feature))
+        if int(mask.sum()) < _MIN_FIT_ROWS:
+            return 0.0, 0.0
+        slope, intercept = np.polyfit(age[mask], feature[mask], 1)
+        return float(slope), float(intercept)
+
+    def fit(self, X, y=None):
+        X = np.asarray(X, dtype=float)
+        age = X[:, self.age_idx]
+        self.ca_rate_slope_, self.ca_rate_intercept_ = self._fit_one(
+            age, X[:, self.ca_rate_idx])
+        self.eeg_slope_, self.eeg_intercept_ = self._fit_one(
+            age, X[:, self.eeg_var_rem_wake_idx])
+        return self
+
+    def transform(self, X):
+        X = np.asarray(X, dtype=float)
+        age = X[:, self.age_idx]
+        ca_rate_resid = X[:, self.ca_rate_idx] - (
+            self.ca_rate_intercept_ + self.ca_rate_slope_ * age)
+        eeg_resid = X[:, self.eeg_var_rem_wake_idx] - (
+            self.eeg_intercept_ + self.eeg_slope_ * age)
+        return np.hstack([X, ca_rate_resid.reshape(-1, 1), eeg_resid.reshape(-1, 1)])