diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..d8e589d --- /dev/null +++ b/.dockerignore @@ -0,0 +1,23 @@ +# Datasets +data/ +**/training_set/ +**/supplementary_set/ +**/*.edf + +# Artifacts +model/ +model_smoke/ +model_full_smoke/ +outputs/ +outputs_smoke/ +__pycache__/ +*.pyc +*.pkl +*.sav +*.joblib + +# OS / IDE +.DS_Store +Thumbs.db +.vscode/ +.idea/ \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a656935 --- /dev/null +++ b/.gitignore @@ -0,0 +1,240 @@ +# Dataset +data/ +data_smoke/ + +# Cache +.feature_cache/ + +# Model artifacts +model/ +model_smoke/ +*.pkl +*.sav +*.joblib + +# Outputs +outputs/ +outputs_smoke/ + +# Python +__pycache__/ +*.pyc + +# OS +.DS_Store +Thumbs.db + +# IDE +.vscode/ +.idea/ + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[codz] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py.cover +.hypothesis/ +.pytest_cache/ +cover/ +graphs/ + +graphs +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# UV +# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +#uv.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock +#poetry.toml + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python. +# https://pdm-project.org/en/latest/usage/project/#working-with-version-control +#pdm.lock +#pdm.toml +.pdm-python +.pdm-build/ + +# pixi +# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control. +#pixi.lock +# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one +# in the .venv directory. It is recommended not to include this directory in version control. +.pixi + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.envrc +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +# Abstra +# Abstra is an AI-powered process automation framework. +# Ignore directories containing user credentials, local state, and settings. +# Learn more at https://abstra.io/docs +.abstra/ + +# Visual Studio Code +# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore +# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore +# and can be added to the global gitignore or merged into this file. However, if you prefer, +# you could uncomment the following to ignore the entire vscode folder +# .vscode/ + +# Ruff stuff: +.ruff_cache/ + +# PyPI configuration file +.pypirc + +# Cursor +# Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to +# exclude from AI features like autocomplete and code analysis. Recommended for sensitive data +# refer to https://docs.cursor.com/context/ignore-files +.cursorignore +.cursorindexingignore + +# Marimo +marimo/_static/ +marimo/_lsp/ +__marimo__/ + +movedata.py \ No newline at end of file diff --git a/AUTHORS.txt b/AUTHORS.txt new file mode 100644 index 0000000..143148b --- /dev/null +++ b/AUTHORS.txt @@ -0,0 +1,4 @@ +Sofia Romagnoli - Universidad de Zaragoza +Diego Cajal - CIBER-BBN +Josseline Madrid - Universidad de Zaragoza +Rodrigo Lozano - Universidad de Zaragoza diff --git a/RemoveTestCache.py b/RemoveTestCache.py new file mode 100644 index 0000000..2046a39 --- /dev/null +++ b/RemoveTestCache.py @@ -0,0 +1,20 @@ +import os +import pandas as pd + +demotest = pd.read_csv("data/test_set/demographics.csv") + +cache_path = ".feature_cache" + +for i in demotest.index: + h = demotest.loc[i, "SiteID"] + f = demotest.loc[i, "BidsFolder"] + session = demotest.loc[i, "SessionID"] + if os.path.exists(os.path.join(cache_path, h, f+"_ses-" + str(session)+".csv")): + print(f"Removing {os.path.join(cache_path, h, f+"_ses-" + str(session)+".csv")}") + os.remove(os.path.join(cache_path, h, f+"_ses-" + str(session)+".csv")) + if os.path.exists(os.path.join(cache_path, h, f+"_ses-" + str(session)+".sav")): + print(f"Removing {os.path.join(cache_path, h, f+"_ses-" + str(session)+".sav")}") + os.remove(os.path.join(cache_path, h, f+"_ses-" + str(session)+".sav")) + +os.remove(os.path.join(cache_path, "exports", "test_features_raw.csv")) +os.remove(os.path.join(cache_path, "exports", "test_features_preprocessed.csv")) \ No newline at end of file diff --git a/docs/01_overview.md b/docs/01_overview.md new file mode 100644 index 0000000..9805383 --- /dev/null +++ b/docs/01_overview.md @@ -0,0 +1,35 @@ +# CINC 2026 – Visión General del Proyecto + +Estamos participando en el Challenge 2026 de Computing in Cardiology. + +El objetivo es predecir deterioro cognitivo a partir de datos de polisomnografía (PSG). + +## Cómo nos evaluarán + +La organización: + +1. Construirá nuestra imagen Docker. +2. Ejecutará `train_model.py`. +3. Ejecutará `run_model.py`. +4. Evaluará las predicciones generadas. + +Por tanto, la reproducibilidad mediante Docker es obligatoria. + +Nuestro objetivo es garantizar que: +- El código se ejecuta sin intervención manual. +- El modelo se entrena correctamente. +- Las predicciones se generan en el formato requerido. + +## Qué se puede modificar y qué no + +❌ No modificar + +- `train_model.py` +- `run_model.py` +- `helper_code.py` +- `evaluate_model.py` + +✅ Modificar/Añadir + +- `team_code.py` <-- Toda la lógica científica y de modelado debe implementarse ahí. +- Helpers, scripts, métodos: añadir a voluntad en `src/` \ No newline at end of file diff --git a/docs/02_docker.md b/docs/02_docker.md new file mode 100644 index 0000000..c24ced2 --- /dev/null +++ b/docs/02_docker.md @@ -0,0 +1,46 @@ +# Uso de Docker + +Este documento define el contexto de ejecución con Docker. + +## Requisitos + +- Docker Desktop instalado (modo Linux containers) +- Dataset descargado desde Kaggle +- Dataset completo disponible en `data/training_set/` (ruta por defecto del proyecto) + +Si tu dataset está en otra ubicación, actualiza la variable de ruta en el script de ejecución. + +## Estructura de trabajo + +Entradas: + +- `data/training_set/` (dataset completo) +- `data/training_smoke/` (dataset reducido para modo desarrollo (smoke)) + +Salidas: + +- `model/` y `outputs/` (flujo completo) +- `model_smoke/` y `outputs_smoke/` (flujo smoke/desarrollo) + +## Orden recomendado de ejecución + +1. Construir imagen Docker (`build`) +2. Preparar dataset smoke (`smoke`) +3. Iterar en modo desarrollo (smoke) (`train-dev` / `run-dev`) +4. Ejecutar validación completa (`train` / `run`) +5. Limpiar artefactos cuando corresponda (`clean`) + +La guía paso a paso está en `docs/04_run_script.md`. + +## Compatibilidad de scripts + +El flujo principal del equipo está documentado con `run.sh` (Git Bash). +También existen equivalentes en PowerShell: `run.ps1` y `scripts/create_smoke.ps1`. + +## Resultado esperado + +Tras ejecutar la generación de predicciones (inferencia) completa, en `outputs/` se genera un `demographics.csv` con: + +- Columnas originales +- `Cognitive_Impairment` +- `Cognitive_Impairment_Probability` \ No newline at end of file diff --git a/docs/03_smoke_dataset.md b/docs/03_smoke_dataset.md new file mode 100644 index 0000000..ae4ee23 --- /dev/null +++ b/docs/03_smoke_dataset.md @@ -0,0 +1,41 @@ +# Dataset smoke (Modo desarrollo) + +Entrenar con el dataset completo tarda aproximadamente 30–40 minutos con el modelo de ejemplo. + +Para desarrollo utilizamos un dataset reducido (5 sujetos por defecto). + +Este documento describe cuándo y por qué usar smoke. +Los comandos de ejecución están centralizados en `docs/04_run_script.md`. + +--- + +## Qué incluye + +- Muestra reducida del dataset (5 sujetos por defecto) +- Estructura compatible con el flujo oficial del proyecto +- Directorio de salida en `data/training_smoke/` +- `demographics.csv` filtrado para que solo incluya los registros copiados al smoke + +## Para qué se usa + +- Validar cambios de código rápidamente +- Detectar errores de integración antes del entrenamiento completo +- Iterar en modo desarrollo (smoke) sin esperar ciclos largos + +## Artefactos asociados + +- Entrenamiento smoke: `model_smoke/` +- Predicciones (inferencia) smoke: `outputs_smoke/` + +## Relación con el flujo principal + +El dataset smoke se crea al inicio del ciclo de desarrollo y se usa junto con `train-dev` y `run-dev`. +El orden detallado de ejecución está en `docs/04_run_script.md`. + +## ¿Cuándo usar smoke? + +- Desarrollo de nuevas funcionalidades +- Comprobación rápida de que el código no rompe +- Validación de cambios en `team_code.py` + +Nunca usar smoke para evaluar rendimiento final. \ No newline at end of file diff --git a/docs/04_run_script.md b/docs/04_run_script.md new file mode 100644 index 0000000..87eef6d --- /dev/null +++ b/docs/04_run_script.md @@ -0,0 +1,130 @@ +# Script unificado de ejecución (`run.sh`) + +Este documento es la guía operativa única para ejecutar el proyecto. +Aquí se define el orden recomendado y los comandos asociados. + +--- + +# Requisitos + +- Docker Desktop instalado +- Dataset descargado en: + +``` +data/training_set/ +data/supplementary_set/ +``` + +⚠️ Si el dataset está en otra ubicación, modificar las variables `$TRAIN_DATA_REL` y `$RUN_DATA_REL` +dentro de `run.sh`. + +⚠️ Ejecutar los comandos desde Git Bash. + +ℹ️ Existen scripts equivalentes en PowerShell (`run.ps1` y `scripts/create_smoke.ps1`) para quienes prefieran ese entorno. + +ℹ️ Para contexto general y definición de artefactos, ver `docs/02_docker.md` y `docs/03_smoke_dataset.md`. +--- + +# Orden de ejecución recomendado + +Desde la raíz del repositorio. + +## 1) Preparar entorno + +### Construir imagen Docker + +```bash +./run.sh build +``` + +Ejecutar la primera vez y cada vez que cambien `requirements.txt` o `Dockerfile`. + +### Crear dataset smoke (5 sujetos) + +```bash +./run.sh smoke +``` + +Genera `data/training_smoke/`. + +## 2) Ciclo en modo desarrollo (smoke) + +### Entrenar en modo desarrollo (smoke) + +```bash +./run.sh train-dev +``` + +Usa `data/training_smoke/` y guarda modelo en `model_smoke/`. + +### Generar predicciones (inferencia) en modo desarrollo (smoke) + +```bash +./run.sh run-dev +``` + +Genera resultados en `outputs_smoke/` y luego imprime métricas de evaluación en consola. + +### Evaluar predicciones existentes en modo desarrollo (smoke) + +```bash +./run.sh eval-dev +``` + +Reutiliza `outputs_smoke/demographics.csv` y muestra AUROC, AUPRC, Accuracy y F-measure sin volver a ejecutar inferencia. + +### Secuencia típica en modo desarrollo (smoke) + +```bash +./run.sh build # solo la primera vez +./run.sh smoke # solo si no existe +./run.sh train-dev +./run.sh run-dev +./run.sh eval-dev # opcional: reevaluar sin correr inferencia +``` + +## 3) Validación completa + +### Entrenar con dataset completo + +```bash +./run.sh train +``` + +Guarda el modelo en `model/`. + +### Generar predicciones (inferencia) completas + +```bash +./run.sh run +``` + +Genera resultados en `outputs/` usando `data/test_set/`. +Si el dataset no tiene etiquetas (como en `test_set`), el script omite la evaluación automáticamente. + +### Evaluar predicciones existentes completas + +```bash +./run.sh eval +``` + +Reutiliza `outputs/demographics.csv` y muestra AUROC, AUPRC, Accuracy y F-measure sin volver a ejecutar inferencia. +Evalúa contra `data/test_set/`. +Si no hay etiquetas en ese set, el script omite la evaluación automáticamente. + +### Evaluar predicciones existentes del dataset smoke + +```bash +./run.sh eval-smoke +``` + +Reutiliza `outputs_smoke/demographics.csv` y muestra AUROC, AUPRC, Accuracy y F-measure sin volver a ejecutar inferencia. + +## 4) Limpieza de artefactos + +```bash +./run.sh clean +``` + +Elimina `model/`, `model_smoke/`, `outputs/` y `outputs_smoke/`. +No elimina datasets. diff --git a/docs/05_optimization_tracking.md b/docs/05_optimization_tracking.md new file mode 100644 index 0000000..f601f8e --- /dev/null +++ b/docs/05_optimization_tracking.md @@ -0,0 +1,151 @@ +# Seguimiento De Optimizaciones + +Este documento registra las optimizaciones de tiempo de entrenamiento aplicadas sobre el código de ejemplo original del PhysioNet Challenge en `team_code.py`. + +## Objetivo + +Mantener un registro claro de los cambios respecto a la base proporcionada por la organización para que el equipo pueda: + +- entender qué optimizaciones se probaron; +- medir su efecto sobre el flujo smoke; +- identificar qué cambios merece la pena conservar; +- revertir cambios concretos si la submission se comporta distinto en el entorno del Challenge. + +## Línea Base + +- Fuente de la línea base: implementación de ejemplo proporcionada por la organización en `team_code.py`. +- Tiempo observado de entrenamiento smoke con `./run.sh train-dev`: alrededor de 22 segundos. +- Comportamiento base: extracción secuencial de features, lecturas repetidas de CSV, recarga repetida de reglas de renombrado de canales y carga no utilizada de anotaciones humanas durante entrenamiento. + +## Cambios Aplicados + +### 1. Eliminación de la carga no utilizada de anotaciones humanas en entrenamiento + +Cambio: + +- Se eliminó la carga de `human_annotations` dentro de `train_model`. +- Se dejó intacta la función auxiliar `extract_human_annotations_features`. + +Motivo: + +- El vector final de entrenamiento solo concatenaba features demográficas, fisiológicas y algorítmicas. +- Las features de anotaciones humanas se calculaban, pero nunca se incluían en el `np.hstack(...)` que se pasaba al clasificador. + +Efecto observado: + +- El tiempo de entrenamiento smoke pasó de unos 22.0 s a 21.891 s. +- Conclusión: la limpieza es correcta a nivel lógico, pero su impacto en tiempo es despreciable en el dataset smoke. + +Riesgo: + +- Bajo. Solo elimina trabajo muerto. + +### 2. Caché de reglas de renombrado de canales + +Cambio: + +- Se añadió una caché en proceso para las reglas de renombrado cargadas desde `channel_table.csv`. +- Se sustituyeron las llamadas repetidas a `load_rename_rules(os.path.abspath(csv_path))` por una consulta a la caché. + +Motivo: + +- `extract_physiological_features` estaba cargando y parseando el mismo CSV para cada registro. + +Efecto observado: + +- El tiempo smoke medido en la siguiente ejecución fue 22.040 s. +- Conclusión: la optimización es correcta, pero no ataca un cuello de botella relevante en smoke. + +Riesgo: + +- Bajo. El comportamiento no cambia salvo por reutilizar reglas ya parseadas. + +### 3. Caché de demographics y etiquetas para entrenamiento + +Cambio: + +- Se añadió una lectura única de `demographics.csv` al inicio de `train_model`. +- Se construyeron: + - una caché de demographics indexada por `(patient_id, session_id)`; + - una caché de diagnósticos indexada por `patient_id`. +- Se reemplazaron las llamadas por registro a `load_demographics(...)` y `load_diagnoses(...)` durante entrenamiento. + +Motivo: + +- El bucle original de entrenamiento releía el mismo CSV para cada registro. + +Efecto observado: + +- El tiempo smoke bajó a 20.837 s. +- Conclusión: es una mejora real, aunque moderada. + +Riesgo: + +- Bajo a medio. +- Asume que las etiquetas de entrenamiento son estables a nivel de paciente cuando se cachean por `patient_id`, igual que hacía el comportamiento original de `load_diagnoses(...)`. + +### 4. Paralelización de la extracción de features en entrenamiento + +Cambio: + +- Se añadió procesamiento paralelo por registro con `ThreadPoolExecutor` dentro de `train_model`. +- Se movió la lógica de extracción por registro a `process_training_record(...)`. +- Se limitó el número de workers con: + +```python +MAX_TRAIN_WORKERS = max(1, min(4, os.cpu_count() or 1)) +``` + +Motivo: + +- Cada registro de entrenamiento se procesa de forma independiente. +- El pipeline mezcla lecturas de archivos EDF y trabajo con NumPy, así que un pool pequeño de hilos puede reducir el tiempo total. + +Efecto observado: + +- El tiempo smoke bajó a 9.578 s en la primera ejecución tras paralelizar. +- Las ejecuciones de seguimiento midieron 9.762 s y 9.655 s. +- Conclusión: esta es la optimización dominante. + +Riesgo: + +- Medio. +- El acceso paralelo a archivos puede comportarse distinto en discos más lentos o en una infraestructura más limitada del Challenge. + +### 5. Caché persistente de features por registro + +Cambio: + +- Se añadió una caché en disco por registro `(site_id, patient_id, session_id)`. +- Antes de extraer features, el pipeline comprueba si ya existe el archivo de caché para ese paciente. +- Si el archivo existe, se reutiliza directamente; si no existe, se calcula y se guarda. + +Motivo: + +- La extracción fisiológica es el cuello de botella dominante. +- En iteraciones sucesivas de entrenamiento o inferencia se estaban recalculando features ya generadas para los mismos registros. + +Efecto esperado: + +- La primera ejecución mantiene un coste parecido al actual. +- Las siguientes ejecuciones sobre el mismo dataset reutilizan las features ya materializadas y reducen de forma importante el tiempo total. + +Riesgo: + +- Bajo a medio. +- Si cambia la lógica de extracción y se quiere regenerar todo, habrá que borrar manualmente la carpeta de caché. +- +## Plan De Rollback + +Si la submission se comporta distinto en el entorno del Challenge, revertir en este orden: + +1. Eliminar la extracción con hilos y restaurar el bucle secuencial original en `train_model`. +2. Eliminar las cachés de metadata de entrenamiento y volver a `load_demographics(...)` / `load_diagnoses(...)`. +3. Eliminar la caché de reglas de renombrado y volver a las llamadas directas a `load_rename_rules(...)`. +4. Rehabilitar la carga de anotaciones humanas solo si el vector de entrenamiento se modifica explícitamente para usar esas features. + +Este orden de rollback elimina primero la optimización de mayor riesgo y deja para el final los cambios de comportamiento más pequeños. + +## Archivos Modificados + +- `team_code.py` \ No newline at end of file diff --git a/evaluate_model.py b/evaluate_model.py index 6fc9466..5aff532 100644 --- a/evaluate_model.py +++ b/evaluate_model.py @@ -4,13 +4,16 @@ # This file contains functions for evaluating models for the Challenge. You can run it as follows: # -# python evaluate_model.py -d labels.csv -o predictions.csv -s scores.csv +# python evaluate_model.py -d labels.csv -o predictions.csv -p prevalence.csv -s scores.csv -t table.csv # -# where 'labels.csv' is a CSV file containing the labels, 'predictions.csv' is a CSV file containing containing the predictions, and -# 'scores.csv' (optional) is a collection of scores for the predictions. +# where 'labels.csv' is one or more CSV files containing the labels, 'predictions.csv' is one or more CSV files containing the +# predictions, 'prevalence.csv' is one or more CSV files containing labels to define the prevalence of the positive class at +# different ages, 'scores.csv' (optional) is a collection of scores for the predictions, and 'table.csv' (optional) is a table +# summary of the predictions at different ages. # -# The Challenge webpage describes the file formats and scoring functions. +# The Challenge webpage describes the file formats and scoring functions for this script. +# Import packages. import argparse import numpy as np import os @@ -18,102 +21,314 @@ import pandas as pd import sys -id_patients = 'BDSPPatientID' -id_labels = 'Cognitive_Impairment' -id_binary_predictions = 'Cognitive_Impairment' -id_probability_predictions = 'Cognitive_Impairment_Probability' +from collections import defaultdict +from sklearn.metrics import roc_auc_score, average_precision_score + +# Define headers. +id_site = 'SiteID' +id_patient = 'BDSPPatientID' +id_label = 'Cognitive_Impairment' +id_age = 'Age' +id_sex = 'Sex' +id_binary_prediction = 'Cognitive_Impairment' +id_probability_prediction = 'Cognitive_Impairment_Probability' # Parse arguments. def get_parser(): description = 'Evaluate the Challenge model.' parser = argparse.ArgumentParser(description=description) - parser.add_argument('-d', '--labels_folder', type=str, required=True) - parser.add_argument('-o', '--predictions_folder', type=str, required=True) + parser.add_argument('-d', '--labels_files', type=str, required=True, nargs='*') + parser.add_argument('-o', '--predictions_files', type=str, required=True, nargs='*') + parser.add_argument('-p', '--prevalence_files', type=str, required=True, nargs='*') parser.add_argument('-s', '--score_file', type=str, required=False) + parser.add_argument('-t', '--table_file', type=str, required=False) return parser -# Compute AUC. -def compute_auc(labels, predictions): - from sklearn.metrics import roc_auc_score, average_precision_score - auroc = roc_auc_score(labels, predictions, average='macro', sample_weight=None, max_fpr=None, multi_class='raise', labels=None) - auprc = average_precision_score(labels, predictions, average='macro', pos_label=1, sample_weight=None) - return auroc, auprc +# Compute the prevalence of the positive class at different ages. +def compute_prevalence(ages, prevalence_labels, prevalence_ages, gap=0): + m = len(ages) + n = len(prevalence_labels) + o = len(prevalence_ages) + assert(n == o) + + unique_ages = np.unique(ages[np.isfinite(ages)]) + age_to_labels = defaultdict(list) + for age in unique_ages: + for i in range(n): + if abs(age-prevalence_ages[i]) <= gap: + age_to_labels[age].append(prevalence_labels[i]) + + age_to_prevalence = dict() + for age in age_to_labels: + age_to_prevalence[age] = max(sum(age_to_labels[age]), 0.5)/len(age_to_labels[age]) + + return age_to_prevalence + +# Compute a prevalence-based reward metric. +def compute_reward(labels, predictions, ages, age_to_prevalence): + m = len(labels) + n = len(predictions) + o = len(ages) + assert(m == n == o) + + scores = np.zeros(n) + num_scores = 0 + + for i in range(n): + if np.isfinite(ages[i]): + p = age_to_prevalence[ages[i]] + p = min(max(p, 0.5/m), 1 - 0.5/m) # Ensure p is strictly greater than 0 and strictly less than 1. + + if labels[i] == 1 and predictions[i] == 1: + scores[i] = 1/p - 1 + elif labels[i] == 0 and predictions[i] == 1: + scores[i] = -1 + if labels[i] == 1 and predictions[i] == 0: + scores[i] = -1 + elif labels[i] == 0 and predictions[i] == 0: + scores[i] = 1/(1-p) - 1 + num_scores += 1 + + score = np.sum(scores)/num_scores + + return score + +# Compute the area under the receiver-operating characteristic curve conditioned on age. +def compute_auroc_age(labels, predictions, ages, gap=0): + m = len(labels) + n = len(predictions) + o = len(ages) + assert(m == n == o) + + idx_pos = [i for i in range(m) if labels[i] == 1] + idx_neg = [i for i in range(m) if labels[i] == 0] + num_pos = len(idx_pos) + num_neg = len(idx_neg) + + numer = 0 + denom = 0 + for i in range(num_pos): + for j in range(num_neg): + if abs(ages[idx_pos[i]] - ages[idx_neg[j]]) <= gap: + if predictions[idx_pos[i]] > predictions[idx_neg[j]]: + numer += 1 + elif predictions[idx_pos[i]] == predictions[idx_neg[j]]: + numer += 0.5 + denom += 1 + return numer/denom + +# Compute the age-weighted mean of the area under the receiver-operating characteristic curve across ages. +def compute_auroc_weighted(labels, predictions, ages, gap=0): + m = len(labels) + n = len(predictions) + o = len(ages) + assert(m == n == o) + + finite_ages = ages[np.isfinite(ages)] + range_ages = np.arange(np.min(finite_ages) - gap, np.max(finite_ages) + gap + 1) + p = len(finite_ages) + q = len(range_ages) + + idx_pos = [i for i in range(m) if labels[i] == 1] + idx_neg = [i for i in range(m) if labels[i] == 0] + num_pos = len(idx_pos) + num_neg = len(idx_neg) + + numer = np.zeros(q) + denom = np.zeros(q) + for k in range(q): + for i in range(num_pos): + for j in range(num_neg): + if abs(ages[idx_pos[i]] - range_ages[k]) <= gap and abs(ages[idx_neg[j]] - range_ages[k]) <= gap: + if predictions[idx_pos[i]] > predictions[idx_neg[j]]: + numer[k] += 1 + elif predictions[idx_pos[i]] == predictions[idx_neg[j]]: + numer[k] += 0.5 + denom[k] += 1 + + weights = np.zeros(q) + for k in range(q): + weights[k] = sum(1 for i in range(p) if abs(finite_ages[i] - range_ages[k]) <= gap) + + for k in range(q): + if denom[k] == 0: # Avoid NaN propagation + weights[k] = 0 + numer[k] = 0 + denom[k] = 1 + + weights = weights / np.sum(weights) + + return np.sum(weights * (numer / denom)) + +# Compute the area under the receiver-operating characteristic curve. +def compute_auroc(labels, predictions): + auroc = roc_auc_score(labels, predictions, sample_weight=None, max_fpr=None, multi_class='raise', labels=None) + return auroc + +# Compute the area under the precision-recall curve. +def compute_auprc(labels, predictions): + auprc = average_precision_score(labels, predictions, pos_label=1, sample_weight=None) + return auprc + +# Compute a confusion matrix. +def compute_confusion_matrix(labels, predictions): + n = np.size(labels) + tp = fp = fn = tn = 0 + for i in range(n): + if labels[i] == 1 and predictions[i] == 1: + tp += 1 + elif labels[i] == 0 and predictions[i] == 1: + fp += 1 + elif labels[i] == 1 and predictions[i] == 0: + fn += 1 + elif labels[i] == 0 and predictions[i] == 0: + tn += 1 + return tp, fp, fn, tn # Compute accuracy. def compute_accuracy(labels, predictions): - from sklearn.metrics import accuracy_score - accuracy = accuracy_score(labels, predictions, normalize=True, sample_weight=None) + tp, fp, fn, tn = compute_confusion_matrix(labels, predictions) + accuracy = (tp + tn) / (tp + fp + fn + tn) return accuracy -# Compute F-measure. +# Compute the F-measure. def compute_f_measure(labels, predictions): - from sklearn.metrics import f1_score - f_measure = f1_score(labels, predictions, pos_label=1, average='binary') + tp, fp, fn, tn = compute_confusion_matrix(labels, predictions) + f_measure = (2*tp) / (2*tp + fp + fn) return f_measure # Evaluate the models. -def evaluate_model(labels_file, predictions_file): - # Load the labels and predictions. - df_labels = pd.read_csv(labels_file) - df_labels.set_index(id_patients, inplace=True) - df_predictions = pd.read_csv(predictions_file) - df_predictions.set_index(id_patients, inplace=True) - - def standardize_bool(val): - s = str(val).strip().upper() - if s in ['TRUE', '1', '1.0', 'T', 'Y', 'YES']: return 1.0 - if s in ['FALSE', '0', '0.0', 'F', 'N', 'NO']: return 0.0 - return np.nan - - # Standardize the labels and predictions to be 0/1. - df_labels[id_labels] = df_labels[id_labels].apply(standardize_bool) - df_predictions[id_binary_predictions] = df_predictions[id_binary_predictions].apply(standardize_bool) +def evaluate_model(labels_files, predictions_files, prevalence_files): + # Load the labels, predictions, and prevalence data. + id_site_patient = id_site + '_' + id_patient + + df_labels = [pd.read_csv(labels_file) for labels_file in labels_files] + df_labels = pd.concat(df_labels) + df_labels.drop_duplicates(inplace=True) + df_labels[id_site_patient] = df_labels[id_site].astype(str) + '_' + df_labels[id_patient].astype(str) + df_labels.set_index(id_site_patient, inplace=True) + + df_predictions = [pd.read_csv(predictions_file) for predictions_file in predictions_files] + df_predictions = pd.concat(df_predictions) + df_predictions.drop_duplicates(inplace=True) + df_predictions[id_site_patient] = df_predictions[id_site].astype(str) + '_' + df_predictions[id_patient].astype(str) + df_predictions.set_index(id_site_patient, inplace=True) + + df_prevalence = [pd.read_csv(prevalence_file) for prevalence_file in prevalence_files] + df_prevalence = pd.concat(df_prevalence) + df_prevalence.drop_duplicates(inplace=True) + df_prevalence[id_site_patient] = df_prevalence[id_site].astype(str) + '_' + df_prevalence[id_patient].astype(str) + df_prevalence.set_index(id_site_patient, inplace=True) # Only consider patients with positive or negative labels. - df_labels = df_labels[(df_labels[id_labels] == 0) | (df_labels[id_labels] == 1)] + df_labels = df_labels[(df_labels[id_label] == 0) | (df_labels[id_label] == 1)] patients = df_labels.index num_patients = len(patients) - # Extract the labels and predictions. + # Extract the labels, predictions, and ages. labels = np.zeros(num_patients) binary_predictions = np.zeros(num_patients) probability_predictions = np.zeros(num_patients) + ages = np.zeros(num_patients) for i, patient in enumerate(patients): - label = df_labels.loc[patient, id_labels] + label = df_labels.loc[patient, id_label] labels[i] = label if patient in df_predictions.index: # Set missing predictions to 0. - binary_prediction = float(df_predictions.loc[patient, id_binary_predictions]) + binary_prediction = float(df_predictions.loc[patient, id_binary_prediction]) if binary_prediction == 0 or binary_prediction == 1: # Set invalid binary predictions to 0. binary_predictions[i] = binary_prediction - probability_prediction = float(df_predictions.loc[patient, id_probability_predictions]) + probability_prediction = float(df_predictions.loc[patient, id_probability_prediction]) if np.isfinite(probability_prediction): # Set invalid probability predictions to 0. probability_predictions[i] = probability_prediction + age = df_labels.loc[patient, id_age] + ages[i] = age + + # Validate the labels, predictions, and ages. + assert(labels.ndim == binary_predictions.ndim == probability_predictions.ndim == ages.ndim == 1) + assert(np.size(labels) == np.size(binary_predictions) == np.size(probability_predictions) == np.size(ages)) + assert(np.size(labels) > 0) + assert(np.all((labels==0) | (labels==1))) + assert(np.all((binary_predictions==0) | (binary_predictions==1))) + assert(np.all(~np.isnan(probability_predictions))) + + # Extract the prevalence of the positive class at different ages. + df_prevalence = df_prevalence[(df_prevalence[id_label] == 0) | (df_prevalence[id_label] == 1)] + prevalence_patients = df_prevalence.index + + num_prevalence_patients = len(prevalence_patients) + prevalence_labels = np.zeros(num_prevalence_patients) + prevalence_ages = np.zeros(num_prevalence_patients) + + for i, patient in enumerate(prevalence_patients): + label = df_prevalence.loc[patient, id_label] + prevalence_labels[i] = label + age = df_prevalence.loc[patient, id_age] + prevalence_ages[i] = age + + age_to_prevalence = compute_prevalence(ages, prevalence_labels, prevalence_ages, gap=2) # Evaluate the predictions. - auroc, auprc = compute_auc(labels, probability_predictions) + reward = compute_reward(labels, binary_predictions, ages, age_to_prevalence) + auroc_age = compute_auroc_age(labels, probability_predictions, ages, gap=2) + auroc_weighted = compute_auroc_weighted(labels, probability_predictions, ages, gap=2) + auroc = compute_auroc(labels, probability_predictions) + auprc = compute_auprc(labels, probability_predictions) accuracy = compute_accuracy(labels, binary_predictions) f_measure = compute_f_measure(labels, binary_predictions) + + table = list() + + header = ['Age', 'Prevalence (prevalence data)', '# positive labels (prevalence data)', '# negative labels (prevalence data)', \ + '# positive labels', '# negative labels', '# positive predictions', '# negative predictions', \ + '# true positives', '# false positives', '# false negatives', '# true negatives'] + table.append(header) - return auroc, auprc, accuracy, f_measure + m = len(prevalence_labels) + n = len(labels) + for age in np.unique(np.concatenate((ages, list(age_to_prevalence)))): + prevalence = age_to_prevalence[age] if age in age_to_prevalence else float('nan') + pa = sum(1 for i in range(m) if prevalence_ages[i] == age and prevalence_labels[i] == 1) + na = sum(1 for i in range(m) if prevalence_ages[i] == age and prevalence_labels[i] == 0) + pb = sum(1 for i in range(n) if ages[i] == age and labels[i] == 1) + nb = sum(1 for i in range(n) if ages[i] == age and labels[i] == 0) + pc = sum(1 for i in range(n) if ages[i] == age and binary_predictions[i] == 1) + nc = sum(1 for i in range(n) if ages[i] == age and binary_predictions[i] == 0) + tp = sum(1 for i in range(n) if ages[i] == age and labels[i] == 1 and binary_predictions[i] == 1) + fp = sum(1 for i in range(n) if ages[i] == age and labels[i] == 0 and binary_predictions[i] == 1) + fn = sum(1 for i in range(n) if ages[i] == age and labels[i] == 1 and binary_predictions[i] == 0) + tn = sum(1 for i in range(n) if ages[i] == age and labels[i] == 0 and binary_predictions[i] == 0) + row = [age, age_to_prevalence[age], pa, na, pb, nb, pc, nc, tp, fp, fn, tn] + table.append(row) + + return reward, auroc_age, auroc_weighted, auroc, auprc, accuracy, f_measure, table # Run the code. def run(args): # Compute the scores for the model predictions. - auroc, auprc, accuracy, f_measure = evaluate_model(args.labels_folder, args.predictions_folder) + reward, auroc_age, auroc_weighted, auroc, auprc, accuracy, f_measure, table = evaluate_model(args.labels_files, args.predictions_files, args.prevalence_files) output_string = \ - f'AUROC: {auroc:.3f}\n' \ + f'Reward: {reward:.3f}\n' + \ + f'Age-conditioned AUROC : {auroc_age:.3f}\n' + \ + f'Age-weighted AUROC : {auroc_weighted:.3f}\n' + \ + f'AUROC: {auroc:.3f}\n' + \ f'AUPRC: {auprc:.3f}\n' + \ - f'Accuracy: {accuracy:.3f}\n' \ + f'Accuracy: {accuracy:.3f}\n' + \ f'F-measure: {f_measure:.3f}\n' - # Output the scores to screen and/or a file. + # Output the scores to the screen or a file. if args.score_file: with open(args.score_file, 'w') as f: f.write(output_string) else: print(output_string) + # Output a table a breakdown of the results by age to a file. + if args.table_file: + with open(args.table_file, 'w') as f: + table_string = '\n'.join('\t'.join(map(str, row)) for row in table) + f.write(table_string) + if __name__ == '__main__': run(get_parser().parse_args(sys.argv[1:])) diff --git a/helper_code.py b/helper_code.py index e9bb2b5..1925dcd 100644 --- a/helper_code.py +++ b/helper_code.py @@ -15,7 +15,7 @@ from typing import Dict, List, Tuple, Any, Union, Tuple, Optional from collections import defaultdict -DEMOGRAPHICS_FILE = 'demographics.csv' +DEMOGRAPHICS_FILE = 'training_set/demographics.csv' PHYSIOLOGICAL_DATA_SUBFOLDER = 'physiological_data' ALGORITHMIC_ANNOTATIONS_SUBFOLDER = 'algorithmic_annotations' HUMAN_ANNOTATIONS_SUBFOLDER = 'human_annotations' diff --git a/outputs_test/demographics.csv b/outputs_test/demographics.csv new file mode 100644 index 0000000..3dba359 --- /dev/null +++ b/outputs_test/demographics.csv @@ -0,0 +1,416 @@ +SiteID,BDSPPatientID,CreationTime,BidsFolder,SessionID,Age,Sex,Race,Ethnicity,BMI,Time_to_Event,Cognitive_Impairment,Last_Known_Visit_Date,Time_to_Last_Visit,Cognitive_Impairment_Probability +S0001,111239981,11/02/2016 20:55,sub-S0001111239981,1.0,77.0,Female,White,Hispanic,37.09,1961.0,True,30/05/2022,2299.0,0.14538460969924927 +S0001,111249761,18/06/2011 2:33,sub-S0001111249761,1.0,75.0,Female,White,Not Hispanic,30.37,,True,02/04/2024,4671.0,0.6455416083335876 +S0001,111261966,30/07/2010 2:55,sub-S0001111261966,1.0,71.0,Female,White,Not Hispanic,30.13,,True,21/09/2023,4800.0,0.5782973766326904 +S0001,111390308,20/12/2008 21:51,sub-S0001111390308,1.0,70.0,Female,White,Not Hispanic,26.9,,True,18/11/2023,5445.0,0.40108874440193176 +S0001,111439208,15/10/2011 0:00,sub-S0001111439208,1.0,63.0,Male,White,Not Hispanic,26.12,,True,05/09/2022,3978.0,0.4706968069076538 +S0001,111457455,04/01/2017 21:47,sub-S0001111457455,1.0,73.0,Female,White,Not Hispanic,28.32,,True,21/04/2024,2663.0,0.9163349866867065 +S0001,111520696,25/02/2011 3:36,sub-S0001111520696,1.0,78.0,Male,White,Not Hispanic,27.68,,True,11/09/2023,4580.0,0.5654754638671875 +S0001,111561964,04/12/2015 22:38,sub-S0001111561964,1.0,68.0,Male,White,Not Hispanic,31.51,,True,30/03/2023,2672.0,0.39139872789382935 +S0001,111608665,07/06/2010 23:28,sub-S0001111608665,1.0,69.0,Male,White,Not Hispanic,44.2,,True,18/05/2022,4362.0,0.6186583042144775 +S0001,111613441,18/11/2016 22:32,sub-S0001111613441,1.0,64.0,Male,White,Not Hispanic,33.52,1632.0,True,24/12/2022,2226.0,0.8189074397087097 +S0001,111625940,01/07/2013 2:02,sub-S0001111625940,1.0,68.0,Male,White,Not Hispanic,27.91,,True,19/06/2023,3639.0,0.39139872789382935 +S0001,111640757,28/04/2010 2:11,sub-S0001111640757,1.0,73.0,Male,White,Not Hispanic,25.72,,True,03/03/2022,4326.0,0.8519553542137146 +S0001,111683364,01/06/2014 22:07,sub-S0001111683364,1.0,66.0,Male,White,Not Hispanic,27.41,,True,11/11/2022,3084.0,0.45043623447418213 +S0001,111695024,07/03/2013 3:03,sub-S0001111695024,1.0,78.0,Female,White,Not Hispanic,34.31,,True,01/09/2024,4195.0,0.6120396256446838 +S0001,111722285,08/03/2009 22:22,sub-S0001111722285,1.0,72.0,Female,Asian,Not Hispanic,22.96,,True,08/02/2018,3258.0,0.5070144534111023 +S0001,111781598,03/08/2019 18:20,sub-S0001111781598,1.0,79.0,Male,White,Not Hispanic,26.15,1190.0,True,15/03/2024,1685.0,0.4199504256248474 +S0001,111844094,22/02/2010 23:31,sub-S0001111844094,1.0,61.0,Male,White,Hispanic,35.28,1560.0,True,20/05/2023,4834.0,0.6737512946128845 +S0001,111860137,05/03/2011 23:11,sub-S0001111860137,1.0,67.0,Female,White,Not Hispanic,34.86,,True,01/01/2023,4319.0,0.6473630666732788 +S0001,111916963,19/07/2010 22:17,sub-S0001111916963,1.0,81.0,Female,White,Not Hispanic,23.53,,True,16/02/2024,4959.0,0.40572720766067505 +S0001,111932587,12/06/2011 22:01,sub-S0001111932587,1.0,81.0,Male,White,Not Hispanic,37.26,,True,22/09/2020,3389.0,0.8153370022773743 +S0001,112040083,14/08/2011 2:36,sub-S0001112040083,1.0,70.0,Male,White,Not Hispanic,25.55,1394.0,True,03/01/2023,4159.0,0.483925461769104 +S0001,112059742,19/12/2011 2:10,sub-S0001112059742,1.0,68.0,Female,White,Not Hispanic,19.13,,True,27/12/2022,4025.0,0.346417635679245 +S0001,112078298,05/06/2019 19:28,sub-S0001112078298,1.0,78.0,Male,White,Not Hispanic,29.24,1126.0,True,12/01/2023,1316.0,0.5654754638671875 +S0001,112087330,09/01/2017 23:07,sub-S0001112087330,1.0,75.0,Female,White,Not Hispanic,36.73,,True,12/06/2024,2710.0,0.6455416083335876 +S0001,112141915,11/11/2012 22:05,sub-S0001112141915,1.0,74.0,Male,White,Not Hispanic,26.39,,True,23/12/2022,3693.0,0.39002496004104614 +S0001,112156568,09/06/2011 3:31,sub-S0001112156568,1.0,71.0,Male,White,Not Hispanic,28.62,,True,15/04/2024,4693.0,0.40006473660469055 +S0001,112167991,20/10/2010 3:38,sub-S0001112167991,1.0,61.0,Female,White,Unavailable,41.93,,True,17/02/2024,4867.0,0.824192225933075 +S0001,112194534,06/05/2017 21:17,sub-S0001112194534,1.0,63.0,Male,White,Not Hispanic,32.76,1169.0,True,03/09/2022,1945.0,0.4706968069076538 +S0001,112229480,03/01/2011 3:14,sub-S0001112229480,1.0,68.0,Male,Black,Not Hispanic,42.22,,True,18/12/2021,4001.0,0.39139872789382935 +S0001,112287792,22/01/2013 21:32,sub-S0001112287792,1.0,56.0,Male,White,Not Hispanic,36.01,,True,05/01/2023,3634.0,0.501916766166687 +S0001,112293206,05/04/2009 22:59,sub-S0001112293206,1.0,66.0,Male,White,Not Hispanic,26.99,,True,09/01/2022,4661.0,0.45043623447418213 +S0001,112313733,26/04/2011 21:45,sub-S0001112313733,1.0,76.0,Male,White,Not Hispanic,28.42,,True,04/10/2022,4178.0,0.2753254175186157 +S0001,112331277,14/12/2012 22:49,sub-S0001112331277,1.0,75.0,Male,White,Not Hispanic,24.57,,True,16/09/2024,4293.0,0.27257463335990906 +S0001,112361957,13/12/2019 20:01,sub-S0001112361957,2.0,53.0,Male,White,Not Hispanic,28.75,1258.0,True,25/05/2023,1258.0,0.6771898865699768 +S0001,112379247,09/09/2011 1:15,sub-S0001112379247,1.0,85.0,Male,White,Not Hispanic,30.75,2240.0,True,17/03/2021,3476.0,0.6813521385192871 +S0001,112382548,15/04/2017 22:31,sub-S0001112382548,1.0,72.0,Male,Black,Not Hispanic,33.76,,True,26/04/2024,2567.0,0.5197765827178955 +S0001,112453372,27/04/2017 22:56,sub-S0001112453372,1.0,75.0,Female,White,Not Hispanic,30.72,1475.0,True,26/08/2023,2311.0,0.6455416083335876 +S0001,112507998,28/08/2012 1:58,sub-S0001112507998,1.0,60.0,Female,White,Not Hispanic,27.2,,True,12/09/2023,4031.0,0.5012294054031372 +S0001,112516993,12/04/2010 22:51,sub-S0001112516993,1.0,64.0,Male,White,Not Hispanic,39.63,,True,16/09/2024,5270.0,0.8189074397087097 +S0001,112601406,05/02/2008 22:55,sub-S0001112601406,1.0,61.0,Female,White,Not Hispanic,30.94,,True,28/03/2023,5529.0,0.824192225933075 +S0001,112649481,09/02/2012 2:23,sub-S0001112649481,1.0,71.0,Male,White,Not Hispanic,29.1,,True,05/11/2023,4286.0,0.40006473660469055 +S0001,112722303,14/05/2016 22:30,sub-S0001112722303,1.0,73.0,Female,White,Not Hispanic,24.55,,True,03/01/2024,2789.0,0.9163349866867065 +S0001,112724496,26/07/2013 0:00,sub-S0001112724496,1.0,68.0,Male,White,Not Hispanic,25.28,,True,05/07/2024,3997.0,0.39139872789382935 +S0001,112747541,23/05/2015 21:47,sub-S0001112747541,1.0,58.0,Female,White,Not Hispanic,29.78,,True,23/12/2023,3135.0,0.46824002265930176 +S0001,112827546,16/09/2017 22:23,sub-S0001112827546,1.0,89.0,Male,White,Not Hispanic,24.14,1236.0,True,08/06/2023,2090.0,0.634381115436554 +S0001,112860352,01/10/2014 21:54,sub-S0001112860352,1.0,63.0,Female,White,Not Hispanic,45.44,,True,03/08/2023,3227.0,0.7037391066551208 +S0001,112893779,08/12/2016 22:35,sub-S0001112893779,1.0,74.0,Male,White,Not Hispanic,29.08,,True,15/12/2023,2562.0,0.39002496004104614 +S0001,112894687,11/11/2009 2:31,sub-S0001112894687,1.0,61.0,Male,Black,Not Hispanic,32.23,,True,18/04/2021,4175.0,0.6737512946128845 +S0001,112895854,12/08/2015 22:19,sub-S0001112895854,1.0,77.0,Female,White,Not Hispanic,29.2,,True,09/12/2022,2675.0,0.14538460969924927 +S0001,112930038,11/03/2017 21:59,sub-S0001112930038,1.0,60.0,Female,White,Not Hispanic,28.67,,True,03/09/2024,2732.0,0.5012294054031372 +S0001,112936324,15/11/2009 21:51,sub-S0001112936324,1.0,73.0,Male,White,Not Hispanic,29.84,,True,27/01/2018,2994.0,0.8519553542137146 +S0001,112960676,23/06/2008 22:16,sub-S0001112960676,1.0,78.0,Male,White,Not Hispanic,31.97,,True,29/08/2017,3353.0,0.5654754638671875 +S0001,112979679,24/04/2013 3:29,sub-S0001112979679,1.0,64.0,Male,White,Not Hispanic,29.51,,True,30/03/2023,3626.0,0.8189074397087097 +S0001,112993336,23/09/2015 23:09,sub-S0001112993336,1.0,87.0,Male,White,Not Hispanic,32.19,,True,11/06/2023,2817.0,0.506742537021637 +S0001,113038747,02/02/2013 0:01,sub-S0001113038747,1.0,76.0,Male,White,Not Hispanic,32.75,,True,26/07/2024,4191.0,0.2753254175186157 +S0001,113078805,15/10/2009 21:18,sub-S0001113078805,1.0,69.0,Female,Unavailable,Not Hispanic,18.54,,True,05/06/2023,4980.0,0.6564435362815857 +S0001,113112903,17/06/2011 2:19,sub-S0001113112903,1.0,80.0,Male,White,Not Hispanic,29.32,,True,11/06/2023,4376.0,0.34214648604393005 +S0001,113127215,18/05/2009 21:48,sub-S0001113127215,1.0,73.0,Female,White,Not Hispanic,24.34,,True,15/04/2019,3618.0,0.9163349866867065 +S0001,113127704,14/01/2013 1:53,sub-S0001113127704,1.0,65.0,Male,White,Not Hispanic,29.42,,True,04/11/2023,3945.0,0.4400545656681061 +S0001,113135566,22/05/2014 23:03,sub-S0001113135566,2.0,67.0,Female,White,Not Hispanic,40.53,1780.0,True,10/02/2023,3185.0,0.6473630666732788 +S0001,113137391,03/09/2009 23:16,sub-S0001113137391,1.0,70.0,Male,White,Not Hispanic,28.64,,True,15/10/2018,3328.0,0.483925461769104 +S0001,113169703,18/01/2012 3:08,sub-S0001113169703,1.0,70.0,Male,White,Not Hispanic,30.71,,True,02/05/2022,3756.0,0.483925461769104 +S0001,113333305,03/01/2013 22:09,sub-S0001113333305,1.0,75.0,Female,White,Not Hispanic,20.92,1170.0,True,04/09/2017,1704.0,0.6455416083335876 +S0001,113356108,22/07/2008 23:05,sub-S0001113356108,1.0,62.0,Female,White,Not Hispanic,37.15,,True,14/07/2023,5469.0,0.7453621029853821 +S0001,113407451,12/04/2011 1:58,sub-S0001113407451,1.0,51.0,Male,White,Not Hispanic,32.42,,True,05/09/2020,3433.0,0.4153312146663666 +S0001,113446885,01/09/2013 3:07,sub-S0001113446885,1.0,75.0,Male,White,Not Hispanic,23.07,,True,17/07/2023,3605.0,0.27257463335990906 +S0001,113452157,14/01/2010 1:40,sub-S0001113452157,1.0,71.0,Female,White,Unavailable,26.79,,True,21/06/2017,2714.0,0.5782973766326904 +S0001,113469297,29/05/2011 23:32,sub-S0001113469297,1.0,70.0,Female,White,Not Hispanic,32.91,,True,05/08/2023,4450.0,0.40108874440193176 +S0001,113578395,31/07/2010 23:00,sub-S0001113578395,1.0,75.0,Male,Others,Not Hispanic,35.67,,True,27/08/2018,2948.0,0.27257463335990906 +S0001,113620011,01/05/2013 3:08,sub-S0001113620011,1.0,67.0,Male,White,Unavailable,26.53,,True,11/03/2022,3235.0,0.9195594787597656 +S0001,113641132,12/12/2010 2:09,sub-S0001113641132,1.0,59.0,Female,White,Unavailable,42.29,,True,09/07/2023,4591.0,0.5383100509643555 +S0001,113841543,11/12/2010 1:35,sub-S0001113841543,1.0,65.0,Male,White,Not Hispanic,25.09,,True,05/05/2023,4527.0,0.4400545656681061 +S0001,113855605,06/05/2010 23:03,sub-S0001113855605,1.0,76.0,Male,White,Not Hispanic,26.95,,True,07/03/2023,4687.0,0.2753254175186157 +S0001,114053187,11/05/2009 22:13,sub-S0001114053187,1.0,61.0,Male,White,Not Hispanic,30.14,,True,27/04/2023,5098.0,0.6737512946128845 +S0001,114102759,13/10/2010 21:57,sub-S0001114102759,1.0,76.0,Male,White,Not Hispanic,30.63,,True,28/04/2024,4945.0,0.2753254175186157 +S0001,114110665,08/07/2010 2:00,sub-S0001114110665,1.0,65.0,Male,White,Not Hispanic,32.62,,True,08/12/2023,4900.0,0.4400545656681061 +S0001,114119353,27/07/2016 22:16,sub-S0001114119353,1.0,67.0,Female,White,Not Hispanic,24.47,,True,14/02/2024,2757.0,0.6473630666732788 +S0001,114124779,10/12/2016 21:48,sub-S0001114124779,1.0,55.0,Male,White,Not Hispanic,34.02,,True,10/02/2024,2617.0,0.19085277616977692 +S0001,114137420,15/03/2017 21:55,sub-S0001114137420,1.0,67.0,Male,White,Not Hispanic,27.8,,True,14/08/2024,2708.0,0.9195594787597656 +S0001,114214742,06/11/2011 2:31,sub-S0001114214742,1.0,61.0,Male,White,Not Hispanic,34.93,,True,03/07/2023,4256.0,0.6737512946128845 +S0001,114249812,27/02/2010 2:34,sub-S0001114249812,1.0,71.0,Female,White,Not Hispanic,32.02,,True,17/06/2023,4857.0,0.5782973766326904 +S0001,114287643,12/11/2011 2:54,sub-S0001114287643,1.0,77.0,Male,White,Not Hispanic,25.71,,True,02/02/2023,4099.0,0.8424844145774841 +S0001,114378497,17/03/2014 21:39,sub-S0001114378497,1.0,71.0,Male,White,Not Hispanic,28.59,,True,08/06/2023,3369.0,0.40006473660469055 +S0001,114400993,14/07/2013 1:41,sub-S0001114400993,1.0,76.0,Male,White,Not Hispanic,22.84,,True,25/06/2024,3998.0,0.2753254175186157 +S0001,114429880,14/04/2010 23:36,sub-S0001114429880,1.0,82.0,Male,White,Not Hispanic,29.93,,True,29/11/2023,4976.0,0.44877195358276367 +S0001,114431014,08/05/2011 3:10,sub-S0001114431014,1.0,60.0,Male,White,Not Hispanic,37.33,,True,19/11/2023,4577.0,0.24748243391513824 +S0001,114564587,27/10/2016 21:17,sub-S0001114564587,1.0,67.0,Female,Black,Not Hispanic,34.14,1947.0,True,01/12/2022,2225.0,0.6473630666732788 +S0001,114617339,13/03/2010 2:08,sub-S0001114617339,1.0,71.0,Female,White,Not Hispanic,26.9,,True,22/03/2022,4391.0,0.5782973766326904 +S0001,114730678,23/11/2010 21:06,sub-S0001114730678,1.0,66.0,Male,White,Not Hispanic,38.26,,True,13/04/2023,4523.0,0.45043623447418213 +S0001,114750697,08/05/2009 21:24,sub-S0001114750697,1.0,85.0,Male,White,Not Hispanic,29.47,,True,28/11/2022,4951.0,0.6813521385192871 +S0001,114833800,28/07/2008 23:31,sub-S0001114833800,1.0,57.0,Female,White,Not Hispanic,23.07,,True,27/04/2023,5385.0,0.6525198221206665 +S0001,114841089,28/08/2010 21:43,sub-S0001114841089,1.0,67.0,Male,White,Not Hispanic,24.17,,True,25/09/2024,5141.0,0.9195594787597656 +S0001,114856764,09/04/2016 22:56,sub-S0001114856764,1.0,62.0,Female,White,Not Hispanic,32.14,,True,20/07/2023,2657.0,0.7453621029853821 +S0001,114912372,29/06/2012 2:14,sub-S0001114912372,1.0,74.0,Female,White,Not Hispanic,33.44,,True,03/09/2022,3717.0,0.33417800068855286 +S0001,114920071,10/09/2014 23:20,sub-S0001114920071,1.0,71.0,Male,White,Not Hispanic,30.85,,True,13/03/2023,3105.0,0.40006473660469055 +S0001,114948418,03/05/2012 3:05,sub-S0001114948418,1.0,72.0,Male,White,Not Hispanic,35.75,,True,29/04/2024,4378.0,0.5197765827178955 +S0001,114970247,22/04/2013 1:50,sub-S0001114970247,1.0,59.0,Male,Unavailable,Hispanic,49.92,,True,06/09/2024,4154.0,0.2705105245113373 +S0001,115035582,26/11/2011 2:06,sub-S0001115035582,1.0,76.0,Male,White,Not Hispanic,24.34,2291.0,True,01/02/2022,3719.0,0.2753254175186157 +S0001,115036729,22/06/2010 21:49,sub-S0001115036729,1.0,69.0,Male,White,Not Hispanic,26.59,,True,18/06/2024,5109.0,0.6186583042144775 +S0001,115045037,18/10/2011 3:50,sub-S0001115045037,1.0,57.0,Male,Others,Hispanic,37.54,,True,11/12/2020,3341.0,0.5427361130714417 +S0001,115047250,25/10/2010 2:08,sub-S0001115047250,1.0,59.0,Male,White,Not Hispanic,25.95,,True,28/11/2023,4781.0,0.2705105245113373 +S0001,115075565,02/03/2017 21:20,sub-S0001115075565,1.0,80.0,Male,White,Not Hispanic,32.37,2128.0,True,23/04/2024,2608.0,0.34214648604393005 +S0001,115092772,06/11/2016 22:21,sub-S0001115092772,1.0,70.0,Male,White,Not Hispanic,34.84,2539.0,True,16/06/2024,2778.0,0.483925461769104 +S0001,115124810,16/11/2014 22:53,sub-S0001115124810,1.0,73.0,Female,White,Not Hispanic,32.09,,True,07/02/2023,3004.0,0.9163349866867065 +S0001,115306466,18/12/2009 2:31,sub-S0001115306466,1.0,71.0,Male,White,Not Hispanic,30.5,,True,05/02/2022,4431.0,0.40006473660469055 +S0001,115509440,10/03/2013 0:00,sub-S0001115509440,1.0,70.0,Male,White,Not Hispanic,23.27,,True,17/07/2021,3051.0,0.483925461769104 +S0001,115549333,12/02/2011 3:15,sub-S0001115549333,1.0,55.0,Female,Black,Not Hispanic,30.99,,True,09/10/2022,4256.0,0.64033043384552 +S0001,115614313,30/08/2008 23:00,sub-S0001115614313,1.0,75.0,Male,White,Not Hispanic,23.15,,True,16/09/2016,2938.0,0.27257463335990906 +S0001,115627888,24/07/2012 1:31,sub-S0001115627888,1.0,85.0,Female,White,Not Hispanic,35.67,1246.0,True,05/06/2016,1411.0,0.6497296094894409 +S0001,115685485,30/01/2009 22:03,sub-S0001115685485,1.0,50.0,Male,White,Not Hispanic,39.77,,True,20/06/2023,5253.0,0.2891353964805603 +S0001,115724572,10/12/2011 1:50,sub-S0001115724572,1.0,57.0,Female,White,Not Hispanic,34.94,,True,27/01/2024,4430.0,0.6525198221206665 +S0001,115804130,02/04/2016 22:00,sub-S0001115804130,2.0,78.0,Male,White,Not Hispanic,26.91,,True,09/06/2023,2623.0,0.5654754638671875 +S0001,115810419,28/08/2011 1:56,sub-S0001115810419,1.0,60.0,Female,White,Not Hispanic,20.89,,True,16/02/2023,4189.0,0.5012294054031372 +S0001,115823207,19/08/2009 23:28,sub-S0001115823207,1.0,80.0,Female,White,Not Hispanic,20.97,,True,02/12/2023,5217.0,0.4938763380050659 +S0001,115866465,22/03/2017 23:41,sub-S0001115866465,1.0,75.0,Male,Unavailable,Not Hispanic,17.78,1445.0,True,13/09/2023,2365.0,0.27257463335990906 +S0001,115881418,14/03/2016 22:35,sub-S0001115881418,1.0,54.0,Male,White,Not Hispanic,36.74,,True,06/06/2023,2639.0,0.47246167063713074 +S0001,115883585,03/03/2018 19:18,sub-S0001115883585,1.0,71.0,Male,White,Not Hispanic,24.12,1138.0,True,05/01/2023,1768.0,0.40006473660469055 +S0001,115946863,19/09/2008 22:55,sub-S0001115946863,1.0,65.0,Male,White,Not Hispanic,24.34,1450.0,True,06/08/2022,5068.0,0.4400545656681061 +S0001,116017075,08/10/2012 22:29,sub-S0001116017075,1.0,61.0,Male,White,Not Hispanic,24.93,,True,26/12/2023,4095.0,0.6737512946128845 +S0001,116045066,20/02/2010 2:15,sub-S0001116045066,1.0,64.0,Female,Black,Not Hispanic,24.11,,True,16/03/2023,4771.0,0.8236727714538574 +S0001,116052557,05/01/2013 21:59,sub-S0001116052557,1.0,70.0,Female,White,Not Hispanic,20.03,,True,04/12/2023,3984.0,0.40108874440193176 +S0001,116140777,30/06/2009 2:02,sub-S0001116140777,1.0,70.0,Male,Asian,Not Hispanic,24.41,,True,10/01/2023,4941.0,0.483925461769104 +S0001,116264624,05/07/2012 1:38,sub-S0001116264624,1.0,71.0,Male,White,Not Hispanic,40.19,1789.0,True,18/11/2021,3422.0,0.40006473660469055 +S0001,116283126,08/04/2009 0:43,sub-S0001116283126,1.0,66.0,Male,White,Not Hispanic,35.58,1892.0,True,29/01/2023,5043.0,0.45043623447418213 +S0001,116289727,10/09/2010 1:57,sub-S0001116289727,1.0,72.0,Female,White,Not Hispanic,25.21,,True,10/04/2021,3864.0,0.5070144534111023 +S0001,116308865,22/08/2011 2:44,sub-S0001116308865,1.0,75.0,Female,White,Not Hispanic,35.02,,True,17/11/2023,4469.0,0.6455416083335876 +S0001,116330821,25/12/2009 1:49,sub-S0001116330821,1.0,74.0,Male,White,Not Hispanic,25.25,,True,26/11/2022,4718.0,0.39002496004104614 +S0001,116342936,07/07/2015 22:35,sub-S0001116342936,1.0,76.0,Male,White,Not Hispanic,24.57,,True,26/08/2023,2971.0,0.2753254175186157 +S0001,116399255,15/05/2016 22:12,sub-S0001116399255,1.0,59.0,Female,White,Not Hispanic,23.36,,True,24/10/2023,2717.0,0.5383100509643555 +S0001,116473509,13/11/2013 22:10,sub-S0001116473509,1.0,77.0,Male,Asian,Not Hispanic,23.16,,True,10/02/2024,3740.0,0.8424844145774841 +S0001,116587467,09/10/2010 2:08,sub-S0001116587467,1.0,71.0,Male,White,Not Hispanic,26.62,,True,24/03/2023,4548.0,0.40006473660469055 +S0001,116604699,02/03/2014 22:54,sub-S0001116604699,1.0,63.0,Female,Others,Hispanic,38.6,,True,19/09/2021,2757.0,0.7037391066551208 +S0001,116662355,26/02/2019 19:45,sub-S0001116662355,1.0,77.0,Male,White,Not Hispanic,23.92,1372.0,True,20/09/2023,1666.0,0.8424844145774841 +S0001,116697035,17/10/2011 1:41,sub-S0001116697035,1.0,61.0,Female,White,Not Hispanic,43.93,,True,23/06/2020,3171.0,0.824192225933075 +S0001,116772588,14/02/2011 1:37,sub-S0001116772588,1.0,74.0,Male,Unavailable,Hispanic,26.24,,True,06/10/2022,4251.0,0.39002496004104614 +S0001,116774437,17/03/2009 21:55,sub-S0001116774437,1.0,68.0,Female,White,Not Hispanic,23.47,,True,04/02/2023,5071.0,0.346417635679245 +S0001,116889560,02/09/2009 21:53,sub-S0001116889560,1.0,53.0,Female,White,Not Hispanic,43.15,,True,19/07/2023,5067.0,0.3207913041114807 +S0001,116901749,15/05/2014 22:43,sub-S0001116901749,1.0,66.0,Male,White,Not Hispanic,33.33,,True,16/04/2024,3623.0,0.45043623447418213 +S0001,116904925,06/11/2012 3:18,sub-S0001116904925,1.0,54.0,Male,White,Not Hispanic,27.32,,True,14/12/2023,4054.0,0.47246167063713074 +S0001,116988457,01/05/2010 22:22,sub-S0001116988457,1.0,63.0,Male,White,Not Hispanic,30.81,,True,14/08/2023,4852.0,0.4706968069076538 +S0001,116998567,21/08/2011 22:05,sub-S0001116998567,1.0,75.0,Male,White,Not Hispanic,27.69,,True,16/07/2024,4712.0,0.27257463335990906 +S0001,117019749,28/08/2009 23:34,sub-S0001117019749,1.0,70.0,Male,White,Not Hispanic,24.03,,True,05/10/2023,5150.0,0.483925461769104 +S0001,117057718,11/02/2015 22:57,sub-S0001117057718,2.0,66.0,Male,White,Not Hispanic,38.24,,True,15/10/2022,2802.0,0.45043623447418213 +S0001,117102202,21/10/2009 1:36,sub-S0001117102202,1.0,59.0,Female,White,Hispanic,20.65,,True,29/11/2021,4421.0,0.5383100509643555 +S0001,117168117,06/10/2008 22:04,sub-S0001117168117,1.0,66.0,Male,White,Not Hispanic,25.79,,True,18/06/2020,4272.0,0.45043623447418213 +S0001,117181095,21/01/2009 22:21,sub-S0001117181095,1.0,55.0,Male,White,Not Hispanic,36.51,,True,26/01/2023,5117.0,0.19085277616977692 +S0001,117236297,16/01/2009 22:09,sub-S0001117236297,1.0,63.0,Female,White,Not Hispanic,31.02,,True,26/02/2024,5518.0,0.7037391066551208 +S0001,117262757,30/05/2013 3:07,sub-S0001117262757,1.0,75.0,Female,White,Not Hispanic,23.16,,True,18/12/2023,3853.0,0.6455416083335876 +S0001,117356069,30/06/2012 2:10,sub-S0001117356069,1.0,62.0,Female,White,Not Hispanic,32.01,1643.0,True,25/05/2018,2154.0,0.7453621029853821 +S0001,117392261,18/08/2016 22:06,sub-S0001117392261,2.0,69.0,Male,White,Not Hispanic,32.33,,True,22/12/2023,2681.0,0.6186583042144775 +S0001,117404228,20/09/2008 22:05,sub-S0001117404228,1.0,79.0,Male,White,Unavailable,22.6,1732.0,True,27/08/2017,3262.0,0.4199504256248474 +S0001,117408887,06/01/2009 22:38,sub-S0001117408887,1.0,80.0,Female,White,Not Hispanic,30.87,,True,22/05/2023,5248.0,0.4938763380050659 +S0001,117438884,17/08/2013 22:52,sub-S0001117438884,1.0,64.0,Female,White,Not Hispanic,33.45,,True,15/05/2023,3557.0,0.8236727714538574 +S0001,117440861,11/11/2011 1:44,sub-S0001117440861,1.0,70.0,Male,White,Not Hispanic,28.95,,True,03/02/2020,3005.0,0.483925461769104 +S0001,117553277,28/09/2015 22:54,sub-S0001117553277,1.0,77.0,Male,White,Not Hispanic,35.95,2467.0,True,28/12/2022,2647.0,0.8424844145774841 +S0001,117671783,20/11/2008 22:59,sub-S0001117671783,1.0,69.0,Male,White,Not Hispanic,29.45,2045.0,True,22/08/2020,4292.0,0.6186583042144775 +S0001,117676967,10/10/2015 23:18,sub-S0001117676967,1.0,78.0,Male,White,Not Hispanic,27.23,,True,23/02/2023,2692.0,0.5654754638671875 +S0001,117784533,12/04/2015 21:35,sub-S0001117784533,1.0,79.0,Female,White,Not Hispanic,30.49,,True,24/11/2023,3147.0,0.7526390552520752 +S0001,117788322,08/09/2010 22:03,sub-S0001117788322,1.0,73.0,Male,White,Not Hispanic,27.8,,True,01/11/2022,4436.0,0.8519553542137146 +S0001,117805802,07/01/2008 23:23,sub-S0001117805802,1.0,58.0,Male,White,Not Hispanic,,,True,30/05/2017,3430.0,0.23777179419994354 +S0001,117809853,24/06/2017 21:33,sub-S0001117809853,1.0,80.0,Male,White,Not Hispanic,26.23,1186.0,True,01/11/2023,2320.0,0.34214648604393005 +S0001,117863261,07/06/2016 21:35,sub-S0001117863261,1.0,69.0,Male,White,Not Hispanic,27.69,1358.0,True,11/08/2023,2620.0,0.6186583042144775 +S0001,117930679,04/12/2013 21:34,sub-S0001117930679,1.0,79.0,Male,White,Not Hispanic,29.23,,True,28/08/2023,3553.0,0.4199504256248474 +S0001,117949855,23/08/2008 22:57,sub-S0001117949855,1.0,68.0,Male,White,Not Hispanic,27.93,,True,27/03/2017,3137.0,0.39139872789382935 +S0001,117953563,10/02/2017 22:57,sub-S0001117953563,1.0,62.0,Female,White,Not Hispanic,30.64,,True,30/05/2024,2665.0,0.7453621029853821 +S0001,117960082,11/08/2012 0:00,sub-S0001117960082,1.0,67.0,Female,White,Not Hispanic,24.79,,True,03/01/2023,3797.0,0.6473630666732788 +S0001,117998576,22/01/2010 23:33,sub-S0001117998576,1.0,70.0,Male,White,Not Hispanic,32.53,,True,09/03/2022,4428.0,0.483925461769104 +S0001,118007341,05/12/2009 2:10,sub-S0001118007341,1.0,68.0,Male,White,Not Hispanic,34.21,,True,06/07/2017,2769.0,0.39139872789382935 +S0001,118007817,08/02/2009 22:36,sub-S0001118007817,1.0,80.0,Female,White,Not Hispanic,28.21,,True,16/01/2024,5454.0,0.4938763380050659 +S0001,118035891,06/11/2008 22:18,sub-S0001118035891,1.0,80.0,Female,White,Not Hispanic,22.57,,True,22/02/2022,4855.0,0.4938763380050659 +S0001,118065598,18/07/2010 2:26,sub-S0001118065598,1.0,70.0,Male,White,Not Hispanic,31.32,,True,19/09/2017,2619.0,0.483925461769104 +S0001,118092066,15/10/2014 22:56,sub-S0001118092066,1.0,88.0,Male,White,Not Hispanic,24.11,1268.0,True,15/06/2019,1703.0,0.6507008671760559 +S0001,118147076,18/09/2015 23:25,sub-S0001118147076,2.0,72.0,Male,White,Not Hispanic,24.44,,True,02/06/2023,2813.0,0.5197765827178955 +S0001,118163191,01/12/2009 22:32,sub-S0001118163191,1.0,68.0,Female,White,Not Hispanic,21.85,,True,23/05/2024,5286.0,0.346417635679245 +S0001,118184191,23/04/2010 3:20,sub-S0001118184191,1.0,77.0,Male,White,Not Hispanic,24.97,,True,26/09/2023,4903.0,0.8424844145774841 +S0001,118216285,07/02/2016 23:01,sub-S0001118216285,1.0,67.0,Male,Black,Not Hispanic,27.08,,True,02/09/2023,2763.0,0.9195594787597656 +S0001,118280628,19/04/2013 22:56,sub-S0001118280628,1.0,65.0,Male,White,Not Hispanic,25.29,,True,27/05/2023,3689.0,0.4400545656681061 +S0001,118283012,31/03/2011 22:17,sub-S0001118283012,1.0,77.0,Female,White,Not Hispanic,39.32,,True,24/03/2023,4375.0,0.14538460969924927 +S0001,118324303,29/06/2011 3:32,sub-S0001118324303,1.0,70.0,Male,White,Not Hispanic,31.54,,True,20/08/2023,4434.0,0.483925461769104 +S0001,118326393,05/03/2017 21:52,sub-S0001118326393,1.0,78.0,Male,White,Not Hispanic,38.26,1298.0,True,07/11/2022,2072.0,0.5654754638671875 +S0001,118395905,10/11/2011 22:16,sub-S0001118395905,1.0,71.0,Male,White,Not Hispanic,22.34,,True,22/03/2024,4515.0,0.40006473660469055 +S0001,118411164,06/02/2016 21:42,sub-S0001118411164,1.0,57.0,Male,White,Not Hispanic,28.5,,True,08/04/2023,2617.0,0.5427361130714417 +S0001,118411262,28/08/2008 22:49,sub-S0001118411262,1.0,78.0,Male,White,Not Hispanic,28.21,,True,12/02/2023,5280.0,0.5654754638671875 +S0001,118413807,29/10/2016 23:38,sub-S0001118413807,1.0,56.0,Male,White,Not Hispanic,24.02,,True,19/09/2024,2881.0,0.501916766166687 +S0001,118429956,15/10/2011 22:04,sub-S0001118429956,1.0,86.0,Male,Others,Not Hispanic,30.79,,True,26/05/2020,3145.0,0.7442228198051453 +S0001,118495502,19/10/2012 22:55,sub-S0001118495502,1.0,77.0,Female,White,Not Hispanic,36.87,,True,09/01/2023,3733.0,0.14538460969924927 +S0001,118513036,03/05/2008 22:37,sub-S0001118513036,1.0,71.0,Male,White,Unavailable,39.17,2462.0,True,22/04/2015,2544.0,0.40006473660469055 +S0001,118515170,13/05/2015 22:34,sub-S0001118515170,2.0,72.0,Male,White,Not Hispanic,23.55,1940.0,True,20/07/2021,2259.0,0.5197765827178955 +S0001,118574571,15/01/2015 22:17,sub-S0001118574571,1.0,63.0,Male,White,Hispanic,29.55,,True,16/02/2023,2953.0,0.4706968069076538 +S0001,118574603,27/05/2016 21:55,sub-S0001118574603,1.0,70.0,Male,White,Not Hispanic,29.05,,True,10/06/2023,2569.0,0.483925461769104 +S0001,118617888,03/11/2012 2:38,sub-S0001118617888,1.0,67.0,Female,Others,Hispanic,38.01,,True,19/04/2024,4184.0,0.6473630666732788 +S0001,118623840,28/03/2008 23:10,sub-S0001118623840,1.0,70.0,Female,Black,Not Hispanic,30.23,,True,27/04/2021,4777.0,0.40108874440193176 +S0001,118652295,07/08/2012 2:17,sub-S0001118652295,1.0,75.0,Female,Unavailable,Not Hispanic,34.06,1705.0,True,21/02/2021,3119.0,0.6455416083335876 +S0001,118930074,03/06/2015 21:53,sub-S0001118930074,1.0,71.0,Male,White,Not Hispanic,25.36,,True,26/11/2022,2732.0,0.40006473660469055 +S0001,118950203,16/04/2010 1:40,sub-S0001118950203,1.0,65.0,Male,White,Not Hispanic,41.19,,True,20/01/2023,4661.0,0.4400545656681061 +S0001,119000158,08/12/2012 1:41,sub-S0001119000158,1.0,74.0,Male,Black,Not Hispanic,40.67,,True,19/12/2020,2932.0,0.39002496004104614 +S0001,119040852,20/01/2009 22:11,sub-S0001119040852,1.0,74.0,Female,White,Not Hispanic,25.16,,True,01/05/2023,5213.0,0.33417800068855286 +S0001,119144776,06/06/2010 2:43,sub-S0001119144776,1.0,86.0,Male,White,Not Hispanic,21.04,,True,05/07/2019,3315.0,0.7442228198051453 +S0001,119200505,23/01/2009 3:56,sub-S0001119200505,1.0,51.0,Male,White,Not Hispanic,38.57,,True,31/01/2024,5485.0,0.4153312146663666 +S0001,119214666,01/09/2011 1:57,sub-S0001119214666,1.0,73.0,Male,White,Not Hispanic,25.02,2032.0,True,28/04/2021,3526.0,0.8519553542137146 +S0001,119239504,20/03/2011 2:25,sub-S0001119239504,1.0,66.0,Male,White,Not Hispanic,39.37,,True,05/09/2019,3090.0,0.45043623447418213 +S0001,119240200,02/08/2011 3:40,sub-S0001119240200,1.0,65.0,Male,White,Not Hispanic,33.83,2404.0,True,27/07/2023,4376.0,0.4400545656681061 +S0001,119336916,30/07/2013 1:41,sub-S0001119336916,1.0,71.0,Male,White,Not Hispanic,32.69,,True,18/07/2023,3639.0,0.40006473660469055 +S0001,119363044,04/03/2013 22:20,sub-S0001119363044,1.0,70.0,Male,White,Not Hispanic,32.85,,True,05/06/2021,3014.0,0.483925461769104 +S0001,119390537,28/07/2011 22:53,sub-S0001119390537,1.0,77.0,Female,White,Not Hispanic,26.17,2185.0,True,19/12/2022,4161.0,0.14538460969924927 +S0001,119415377,25/07/2015 22:19,sub-S0001119415377,1.0,70.0,Female,White,Not Hispanic,30.95,,True,10/08/2023,2937.0,0.40108874440193176 +S0001,119444122,17/10/2010 1:58,sub-S0001119444122,1.0,83.0,Male,White,Not Hispanic,22.98,,True,16/02/2020,3408.0,0.8366506099700928 +S0001,119501950,02/05/2009 1:36,sub-S0001119501950,1.0,77.0,Male,White,Not Hispanic,28.68,,True,13/10/2022,4911.0,0.8424844145774841 +S0001,119540284,05/02/2011 1:52,sub-S0001119540284,1.0,73.0,Male,White,Not Hispanic,28.82,,True,27/09/2022,4251.0,0.8519553542137146 +S0001,119592132,06/07/2017 22:58,sub-S0001119592132,1.0,82.0,Female,White,Hispanic,22.59,1752.0,True,08/01/2024,2376.0,0.5433937311172485 +S0001,119662091,05/05/2010 22:56,sub-S0001119662091,1.0,66.0,Female,White,Not Hispanic,41.12,,True,26/01/2022,4283.0,0.40600818395614624 +S0001,119693286,03/11/2012 22:36,sub-S0001119693286,1.0,70.0,Male,White,Not Hispanic,48.82,,True,05/05/2021,3104.0,0.483925461769104 +S0001,119775090,12/06/2013 22:39,sub-S0001119775090,1.0,53.0,Female,Unavailable,Not Hispanic,27.06,,True,01/07/2023,3670.0,0.3207913041114807 +S0001,119799817,06/05/2012 21:50,sub-S0001119799817,1.0,84.0,Male,White,Not Hispanic,24.1,,True,10/09/2023,4143.0,0.7442463636398315 +S0001,119825976,11/07/2013 3:04,sub-S0001119825976,1.0,77.0,Male,White,Not Hispanic,32.42,,True,25/10/2020,2662.0,0.8424844145774841 +S0001,119843121,03/08/2015 22:34,sub-S0001119843121,1.0,83.0,Male,White,Not Hispanic,27.21,1281.0,True,29/08/2021,2217.0,0.8366506099700928 +S0001,119862121,02/11/2011 21:35,sub-S0001119862121,1.0,64.0,Female,White,Not Hispanic,33.96,,True,12/02/2024,4484.0,0.8236727714538574 +S0001,119871328,15/01/2013 23:53,sub-S0001119871328,1.0,70.0,Male,White,Not Hispanic,33.2,,True,26/06/2024,4179.0,0.483925461769104 +S0001,119889263,30/11/2018 22:50,sub-S0001119889263,1.0,79.0,Male,White,Not Hispanic,35.13,1665.0,True,06/07/2024,2044.0,0.4199504256248474 +S0001,119924456,27/03/2010 21:12,sub-S0001119924456,1.0,79.0,Male,White,Unavailable,27.3,2153.0,True,06/11/2017,2780.0,0.4199504256248474 +S0001,119932412,08/07/2013 0:48,sub-S0001119932412,1.0,63.0,Male,White,Not Hispanic,36.85,,True,22/01/2024,3849.0,0.4706968069076538 +S0001,119965834,17/06/2010 2:19,sub-S0001119965834,1.0,60.0,Male,Unavailable,Not Hispanic,29.45,,True,27/07/2022,4422.0,0.24748243391513824 +S0001,119989726,03/05/2012 1:34,sub-S0001119989726,1.0,68.0,Male,White,Not Hispanic,28.51,1597.0,True,24/08/2023,4129.0,0.39139872789382935 +S0001,119998090,18/08/2019 20:27,sub-S0001119998090,1.0,75.0,Male,White,Not Hispanic,34.29,1402.0,True,12/08/2024,1820.0,0.27257463335990906 +S0001,120005093,19/10/2010 21:46,sub-S0001120005093,1.0,50.0,Male,White,Hispanic,30.31,,True,12/05/2024,4953.0,0.2891353964805603 +S0001,120033988,30/12/2009 21:56,sub-S0001120033988,1.0,82.0,Male,White,Not Hispanic,20.01,,True,20/11/2022,4707.0,0.44877195358276367 +S0001,120089914,03/05/2008 21:46,sub-S0001120089914,1.0,69.0,Male,White,Not Hispanic,32.67,,True,01/06/2021,4776.0,0.6186583042144775 +S0001,120158803,09/01/2011 1:59,sub-S0001120158803,1.0,72.0,Male,White,Not Hispanic,24.77,,True,07/11/2019,3223.0,0.5197765827178955 +S0001,120211143,14/02/2017 21:06,sub-S0001120211143,1.0,73.0,Female,White,Not Hispanic,24.17,,True,16/03/2024,2586.0,0.9163349866867065 +S0001,120276000,07/07/2015 21:46,sub-S0001120276000,1.0,66.0,Male,White,Not Hispanic,30.72,,True,28/02/2024,3157.0,0.45043623447418213 +S0001,120276384,18/05/2012 0:00,sub-S0001120276384,1.0,70.0,Male,White,Not Hispanic,30.18,1323.0,True,22/10/2023,4174.0,0.483925461769104 +S0001,120297473,04/11/2016 22:17,sub-S0001120297473,1.0,80.0,Female,White,Not Hispanic,29.32,,True,11/03/2024,2683.0,0.4938763380050659 +S0001,120336844,25/07/2012 1:57,sub-S0001120336844,1.0,74.0,Female,Others,Hispanic,20.67,,True,24/01/2023,3834.0,0.33417800068855286 +S0001,120377604,17/05/2011 2:00,sub-S0001120377604,1.0,67.0,Male,White,Not Hispanic,24.47,,True,31/01/2023,4276.0,0.9195594787597656 +S0001,120378159,28/07/2016 21:53,sub-S0001120378159,1.0,88.0,Male,White,Not Hispanic,32.29,,True,22/01/2024,2733.0,0.6507008671760559 +S0001,120385725,12/09/2008 23:04,sub-S0001120385725,1.0,70.0,Male,White,Not Hispanic,29.36,,True,14/10/2023,5509.0,0.483925461769104 +S0001,120409024,17/12/2011 1:36,sub-S0001120409024,1.0,75.0,Male,White,Not Hispanic,27.39,,True,19/10/2022,3958.0,0.27257463335990906 +S0001,120412309,11/04/2014 21:30,sub-S0001120412309,1.0,75.0,Male,White,Not Hispanic,27.46,,True,23/05/2023,3328.0,0.27257463335990906 +S0001,120527660,11/06/2010 3:10,sub-S0001120527660,1.0,77.0,Female,White,Not Hispanic,26.56,,True,17/09/2023,4845.0,0.14538460969924927 +S0001,120544140,13/12/2013 22:51,sub-S0001120544140,1.0,64.0,Female,Others,Hispanic,34.87,,True,23/07/2024,3874.0,0.8236727714538574 +S0001,120577416,02/05/2016 21:39,sub-S0001120577416,1.0,72.0,Female,White,Not Hispanic,31.51,1151.0,True,21/07/2023,2635.0,0.5070144534111023 +S0001,120596213,25/04/2016 22:23,sub-S0001120596213,1.0,54.0,Female,White,Not Hispanic,26.8,,True,19/08/2023,2671.0,0.6143807172775269 +S0001,120633409,09/02/2012 22:12,sub-S0001120633409,1.0,72.0,Female,White,Not Hispanic,37.74,,True,22/03/2022,3693.0,0.5070144534111023 +S0001,120673661,23/06/2012 2:51,sub-S0001120673661,2.0,55.0,Male,White,Not Hispanic,28.4,,True,21/12/2022,3832.0,0.19085277616977692 +S0001,120688704,02/11/2016 21:58,sub-S0001120688704,1.0,81.0,Female,White,Not Hispanic,34.85,,True,30/12/2023,2613.0,0.40572720766067505 +S0001,120707455,08/07/2017 21:51,sub-S0001120707455,1.0,60.0,Male,White,Not Hispanic,26.21,1222.0,True,06/10/2024,2646.0,0.24748243391513824 +S0001,120717749,21/06/2018 19:21,sub-S0001120717749,1.0,59.0,Male,White,Not Hispanic,25.44,1453.0,True,27/01/2023,1680.0,0.2705105245113373 +S0001,120747382,30/05/2010 2:51,sub-S0001120747382,1.0,74.0,Male,White,Not Hispanic,31.95,,True,06/03/2023,4662.0,0.39002496004104614 +S0001,120752810,28/12/2013 21:44,sub-S0001120752810,1.0,70.0,Male,Black,Not Hispanic,31.37,,True,18/02/2024,3703.0,0.483925461769104 +S0001,120764087,17/07/2010 2:53,sub-S0001120764087,1.0,72.0,Female,White,Not Hispanic,25.84,1113.0,True,19/08/2018,2954.0,0.5070144534111023 +S0001,120783618,22/07/2012 22:06,sub-S0001120783618,1.0,80.0,Female,Black,Not Hispanic,35.09,,True,05/05/2020,2843.0,0.4938763380050659 +S0001,120821892,04/10/2015 21:47,sub-S0001120821892,1.0,62.0,Male,Black,Hispanic,35.6,,True,13/12/2022,2626.0,0.7553417086601257 +S0001,120827718,15/06/2013 21:46,sub-S0001120827718,1.0,58.0,Male,White,Not Hispanic,28.74,,True,25/05/2023,3630.0,0.23777179419994354 +S0001,120830792,07/08/2011 3:20,sub-S0001120830792,1.0,79.0,Male,White,Not Hispanic,26.98,,True,26/03/2024,4614.0,0.4199504256248474 +S0001,120869188,08/08/2009 3:19,sub-S0001120869188,1.0,71.0,Male,White,Not Hispanic,28.06,2074.0,True,08/10/2022,4808.0,0.40006473660469055 +S0001,121017443,03/05/2013 23:30,sub-S0001121017443,1.0,64.0,Female,White,Not Hispanic,28.07,,True,23/06/2023,3702.0,0.8236727714538574 +S0001,121017473,19/01/2015 21:38,sub-S0001121017473,1.0,69.0,Female,White,Not Hispanic,34.77,,True,18/09/2022,2798.0,0.6564435362815857 +S0001,121038180,02/08/2014 1:48,sub-S0001121038180,1.0,62.0,Female,Unavailable,Hispanic,36.6,,True,03/04/2023,3165.0,0.7453621029853821 +S0001,121063935,12/05/2010 3:05,sub-S0001121063935,1.0,74.0,Female,White,Not Hispanic,32.44,,True,27/10/2023,4915.0,0.33417800068855286 +S0001,121092939,03/03/2010 22:03,sub-S0001121092939,1.0,71.0,Female,White,Not Hispanic,33.16,,True,23/06/2024,5225.0,0.5782973766326904 +S0001,121121018,01/01/2016 23:04,sub-S0001121121018,1.0,66.0,Male,White,Not Hispanic,24.78,,True,09/07/2024,3111.0,0.45043623447418213 +S0001,121255336,08/09/2015 23:09,sub-S0001121255336,2.0,76.0,Female,White,Not Hispanic,34.34,,True,06/10/2022,2584.0,0.5227447152137756 +S0001,121264507,22/02/2011 1:53,sub-S0001121264507,1.0,61.0,Male,White,Not Hispanic,27.58,,True,09/05/2023,4458.0,0.6737512946128845 +S0001,121295515,17/08/2015 23:17,sub-S0001121295515,1.0,63.0,Female,White,Not Hispanic,24.72,,True,28/01/2023,2720.0,0.7037391066551208 +S0001,121337883,25/12/2012 2:39,sub-S0001121337883,1.0,63.0,Female,White,Not Hispanic,37.31,,True,09/07/2024,4213.0,0.7037391066551208 +S0001,121370986,22/08/2011 2:21,sub-S0001121370986,1.0,69.0,Female,White,Not Hispanic,29.64,,True,30/07/2021,3629.0,0.6564435362815857 +S0001,121449477,17/07/2009 22:08,sub-S0001121449477,1.0,65.0,Male,White,Not Hispanic,28.18,,True,20/02/2023,4965.0,0.4400545656681061 +S0001,121488998,04/08/2012 1:56,sub-S0001121488998,1.0,73.0,Male,White,Not Hispanic,34.9,,True,17/01/2024,4182.0,0.8519553542137146 +S0001,121529475,08/11/2010 2:17,sub-S0001121529475,1.0,67.0,Male,White,Not Hispanic,30.7,,True,02/04/2023,4527.0,0.9195594787597656 +S0001,121573952,15/03/2009 22:57,sub-S0001121573952,1.0,72.0,Male,White,Not Hispanic,45.57,,True,07/01/2018,3219.0,0.5197765827178955 +S0001,121574219,18/10/2010 3:07,sub-S0001121574219,1.0,65.0,Female,White,Hispanic,28.78,,True,03/02/2023,4490.0,0.7371628880500793 +S0001,121594285,11/09/2008 22:20,sub-S0001121594285,1.0,79.0,Female,White,Not Hispanic,28.01,,True,27/01/2020,4154.0,0.7526390552520752 +S0001,121693594,02/08/2009 2:26,sub-S0001121693594,1.0,85.0,Male,White,Not Hispanic,24.7,,True,03/06/2019,3591.0,0.6813521385192871 +S0001,121695905,02/08/2010 3:15,sub-S0001121695905,1.0,75.0,Male,White,Not Hispanic,30.41,2063.0,True,14/07/2016,2172.0,0.27257463335990906 +S0001,121716656,16/10/2015 22:19,sub-S0001121716656,1.0,60.0,Male,White,Not Hispanic,29.17,,True,04/10/2023,2909.0,0.24748243391513824 +S0001,121887520,05/10/2010 2:03,sub-S0001121887520,1.0,81.0,Male,White,Not Hispanic,26.6,,True,25/10/2017,2576.0,0.8153370022773743 +S0001,121903400,26/03/2015 22:04,sub-S0001121903400,1.0,50.0,Female,White,Not Hispanic,21.84,,True,01/09/2023,3080.0,0.5013509392738342 +S0001,121953316,21/08/2014 21:46,sub-S0001121953316,1.0,73.0,Male,White,Not Hispanic,29.21,,True,02/08/2023,3267.0,0.8519553542137146 +S0001,122023932,15/02/2011 0:00,sub-S0001122023932,1.0,79.0,Male,White,Not Hispanic,26.2,1762.0,True,18/12/2022,4324.0,0.4199504256248474 +S0001,122024052,22/07/2012 3:19,sub-S0001122024052,1.0,62.0,Male,White,Not Hispanic,28.21,,True,24/03/2022,3531.0,0.7553417086601257 +S0001,122053605,22/10/2012 2:57,sub-S0001122053605,1.0,62.0,Female,White,Not Hispanic,27.31,,True,06/09/2024,4336.0,0.7453621029853821 +S0001,122094999,23/02/2011 1:41,sub-S0001122094999,1.0,67.0,Male,White,Not Hispanic,36.13,,True,28/06/2023,4507.0,0.9195594787597656 +S0001,122115803,16/07/2013 23:07,sub-S0001122115803,1.0,60.0,Male,White,Not Hispanic,34.03,,True,08/08/2023,3674.0,0.24748243391513824 +S0001,122160941,09/10/2015 22:55,sub-S0001122160941,1.0,76.0,Female,White,Not Hispanic,32.75,,True,21/09/2023,2903.0,0.5227447152137756 +S0001,122221329,11/10/2015 22:40,sub-S0001122221329,2.0,68.0,Male,White,Not Hispanic,31.48,,True,19/01/2024,3021.0,0.39139872789382935 +S0001,122226335,22/08/2012 0:00,sub-S0001122226335,1.0,73.0,Female,White,Not Hispanic,38.24,1143.0,True,11/07/2020,2880.0,0.9163349866867065 +S0001,122360618,06/10/2008 21:30,sub-S0001122360618,1.0,70.0,Male,White,Not Hispanic,30.17,1121.0,True,04/01/2023,5202.0,0.483925461769104 +I0002,150000686,04/01/2019 22:29,sub-I0002150000686,1.0,82.0,Male,Black,Not Hispanic,34.25,1104.0,True,24/11/2023,1784.0,0.44877195358276367 +I0002,150002879,18/12/2016 20:58,sub-I0002150002879,1.0,75.0,Male,White,Not Hispanic,19.62,1313.0,True,21/12/2020,1463.0,0.27257463335990906 +I0002,150003895,10/08/2017 21:46,sub-I0002150003895,1.0,76.0,Male,White,Not Hispanic,,,True,08/04/2024,2432.0,0.2753254175186157 +I0002,150004526,14/12/2016 22:36,sub-I0002150004526,1.0,76.0,Male,White,Not Hispanic,22.19,1487.0,True,03/07/2021,1661.0,0.2753254175186157 +I0002,150004858,27/08/2016 21:53,sub-I0002150004858,2.0,78.0,Male,White,Not Hispanic,,,True,15/06/2023,2482.0,0.5654754638671875 +I0002,150005910,03/06/2018 22:06,sub-I0002150005910,1.0,55.0,Male,White,Not Hispanic,,1335.0,True,02/02/2025,2435.0,0.19085277616977692 +I0002,150006084,30/05/2017 21:42,sub-I0002150006084,1.0,70.0,Female,White,Not Hispanic,32.53,,True,03/06/2023,2194.0,0.40108874440193176 +I0002,150022286,03/11/2016 21:18,sub-I0002150022286,1.0,69.0,Male,White,Not Hispanic,,,True,20/10/2023,2541.0,0.6186583042144775 +I0002,150023281,14/04/2018 21:02,sub-I0002150023281,1.0,66.0,Female,White,Not Hispanic,34.24,1234.0,True,13/06/2023,1885.0,0.40600818395614624 +I0002,150024576,22/07/2017 22:11,sub-I0002150024576,2.0,74.0,Male,White,Not Hispanic,,1303.0,True,17/06/2024,2521.0,0.39002496004104614 +I0002,150027698,01/03/2017 22:43,sub-I0002150027698,1.0,70.0,Male,White,Not Hispanic,,,True,09/08/2023,2351.0,0.483925461769104 +I0002,150028152,12/09/2016 22:19,sub-I0002150028152,1.0,62.0,Male,Others,Not Hispanic,,,True,13/09/2023,2556.0,0.7553417086601257 +I0002,150028177,26/08/2016 22:33,sub-I0002150028177,1.0,64.0,Male,White,Not Hispanic,26.31,,True,09/03/2023,2385.0,0.8189074397087097 +I0002,150028179,27/12/2017 21:15,sub-I0002150028179,2.0,81.0,Female,Black,Not Hispanic,,,True,10/12/2024,2539.0,0.40572720766067505 +I0002,150028223,05/01/2018 21:39,sub-I0002150028223,1.0,71.0,Female,White,Not Hispanic,,,True,05/01/2025,2556.0,0.5782973766326904 +I0002,150028247,09/09/2017 23:38,sub-I0002150028247,1.0,74.0,Female,White,Not Hispanic,,,True,26/08/2024,2542.0,0.33417800068855286 +I0002,150028374,17/08/2016 22:47,sub-I0002150028374,1.0,66.0,Female,Asian,Not Hispanic,,,True,24/04/2023,2440.0,0.40600818395614624 +I0002,150028391,06/02/2017 22:43,sub-I0002150028391,1.0,55.0,Male,White,Not Hispanic,,,True,13/09/2023,2409.0,0.19085277616977692 +I0002,150028394,22/10/2016 21:53,sub-I0002150028394,5.0,66.0,Male,Unavailable,Unavailable,,,True,09/07/2023,2450.0,0.45043623447418213 +I0002,150028694,23/10/2016 21:32,sub-I0002150028694,1.0,73.0,Female,Black,Not Hispanic,28.67,1502.0,True,01/06/2023,2411.0,0.9163349866867065 +I0002,150028714,11/06/2017 22:07,sub-I0002150028714,1.0,74.0,Female,White,Not Hispanic,15.1,,True,10/12/2023,2372.0,0.33417800068855286 +I0002,150028859,25/07/2018 22:20,sub-I0002150028859,1.0,52.0,Male,Unavailable,Not Hispanic,,,True,13/01/2025,2363.0,0.6616770625114441 +I0002,150028912,27/12/2017 22:59,sub-I0002150028912,2.0,73.0,Male,White,Not Hispanic,40.7,1865.0,True,11/02/2024,2236.0,0.8519553542137146 +I0002,150028988,26/04/2017 23:09,sub-I0002150028988,1.0,76.0,Female,Unavailable,Unavailable,,,True,03/11/2023,2381.0,0.5227447152137756 +I0002,150029061,15/05/2018 22:13,sub-I0002150029061,1.0,80.0,Male,White,Not Hispanic,30.17,,True,08/10/2024,2337.0,0.34214648604393005 +I0002,150029213,08/06/2018 22:29,sub-I0002150029213,1.0,64.0,Female,Black,Not Hispanic,,,True,30/09/2024,2305.0,0.8236727714538574 +I0002,150029266,06/09/2017 21:18,sub-I0002150029266,1.0,86.0,Male,White,Not Hispanic,26.63,,True,15/01/2024,2321.0,0.7442228198051453 +I0002,150029589,17/12/2018 22:39,sub-I0002150029589,1.0,75.0,Male,Asian,Not Hispanic,,,True,11/02/2025,2247.0,0.27257463335990906 +I0002,150029618,20/07/2018 22:28,sub-I0002150029618,1.0,77.0,Male,White,Not Hispanic,,,True,22/09/2024,2255.0,0.8424844145774841 +I0002,150031893,01/05/2019 22:24,sub-I0002150031893,1.0,74.0,Female,Black,Not Hispanic,,1113.0,True,19/01/2024,1723.0,0.33417800068855286 +I0002,150033481,09/08/2020 22:47,sub-I0002150033481,1.0,76.0,Female,White,Not Hispanic,,1164.0,True,01/11/2023,1178.0,0.5227447152137756 +I0002,150058421,08/11/2016 22:27,sub-I0002150058421,1.0,71.0,Female,White,Not Hispanic,,,True,22/05/2023,2385.0,0.5782973766326904 +I0002,150656547,06/04/2017 21:58,sub-I0002150656547,1.0,73.0,Male,White,Not Hispanic,,,True,10/02/2024,2500.0,0.8519553542137146 +I0006,179000007,14/11/2014,sub-I0006179000007,1.0,70.0,Female,White,Not Hispanic,,,True,08/01/2022,2612.0,0.40108874440193176 +I0006,179000101,01/09/2014,sub-I0006179000101,1.0,78.0,Female,Black,Not Hispanic,,,True,02/04/2022,2770.0,0.6120396256446838 +I0006,179000125,25/03/2015,sub-I0006179000125,1.0,74.0,Female,White,Not Hispanic,41.13,,True,26/12/2022,2833.0,0.33417800068855286 +I0006,179000270,17/01/2015,sub-I0006179000270,1.0,77.0,Male,White,Not Hispanic,24.74,,True,14/05/2022,2674.0,0.8424844145774841 +I0006,179000424,15/03/2013,sub-I0006179000424,1.0,68.0,Male,White,Not Hispanic,33.78,,True,07/05/2022,3340.0,0.39139872789382935 +I0006,179000812,06/11/2012,sub-I0006179000812,1.0,66.0,Male,White,Not Hispanic,28.19,,True,28/11/2022,3674.0,0.45043623447418213 +I0006,179000833,05/06/2012,sub-I0006179000833,1.0,51.0,Male,Black,Not Hispanic,31.2,,True,03/12/2021,3468.0,0.4153312146663666 +I0006,179001123,03/04/2014,sub-I0006179001123,1.0,68.0,Male,Black,Not Hispanic,21.53,,True,20/11/2021,2788.0,0.39139872789382935 +I0006,179001138,06/03/2016,sub-I0006179001138,1.0,72.0,Male,White,Unavailable,33.43,1798.0,True,06/12/2021,2101.0,0.5197765827178955 +I0006,179001182,29/08/2013,sub-I0006179001182,1.0,51.0,Female,Black,Not Hispanic,40.18,,True,13/07/2023,3605.0,0.509002149105072 +I0006,179001184,08/06/2012,sub-I0006179001184,1.0,68.0,Male,White,Not Hispanic,,1546.0,True,20/04/2022,3603.0,0.39139872789382935 +I0006,179001218,19/10/2018,sub-I0006179001218,1.0,73.0,Female,White,Not Hispanic,,1470.0,True,17/06/2023,1702.0,0.9163349866867065 +I0006,179001240,25/12/2016,sub-I0006179001240,1.0,78.0,Female,White,Not Hispanic,19.21,1220.0,True,16/01/2022,1848.0,0.6120396256446838 +I0006,179001308,19/02/2014,sub-I0006179001308,1.0,67.0,Female,Asian,Not Hispanic,32.98,,True,15/02/2022,2918.0,0.6473630666732788 +I0006,179001370,22/04/2012,sub-I0006179001370,1.0,68.0,Male,White,Not Hispanic,,1126.0,True,14/06/2016,1514.0,0.39139872789382935 +I0006,179001582,10/03/2015,sub-I0006179001582,1.0,74.0,Female,White,Not Hispanic,29.77,,True,20/04/2023,2963.0,0.33417800068855286 +I0006,179001769,10/02/2013,sub-I0006179001769,1.0,65.0,Male,White,Hispanic,29.65,2031.0,True,10/10/2021,3164.0,0.4400545656681061 +I0006,179001910,05/09/2017,sub-I0006179001910,1.0,60.0,Female,White,Hispanic,29.43,1849.0,True,25/01/2023,1968.0,0.5012294054031372 +I0006,179001928,05/01/2012,sub-I0006179001928,1.0,70.0,Male,White,Not Hispanic,,,True,15/01/2022,3663.0,0.483925461769104 +I0006,179002049,18/05/2016,sub-I0006179002049,1.0,77.0,Male,White,Not Hispanic,31.73,,True,28/05/2023,2566.0,0.8424844145774841 +I0006,179002155,22/07/2013,sub-I0006179002155,1.0,71.0,Male,White,Not Hispanic,30.92,,True,27/10/2021,3019.0,0.40006473660469055 +I0006,179002261,29/10/2014,sub-I0006179002261,1.0,64.0,Male,White,Not Hispanic,40.2,,True,02/08/2022,2834.0,0.8189074397087097 +I0006,179002374,09/08/2014,sub-I0006179002374,1.0,86.0,Male,White,Not Hispanic,21.67,1104.0,True,03/09/2019,1851.0,0.7442228198051453 +I0006,179002452,22/06/2013,sub-I0006179002452,1.0,71.0,Female,White,Not Hispanic,27.14,,True,23/05/2022,3257.0,0.5782973766326904 +I0006,179002531,26/03/2018,sub-I0006179002531,1.0,73.0,Male,White,Not Hispanic,,1212.0,True,30/04/2022,1496.0,0.8519553542137146 +I0006,179002653,25/11/2013,sub-I0006179002653,1.0,66.0,Male,White,Not Hispanic,22.71,,True,12/03/2022,3029.0,0.45043623447418213 +I0006,179003102,26/09/2013,sub-I0006179003102,1.0,60.0,Female,Black,Not Hispanic,26.24,,True,11/01/2022,3029.0,0.5012294054031372 +I0006,179003203,05/03/2015,sub-I0006179003203,1.0,78.0,Male,Black,Not Hispanic,25.39,1309.0,True,23/03/2020,1845.0,0.5654754638671875 +I0006,179003237,18/10/2013,sub-I0006179003237,1.0,61.0,Female,Black,Not Hispanic,,,True,26/10/2022,3295.0,0.824192225933075 +I0006,179003508,17/07/2015,sub-I0006179003508,1.0,73.0,Female,White,Not Hispanic,24.38,2461.0,True,10/06/2023,2885.0,0.9163349866867065 +I0006,179003575,05/03/2015,sub-I0006179003575,1.0,72.0,Female,Black,Not Hispanic,41.0,,True,11/06/2023,3020.0,0.5070144534111023 +I0006,179004102,19/02/2017,sub-I0006179004102,1.0,66.0,Female,White,Not Hispanic,24.57,1468.0,True,31/10/2022,2080.0,0.40600818395614624 +I0006,179004330,25/12/2014,sub-I0006179004330,1.0,69.0,Female,Black,Not Hispanic,43.81,1513.0,True,25/03/2022,2647.0,0.6564435362815857 +I0006,179004461,04/01/2015,sub-I0006179004461,3.0,69.0,Female,White,Not Hispanic,26.07,,True,07/03/2022,2619.0,0.6564435362815857 +I0006,179004462,27/01/2016,sub-I0006179004462,1.0,68.0,Male,White,Not Hispanic,27.48,,True,15/02/2023,2576.0,0.39139872789382935 +I0006,179004563,09/07/2014,sub-I0006179004563,1.0,64.0,Male,White,Not Hispanic,,,True,01/01/2022,2733.0,0.8189074397087097 +I0006,179004611,29/04/2014,sub-I0006179004611,2.0,64.0,Male,White,Not Hispanic,54.38,,True,23/08/2022,3038.0,0.8189074397087097 +I0006,179004802,04/05/2014,sub-I0006179004802,1.0,71.0,Male,White,Not Hispanic,24.84,,True,23/11/2021,2760.0,0.40006473660469055 +I0006,179004989,30/09/2013,sub-I0006179004989,1.0,71.0,Female,Black,Not Hispanic,,,True,04/04/2021,2743.0,0.5782973766326904 +I0006,179005139,26/10/2014,sub-I0006179005139,1.0,70.0,Male,Asian,Not Hispanic,,,True,24/02/2022,2678.0,0.483925461769104 +I0006,179005300,24/11/2013,sub-I0006179005300,1.0,85.0,Male,White,Not Hispanic,23.37,,True,14/06/2022,3124.0,0.6813521385192871 +I0006,179005468,30/11/2015,sub-I0006179005468,1.0,79.0,Male,White,Not Hispanic,43.16,1302.0,True,01/11/2021,2163.0,0.4199504256248474 +I0006,179005535,14/01/2014,sub-I0006179005535,1.0,66.0,Female,Black,Not Hispanic,24.65,,True,22/08/2022,3142.0,0.40600818395614624 +I0006,179005646,15/06/2012,sub-I0006179005646,1.0,82.0,Female,White,Not Hispanic,29.72,2553.0,True,24/07/2021,3326.0,0.5433937311172485 +I0006,179005751,16/06/2012,sub-I0006179005751,1.0,68.0,Female,White,Not Hispanic,26.96,,True,30/07/2021,3331.0,0.346417635679245 +I0006,179005925,14/08/2014,sub-I0006179005925,1.0,72.0,Male,White,Not Hispanic,28.16,,True,19/06/2022,2866.0,0.5197765827178955 +I0006,179006062,22/12/2015,sub-I0006179006062,1.0,69.0,Male,White,Not Hispanic,29.5,,True,15/02/2023,2612.0,0.6186583042144775 +I0006,179006142,31/12/2014,sub-I0006179006142,1.0,73.0,Male,White,Not Hispanic,27.19,2220.0,True,04/07/2022,2742.0,0.8519553542137146 +I0006,179006234,18/11/2013,sub-I0006179006234,1.0,50.0,Female,White,Not Hispanic,35.17,,True,29/07/2021,2810.0,0.5013509392738342 +I0006,179006362,21/08/2015,sub-I0006179006362,1.0,68.0,Male,White,Unavailable,,,True,02/12/2022,2660.0,0.39139872789382935 +I0006,179006516,06/08/2015,sub-I0006179006516,1.0,67.0,Male,White,Not Hispanic,31.8,2451.0,True,05/07/2022,2525.0,0.9195594787597656 +I0006,179006608,07/02/2017,sub-I0006179006608,1.0,74.0,Male,White,Not Hispanic,30.88,1565.0,True,27/10/2021,1723.0,0.39002496004104614 +I0006,179006626,24/04/2013,sub-I0006179006626,1.0,71.0,Female,White,Not Hispanic,31.41,,True,14/02/2022,3218.0,0.5782973766326904 +I0006,179006729,15/04/2013,sub-I0006179006729,1.0,75.0,Female,White,Not Hispanic,26.66,,True,07/02/2022,3220.0,0.6455416083335876 +I0006,179006730,15/09/2012,sub-I0006179006730,1.0,65.0,Female,White,Not Hispanic,27.18,,True,01/06/2022,3546.0,0.7371628880500793 +I0006,179006973,21/05/2013,sub-I0006179006973,1.0,68.0,Male,Black,Not Hispanic,28.46,,True,24/01/2022,3170.0,0.39139872789382935 +I0006,179007120,29/06/2012,sub-I0006179007120,1.0,70.0,Male,White,Not Hispanic,27.02,,True,19/10/2022,3764.0,0.483925461769104 +I0006,179007327,24/06/2013,sub-I0006179007327,1.0,75.0,Female,Black,Not Hispanic,34.41,,True,01/11/2022,3417.0,0.6455416083335876 +I0006,179007370,04/09/2015,sub-I0006179007370,1.0,62.0,Female,Black,Not Hispanic,31.17,,True,06/09/2022,2559.0,0.7453621029853821 +I0006,179007802,10/02/2017,sub-I0006179007802,1.0,69.0,Male,White,Not Hispanic,,1344.0,True,12/11/2021,1736.0,0.6186583042144775 +I0006,179007995,12/03/2012,sub-I0006179007995,1.0,66.0,Male,White,Not Hispanic,35.86,,True,14/01/2021,3230.0,0.45043623447418213 +I0006,179008230,22/09/2013,sub-I0006179008230,1.0,66.0,Female,Black,Not Hispanic,30.32,1130.0,True,27/04/2023,3504.0,0.40600818395614624 +I0006,179008398,31/07/2015,sub-I0006179008398,1.0,73.0,Female,White,Not Hispanic,35.08,,True,01/06/2023,2862.0,0.9163349866867065 +I0006,179008802,11/01/2018,sub-I0006179008802,1.0,77.0,Male,White,Not Hispanic,,1256.0,True,23/06/2022,1624.0,0.8424844145774841 +I0006,179008931,25/12/2015,sub-I0006179008931,1.0,73.0,Female,White,Not Hispanic,34.72,,True,10/03/2023,2632.0,0.9163349866867065 +I0006,179009256,08/08/2015,sub-I0006179009256,1.0,76.0,Male,White,Not Hispanic,26.1,,True,23/05/2023,2845.0,0.2753254175186157 +I0006,179009387,14/11/2014,sub-I0006179009387,1.0,84.0,Female,Black,Not Hispanic,21.9,,True,07/01/2022,2611.0,0.48952510952949524 +I0006,179009535,12/11/2014,sub-I0006179009535,1.0,66.0,Female,Black,Not Hispanic,,,True,24/04/2022,2720.0,0.40600818395614624 +I0006,179009648,24/01/2015,sub-I0006179009648,1.0,54.0,Male,White,Unavailable,,,True,27/02/2022,2591.0,0.47246167063713074 +I0006,179009818,05/06/2013,sub-I0006179009818,1.0,79.0,Female,Black,Not Hispanic,,1149.0,True,01/04/2023,3587.0,0.7526390552520752 +I0006,179009848,22/12/2014,sub-I0006179009848,1.0,51.0,Female,White,Hispanic,22.51,,True,21/02/2022,2618.0,0.509002149105072 +I0006,179010140,30/04/2013,sub-I0006179010140,1.0,73.0,Female,Black,Not Hispanic,27.14,,True,10/02/2022,3208.0,0.9163349866867065 +I0006,179010216,14/11/2015,sub-I0006179010216,1.0,62.0,Female,Black,Not Hispanic,41.21,,True,25/08/2023,2841.0,0.7453621029853821 +I0006,179010224,21/10/2012,sub-I0006179010224,1.0,71.0,Male,White,Not Hispanic,29.27,,True,13/07/2022,3552.0,0.40006473660469055 +I0006,179010345,10/10/2015,sub-I0006179010345,1.0,59.0,Male,Black,Not Hispanic,41.62,,True,27/04/2023,2756.0,0.2705105245113373 +I0006,179010644,20/04/2014,sub-I0006179010644,1.0,69.0,Female,White,Not Hispanic,27.44,,True,11/01/2023,3188.0,0.6564435362815857 +I0006,179010757,16/04/2014,sub-I0006179010757,1.0,69.0,Female,Black,Not Hispanic,40.38,,True,17/12/2022,3167.0,0.6564435362815857 +I0006,179010780,20/06/2014,sub-I0006179010780,1.0,57.0,Female,Black,Not Hispanic,44.08,,True,10/11/2023,3430.0,0.6525198221206665 +I0006,179011124,30/07/2011,sub-I0006179011124,1.0,79.0,Male,Black,Not Hispanic,29.84,,True,23/04/2021,3555.0,0.4199504256248474 +I0006,179011383,20/07/2015,sub-I0006179011383,1.0,61.0,Female,Black,Not Hispanic,30.85,,True,25/09/2022,2624.0,0.824192225933075 +I0006,179011615,04/11/2016,sub-I0006179011615,1.0,62.0,Male,White,Not Hispanic,32.43,1282.0,True,10/12/2021,1862.0,0.7553417086601257 +I0006,179011701,01/11/2015,sub-I0006179011701,1.0,76.0,Male,Black,Not Hispanic,65.65,,True,05/06/2023,2773.0,0.2753254175186157 +I0006,179012080,11/02/2012,sub-I0006179012080,1.0,62.0,Male,White,Not Hispanic,41.97,,True,29/06/2022,3791.0,0.7553417086601257 +I0006,179012153,12/10/2015,sub-I0006179012153,1.0,69.0,Male,White,Not Hispanic,32.44,,True,19/10/2022,2564.0,0.6186583042144775 +I0006,179012255,27/01/2013,sub-I0006179012255,1.0,73.0,Male,White,Not Hispanic,41.54,,True,07/09/2020,2780.0,0.8519553542137146 +I0006,179012526,01/12/2015,sub-I0006179012526,1.0,67.0,Male,White,Not Hispanic,,,True,26/04/2023,2703.0,0.9195594787597656 +I0006,179013071,07/07/2017,sub-I0006179013071,1.0,79.0,Male,White,Not Hispanic,25.45,1114.0,True,26/08/2021,1511.0,0.4199504256248474 +I0006,179013154,18/10/2015,sub-I0006179013154,1.0,69.0,Female,White,Not Hispanic,22.75,,True,21/11/2022,2591.0,0.6564435362815857 +I0006,179014393,13/07/2015,sub-I0006179014393,1.0,50.0,Male,White,Not Hispanic,35.92,,True,17/05/2023,2865.0,0.2891353964805603 +I0006,179014446,24/06/2016,sub-I0006179014446,1.0,76.0,Female,White,Not Hispanic,24.42,1710.0,True,19/03/2023,2459.0,0.5227447152137756 +I0006,179014825,16/06/2015,sub-I0006179014825,1.0,75.0,Male,Unavailable,Unavailable,,1537.0,True,14/06/2020,1825.0,0.27257463335990906 +I0006,179015621,03/05/2014,sub-I0006179015621,1.0,77.0,Male,Black,Not Hispanic,36.76,,True,19/07/2023,3364.0,0.8424844145774841 +I0006,179015933,17/10/2016,sub-I0006179015933,1.0,70.0,Female,White,Not Hispanic,22.28,1542.0,True,09/07/2021,1726.0,0.40108874440193176 +I0006,179016086,23/03/2017,sub-I0006179016086,1.0,72.0,Male,White,Not Hispanic,33.8,1192.0,True,27/01/2023,2136.0,0.5197765827178955 +I0006,179016109,26/03/2018,sub-I0006179016109,1.0,51.0,Female,Black,Not Hispanic,26.82,1280.0,True,03/01/2022,1379.0,0.509002149105072 diff --git a/requirements.txt b/requirements.txt index 28b6875..00c7ede 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,4 +4,5 @@ numpy==2.0.2 pandas==2.2.2 scikit-learn==1.6.0 scipy==1.13.1 -tqdm==4.67.1 \ No newline at end of file +tqdm==4.67.1 +xgboost==3.2.0 \ No newline at end of file diff --git a/run.ps1 b/run.ps1 new file mode 100644 index 0000000..e33cae4 --- /dev/null +++ b/run.ps1 @@ -0,0 +1,275 @@ +param( + [Parameter(Mandatory=$true)] + [ValidateSet( + "build", + "smoke", + "train", + "train-smoke", + "run", + "run-smoke", + "eval", + "eval-smoke", + "train-dev", + "run-dev", + "eval-dev", + "clean" + )] + [string]$Command +) + +# ============================================ +# CONFIGURACIÓN +# ============================================ + +# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> +# IMPORTANTE: +# Si tu dataset no está en data/training_set o data/test_set, +# modifica estas rutas. +# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> +$TRAIN_DATA_REL = "data/training_set" +$RUN_DATA_REL = "data/test_set" +$SMOKE_DATA_REL = "data/training_smoke" + +$IMAGE_NAME = "cinc2026" + +$MODEL_FULL_REL = "model" +$MODEL_SMOKE_REL = "model_smoke" +$FEATURE_CACHE_REL = ".feature_cache" + +$OUT_FULL_REL = "outputs" +$OUT_SMOKE_REL = "outputs_smoke" +$DEMOGRAPHICS_FILE = "demographics.csv" + +# ============================================ +# FUNCIONES AUXILIARES +# ============================================ + +function Get-AbsolutePath($relativePath) { + return (Resolve-Path $relativePath).Path +} + +function Ensure-Directory($path) { + if (!(Test-Path $path)) { + New-Item -ItemType Directory -Force -Path $path | Out-Null + } +} + +function Invoke-Evaluation($DataPath, $OutputPath, $Label) { + Write-Host "Evaluating $Label predictions..." + docker run --rm ` + -v "${DataPath}:/challenge/eval_data:ro" ` + -v "${OutputPath}:/challenge/eval_outputs:ro" ` + $IMAGE_NAME ` + python evaluate_model.py -d "/challenge/eval_data/$DEMOGRAPHICS_FILE" -o "/challenge/eval_outputs/$DEMOGRAPHICS_FILE" +} + +function Invoke-EvaluationDev($CodePath, $DataPath, $OutputPath, $Label) { + Write-Host "Evaluating $Label predictions..." + docker run --rm ` + -v "${CodePath}:/challenge" ` + -v "${DataPath}:/challenge/eval_data:ro" ` + $IMAGE_NAME ` + python evaluate_model.py -d "/challenge/eval_data/$DEMOGRAPHICS_FILE" -o "$OutputPath/$DEMOGRAPHICS_FILE" +} + +function Test-DatasetHasLabels($DataPath) { + $demographicsPath = Join-Path $DataPath $DEMOGRAPHICS_FILE + if (!(Test-Path $demographicsPath)) { + return $false + } + + $header = Get-Content -Path $demographicsPath -TotalCount 1 + return $header -match "Cognitive_Impairment" +} + +# ============================================ +# COMANDOS +# ============================================ + +function Build-Image { + docker build -t $IMAGE_NAME . +} + +function Create-Smoke { + Write-Host "Creando dataset smoke..." + powershell -ExecutionPolicy Bypass -File scripts/create_smoke.ps1 +} + +function Train-Full { + + $FULL_DATA = Get-AbsolutePath $TRAIN_DATA_REL + $MODEL_FULL = Join-Path (Get-AbsolutePath ".") $MODEL_FULL_REL + $FEATURE_CACHE = Join-Path (Get-AbsolutePath ".") $FEATURE_CACHE_REL + + Ensure-Directory $MODEL_FULL + Ensure-Directory $FEATURE_CACHE + + docker run --rm ` + -v "${FULL_DATA}:/challenge/training_data:ro" ` + -v "${MODEL_FULL}:/challenge/model" ` + -v "${FEATURE_CACHE}:/challenge/.feature_cache" ` + $IMAGE_NAME ` + python train_model.py -d training_data -m model -v +} + +function Train-Smoke { + + $SMOKE_DATA = Get-AbsolutePath $SMOKE_DATA_REL + $MODEL_SMOKE = Join-Path (Get-AbsolutePath ".") $MODEL_SMOKE_REL + $FEATURE_CACHE = Join-Path (Get-AbsolutePath ".") $FEATURE_CACHE_REL + + Ensure-Directory $MODEL_SMOKE + Ensure-Directory $FEATURE_CACHE + + docker run --rm ` + -v "${SMOKE_DATA}:/challenge/training_data:ro" ` + -v "${MODEL_SMOKE}:/challenge/model" ` + -v "${FEATURE_CACHE}:/challenge/.feature_cache" ` + $IMAGE_NAME ` + python train_model.py -d training_data -m model -v +} + +function Run-Full { + + $RUN_DATA = Get-AbsolutePath $RUN_DATA_REL + $MODEL_FULL = Get-AbsolutePath $MODEL_FULL_REL + $OUT_FULL = Join-Path (Get-AbsolutePath ".") $OUT_FULL_REL + $FEATURE_CACHE = Join-Path (Get-AbsolutePath ".") $FEATURE_CACHE_REL + + Ensure-Directory $OUT_FULL + Ensure-Directory $FEATURE_CACHE + + docker run --rm ` + -v "${RUN_DATA}:/challenge/holdout_data:ro" ` + -v "${MODEL_FULL}:/challenge/model:ro" ` + -v "${OUT_FULL}:/challenge/holdout_outputs" ` + -v "${FEATURE_CACHE}:/challenge/.feature_cache" ` + $IMAGE_NAME ` + python run_model.py -d holdout_data -m model -o holdout_outputs -v + + if (Test-DatasetHasLabels $RUN_DATA) { + Invoke-Evaluation $RUN_DATA $OUT_FULL "run-dataset" + } else { + Write-Host "Skipping evaluation for run dataset (labels not present in $RUN_DATA_REL/$DEMOGRAPHICS_FILE)." + } +} + +function Run-Smoke { + + $SMOKE_DATA = Get-AbsolutePath $SMOKE_DATA_REL + $MODEL_SMOKE = Get-AbsolutePath $MODEL_SMOKE_REL + $OUT_SMOKE = Join-Path (Get-AbsolutePath ".") $OUT_SMOKE_REL + $FEATURE_CACHE = Join-Path (Get-AbsolutePath ".") $FEATURE_CACHE_REL + + Ensure-Directory $OUT_SMOKE + Ensure-Directory $FEATURE_CACHE + + docker run --rm ` + -v "${SMOKE_DATA}:/challenge/holdout_data:ro" ` + -v "${MODEL_SMOKE}:/challenge/model:ro" ` + -v "${OUT_SMOKE}:/challenge/holdout_outputs" ` + -v "${FEATURE_CACHE}:/challenge/.feature_cache" ` + $IMAGE_NAME ` + python run_model.py -d holdout_data -m model -o holdout_outputs -v + + Invoke-Evaluation $SMOKE_DATA $OUT_SMOKE "smoke" +} + +function Eval-Full { + + $RUN_DATA = Get-AbsolutePath $RUN_DATA_REL + $OUT_FULL = Get-AbsolutePath $OUT_FULL_REL + + if (Test-DatasetHasLabels $RUN_DATA) { + Invoke-Evaluation $RUN_DATA $OUT_FULL "run-dataset" + } else { + Write-Host "Skipping evaluation for run dataset (labels not present in $RUN_DATA_REL/$DEMOGRAPHICS_FILE)." + } +} + +function Eval-Smoke { + + $SMOKE_DATA = Get-AbsolutePath $SMOKE_DATA_REL + $OUT_SMOKE = Get-AbsolutePath $OUT_SMOKE_REL + + Invoke-Evaluation $SMOKE_DATA $OUT_SMOKE "smoke" +} + +# ====================== +# MODO DESARROLLO (SIN REBUILD) +# ====================== + +function Train-Dev { + + $CODE_PATH = Get-AbsolutePath "." + $SMOKE_DATA = Get-AbsolutePath $SMOKE_DATA_REL + $MODEL_SMOKE = Join-Path $CODE_PATH $MODEL_SMOKE_REL + + Ensure-Directory $MODEL_SMOKE + + docker run --rm ` + -v "${CODE_PATH}:/challenge" ` + -v "${SMOKE_DATA}:/challenge/training_data:ro" ` + -v "${MODEL_SMOKE}:/challenge/model" ` + $IMAGE_NAME ` + python train_model.py -d training_data -m model -v +} + +function Run-Dev { + + $CODE_PATH = Get-AbsolutePath "." + $SMOKE_DATA = Get-AbsolutePath $SMOKE_DATA_REL + $MODEL_SMOKE = Get-AbsolutePath $MODEL_SMOKE_REL + $OUT_SMOKE = Join-Path $CODE_PATH $OUT_SMOKE_REL + + Ensure-Directory $OUT_SMOKE + + docker run --rm ` + -v "${CODE_PATH}:/challenge" ` + -v "${SMOKE_DATA}:/challenge/holdout_data:ro" ` + -v "${MODEL_SMOKE}:/challenge/model:ro" ` + -v "${OUT_SMOKE}:/challenge/holdout_outputs" ` + $IMAGE_NAME ` + python run_model.py -d holdout_data -m model -o holdout_outputs -v + + Invoke-EvaluationDev $CODE_PATH $SMOKE_DATA "/challenge/holdout_outputs" "development smoke" +} + +function Eval-Dev { + + $CODE_PATH = Get-AbsolutePath "." + $SMOKE_DATA = Get-AbsolutePath $SMOKE_DATA_REL + + Invoke-EvaluationDev $CODE_PATH $SMOKE_DATA "/challenge/holdout_outputs" "development smoke" +} + +function Clean-All { + + Remove-Item -Recurse -Force $MODEL_FULL_REL -ErrorAction SilentlyContinue + Remove-Item -Recurse -Force $MODEL_SMOKE_REL -ErrorAction SilentlyContinue + Remove-Item -Recurse -Force $OUT_FULL_REL -ErrorAction SilentlyContinue + Remove-Item -Recurse -Force $OUT_SMOKE_REL -ErrorAction SilentlyContinue + + Write-Host "Modelos y outputs eliminados." +} + +# ============================================ +# SWITCH PRINCIPAL +# ============================================ + +switch ($Command) { + + "build" { Build-Image } + "smoke" { Create-Smoke } + "train" { Train-Full } + "train-smoke" { Train-Smoke } + "run" { Run-Full } + "run-smoke" { Run-Smoke } + "eval" { Eval-Full } + "eval-smoke" { Eval-Smoke } + "train-dev" { Train-Dev } + "run-dev" { Run-Dev } + "eval-dev" { Eval-Dev } + "clean" { Clean-All } + +} \ No newline at end of file diff --git a/run.sh b/run.sh new file mode 100644 index 0000000..7569760 --- /dev/null +++ b/run.sh @@ -0,0 +1,316 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -lt 1 ]]; then + echo "Usage: $0 " + exit 1 +fi + +COMMAND="$1" + +# ============================================ +# CONFIGURATION +# ============================================ + +TRAIN_DATA_REL="data/training_set" +RUN_DATA_REL="data/test_set" +SMOKE_DATA_REL="data/training_smoke" + +IMAGE_NAME="cinc2026" + +MODEL_FULL_REL="model" +MODEL_SMOKE_REL="model_smoke" +FEATURE_CACHE_REL=".feature_cache" + +OUT_FULL_REL="outputs" +OUT_SMOKE_REL="outputs_smoke" +DEMOGRAPHICS_FILE="demographics.csv" + +# ============================================ +# HELPERS +# ============================================ + +get_absolute_path() { + local rel_path="$1" + (cd "$rel_path" && pwd) +} + +ensure_directory() { + local dir_path="$1" + mkdir -p "$dir_path" +} + +to_docker_path() { + local host_path="$1" + + if command -v cygpath >/dev/null 2>&1; then + cygpath -m "$host_path" + else + echo "$host_path" + fi +} + +docker_cli() { + MSYS_NO_PATHCONV=1 MSYS2_ARG_CONV_EXCL="*" docker "$@" +} + +evaluate_predictions() { + local data_dir="$1" + local output_dir="$2" + local label="$3" + local data_dir_docker output_dir_docker + + data_dir_docker="$(to_docker_path "$data_dir")" + output_dir_docker="$(to_docker_path "$output_dir")" + + echo "Evaluating ${label} predictions..." + docker_cli run --rm \ + -v "${data_dir_docker}:/challenge/eval_data:ro" \ + -v "${output_dir_docker}:/challenge/eval_outputs:ro" \ + "$IMAGE_NAME" \ + python evaluate_model.py \ + -d "/challenge/eval_data/${DEMOGRAPHICS_FILE}" \ + -o "/challenge/eval_outputs/${DEMOGRAPHICS_FILE}" +} + +evaluate_predictions_dev() { + local code_path="$1" + local data_path="$2" + local output_path="$3" + local label="$4" + local code_path_docker data_path_docker + + code_path_docker="$(to_docker_path "$code_path")" + data_path_docker="$(to_docker_path "$data_path")" + + echo "Evaluating ${label} predictions..." + docker_cli run --rm \ + -v "${code_path_docker}:/challenge" \ + -v "${data_path_docker}:/challenge/eval_data:ro" \ + "$IMAGE_NAME" \ + python evaluate_model.py \ + -d "/challenge/eval_data/${DEMOGRAPHICS_FILE}" \ + -o "$output_path/${DEMOGRAPHICS_FILE}" +} + +dataset_has_labels() { + local data_dir="$1" + local demographics_path="$data_dir/$DEMOGRAPHICS_FILE" + + [[ -f "$demographics_path" ]] && head -n 1 "$demographics_path" | grep -q "Cognitive_Impairment" +} + +build_image() { + docker_cli build -t "$IMAGE_NAME" . +} + +create_smoke() { + echo "Creating smoke dataset..." + bash scripts/create_smoke.sh +} + +train_full() { + local full_data model_full + local feature_cache + local full_data_docker model_full_docker feature_cache_docker + + full_data="$(get_absolute_path "$TRAIN_DATA_REL")" + model_full="$(get_absolute_path ".")/${MODEL_FULL_REL}" + feature_cache="$(get_absolute_path ".")/${FEATURE_CACHE_REL}" + full_data_docker="$(to_docker_path "$full_data")" + model_full_docker="$(to_docker_path "$model_full")" + feature_cache_docker="$(to_docker_path "$feature_cache")" + + ensure_directory "$model_full" + ensure_directory "$feature_cache" + + docker_cli run --rm \ + -v "${full_data_docker}:/challenge/training_data:ro" \ + -v "${model_full_docker}:/challenge/model" \ + -v "${feature_cache_docker}:/challenge/.feature_cache" \ + "$IMAGE_NAME" \ + python train_model.py -d training_data -m model -v +} + +train_smoke() { + local smoke_data model_smoke + local feature_cache + local smoke_data_docker model_smoke_docker feature_cache_docker + + smoke_data="$(get_absolute_path "$SMOKE_DATA_REL")" + model_smoke="$(get_absolute_path ".")/${MODEL_SMOKE_REL}" + feature_cache="$(get_absolute_path ".")/${FEATURE_CACHE_REL}" + smoke_data_docker="$(to_docker_path "$smoke_data")" + model_smoke_docker="$(to_docker_path "$model_smoke")" + feature_cache_docker="$(to_docker_path "$feature_cache")" + + ensure_directory "$model_smoke" + ensure_directory "$feature_cache" + + docker_cli run --rm \ + -v "${smoke_data_docker}:/challenge/training_data:ro" \ + -v "${model_smoke_docker}:/challenge/model" \ + -v "${feature_cache_docker}:/challenge/.feature_cache" \ + "$IMAGE_NAME" \ + python train_model.py -d training_data -m model -v +} + +run_full() { + local run_data model_full out_full + local feature_cache + local run_data_docker model_full_docker out_full_docker feature_cache_docker + + run_data="$(get_absolute_path "$RUN_DATA_REL")" + model_full="$(get_absolute_path "$MODEL_FULL_REL")" + out_full="$(get_absolute_path ".")/${OUT_FULL_REL}" + feature_cache="$(get_absolute_path ".")/${FEATURE_CACHE_REL}" + run_data_docker="$(to_docker_path "$run_data")" + model_full_docker="$(to_docker_path "$model_full")" + out_full_docker="$(to_docker_path "$out_full")" + feature_cache_docker="$(to_docker_path "$feature_cache")" + + ensure_directory "$out_full" + ensure_directory "$feature_cache" + + docker_cli run --rm \ + -v "${run_data_docker}:/challenge/holdout_data:ro" \ + -v "${model_full_docker}:/challenge/model:ro" \ + -v "${out_full_docker}:/challenge/holdout_outputs" \ + -v "${feature_cache_docker}:/challenge/.feature_cache" \ + "$IMAGE_NAME" \ + python run_model.py -d holdout_data -m model -o holdout_outputs -v + + if dataset_has_labels "$run_data"; then + evaluate_predictions "$run_data" "$out_full" "run-dataset" + else + echo "Skipping evaluation for run dataset (labels not present in ${RUN_DATA_REL}/${DEMOGRAPHICS_FILE})." + fi +} + +run_smoke() { + local smoke_data model_smoke out_smoke + local feature_cache + local smoke_data_docker model_smoke_docker out_smoke_docker feature_cache_docker + + smoke_data="$(get_absolute_path "$SMOKE_DATA_REL")" + model_smoke="$(get_absolute_path "$MODEL_SMOKE_REL")" + out_smoke="$(get_absolute_path ".")/${OUT_SMOKE_REL}" + feature_cache="$(get_absolute_path ".")/${FEATURE_CACHE_REL}" + smoke_data_docker="$(to_docker_path "$smoke_data")" + model_smoke_docker="$(to_docker_path "$model_smoke")" + out_smoke_docker="$(to_docker_path "$out_smoke")" + feature_cache_docker="$(to_docker_path "$feature_cache")" + + ensure_directory "$out_smoke" + ensure_directory "$feature_cache" + + docker_cli run --rm \ + -v "${smoke_data_docker}:/challenge/holdout_data:ro" \ + -v "${model_smoke_docker}:/challenge/model:ro" \ + -v "${out_smoke_docker}:/challenge/holdout_outputs" \ + -v "${feature_cache_docker}:/challenge/.feature_cache" \ + "$IMAGE_NAME" \ + python run_model.py -d holdout_data -m model -o holdout_outputs -v + + evaluate_predictions "$smoke_data" "$out_smoke" "smoke" +} + +eval_full() { + local run_data out_full + + run_data="$(get_absolute_path "$RUN_DATA_REL")" + out_full="$(get_absolute_path "$OUT_FULL_REL")" + + if dataset_has_labels "$run_data"; then + evaluate_predictions "$run_data" "$out_full" "run-dataset" + else + echo "Skipping evaluation for run dataset (labels not present in ${RUN_DATA_REL}/${DEMOGRAPHICS_FILE})." + fi +} + +eval_smoke() { + local smoke_data out_smoke + + smoke_data="$(get_absolute_path "$SMOKE_DATA_REL")" + out_smoke="$(get_absolute_path "$OUT_SMOKE_REL")" + + evaluate_predictions "$smoke_data" "$out_smoke" "smoke" +} + +# ===================== +# DEVELOPMENT MODE (NO REBUILD) +# ===================== + +train_dev() { + local code_path smoke_data model_smoke + local code_path_docker smoke_data_docker + + code_path="$(get_absolute_path ".")" + smoke_data="$(get_absolute_path "$SMOKE_DATA_REL")" + model_smoke="${code_path}/${MODEL_SMOKE_REL}" + code_path_docker="$(to_docker_path "$code_path")" + smoke_data_docker="$(to_docker_path "$smoke_data")" + + ensure_directory "$model_smoke" + + docker_cli run --rm \ + -v "${code_path_docker}:/challenge" \ + -v "${smoke_data_docker}:/challenge/data_smoke:ro" \ + "$IMAGE_NAME" \ + python train_model.py -d /challenge/data_smoke -m /challenge/model_smoke -v +} + +run_dev() { + local code_path smoke_data out_smoke + local code_path_docker smoke_data_docker + + code_path="$(get_absolute_path ".")" + smoke_data="$(get_absolute_path "$SMOKE_DATA_REL")" + out_smoke="${code_path}/${OUT_SMOKE_REL}" + code_path_docker="$(to_docker_path "$code_path")" + smoke_data_docker="$(to_docker_path "$smoke_data")" + + ensure_directory "$out_smoke" + + docker_cli run --rm \ + -v "${code_path_docker}:/challenge" \ + -v "${smoke_data_docker}:/challenge/data_smoke:ro" \ + "$IMAGE_NAME" \ + python run_model.py -d /challenge/data_smoke -m /challenge/model_smoke -o /challenge/outputs_smoke -v + + evaluate_predictions_dev "$code_path" "$smoke_data" "/challenge/outputs_smoke" "development smoke" +} + +eval_dev() { + local code_path smoke_data + + code_path="$(get_absolute_path ".")" + smoke_data="$(get_absolute_path "$SMOKE_DATA_REL")" + + evaluate_predictions_dev "$code_path" "$smoke_data" "/challenge/outputs_smoke" "development smoke" +} + +clean_all() { + rm -rf "$MODEL_FULL_REL" "$MODEL_SMOKE_REL" "$OUT_FULL_REL" "$OUT_SMOKE_REL" + echo "Models and outputs removed." +} + +case "$COMMAND" in + build) build_image ;; + smoke) create_smoke ;; + train) train_full ;; + train-smoke) train_smoke ;; + run) run_full ;; + run-smoke) run_smoke ;; + eval) eval_full ;; + eval-smoke) eval_smoke ;; + train-dev) train_dev ;; + run-dev) run_dev ;; + eval-dev) eval_dev ;; + clean) clean_all ;; + *) + echo "Invalid command: $COMMAND" + echo "Valid commands: build, smoke, train, train-smoke, run, run-smoke, eval, eval-smoke, train-dev, run-dev, eval-dev, clean" + exit 1 + ;; +esac diff --git a/scripts/copy_new_files.ps1 b/scripts/copy_new_files.ps1 new file mode 100644 index 0000000..3e8052c --- /dev/null +++ b/scripts/copy_new_files.ps1 @@ -0,0 +1,69 @@ +# Script para copiar archivos nuevos de prev_data a test_set +# Copia archivos que existen en X:\bsicos01\__comun\Physionet\prev_data\training_set +# pero que NO existen en C:\BSICoS\CincChallenge2026\python-example-2026\data\training_set +# Los archivos nuevos se copian a data\test_set + +# Definir rutas +$sourceDir = "X:\bsicos01\__comun\Physionet\prev_data\training_set" +$trainingSetDir = "C:\BSICoS\CincChallenge2026\python-example-2026\data\training_set" +$testSetDir = "C:\BSICoS\CincChallenge2026\python-example-2026\data\test_set" + +# Validar que existan los directorios de origen y destino +if (-not (Test-Path $sourceDir)) { + Write-Host "ERROR: Directorio fuente no encontrado: $sourceDir" -ForegroundColor Red + exit 1 +} + +if (-not (Test-Path $trainingSetDir)) { + Write-Host "ERROR: Directorio training_set no encontrado: $trainingSetDir" -ForegroundColor Red + exit 1 +} + +# Crear directorio test_set si no existe +if (-not (Test-Path $testSetDir)) { + Write-Host "Creando directorio: $testSetDir" -ForegroundColor Cyan + New-Item -ItemType Directory -Path $testSetDir -Force | Out-Null +} + +# Obtener archivos recursivamente de ambos directorios +Write-Host "Leyendo archivos..." -ForegroundColor Cyan + +# Obtener nombres relativos de archivos en training_set actual +$trainingFiles = @{} +Get-ChildItem -Path $trainingSetDir -Recurse -File | ForEach-Object { + $relativePath = $_.FullName.Substring($trainingSetDir.Length + 1) + $trainingFiles[$relativePath] = $true +} + +Write-Host "Se encontraron $($trainingFiles.Count) archivos en training_set actual" -ForegroundColor Yellow + +# Procesar archivos de la fuente +$newFilesCount = 0 +$sourceFiles = Get-ChildItem -Path $sourceDir -Recurse -File + +Write-Host "Procesando $($sourceFiles.Count) archivos de la fuente..." -ForegroundColor Cyan + +foreach ($file in $sourceFiles) { + $relativePath = $file.FullName.Substring($sourceDir.Length + 1) + + # Si el archivo NO existe en training_set, copiar a test_set + if (-not $trainingFiles.ContainsKey($relativePath)) { + $destFile = Join-Path $testSetDir $relativePath + $destDir = Split-Path -Parent $destFile + + # Crear directorio de destino si no existe + if (-not (Test-Path $destDir)) { + New-Item -ItemType Directory -Path $destDir -Force | Out-Null + } + + Write-Host "Copiando: $relativePath" -ForegroundColor Green + Copy-Item -Path $file.FullName -Destination $destFile -Force + $newFilesCount++ + } +} + +Write-Host "" +Write-Host "RESUMEN:" -ForegroundColor Yellow +Write-Host "Archivos nuevos copiados: $newFilesCount" -ForegroundColor Green +Write-Host "Ubicación: $testSetDir" -ForegroundColor Cyan +Write-Host "Script completado exitosamente" -ForegroundColor Green diff --git a/scripts/create_smoke.ps1 b/scripts/create_smoke.ps1 new file mode 100644 index 0000000..c8f45dd --- /dev/null +++ b/scripts/create_smoke.ps1 @@ -0,0 +1,98 @@ +# ============================================ +# Create smoke training dataset +# ============================================ + +# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> +# IMPORTANT: +# Each team member must modify this path to +# match their local dataset location. +# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> +$FULL_DATA_PATH = "data/training_set" # <-- CHANGE THIS IF NEEDED + +$SMOKE_PATH = "data/training_smoke" +$N_RECORDS = 5 + +Write-Host "Creating smoke dataset..." +Write-Host "Source: $FULL_DATA_PATH" +Write-Host "Destination: $SMOKE_PATH" + +Remove-Item -Recurse -Force $SMOKE_PATH -ErrorAction SilentlyContinue +New-Item -ItemType Directory -Force -Path $SMOKE_PATH | Out-Null + +$selectedRecords = New-Object System.Collections.Generic.List[object] + +# Select first N EDF files +$edfs = Get-ChildItem "$FULL_DATA_PATH/physiological_data" -Recurse -Filter *.edf | + Sort-Object FullName | + Select-Object -First $N_RECORDS + +foreach ($f in $edfs) { + $rel = $f.FullName.Substring((Resolve-Path $FULL_DATA_PATH).Path.Length).TrimStart('\') + $target = Join-Path $SMOKE_PATH $rel + New-Item -ItemType Directory -Force -Path (Split-Path $target) | Out-Null + Copy-Item $f.FullName $target + + $stem = [System.IO.Path]::GetFileNameWithoutExtension($f.Name) + $parts = $stem -split '_ses-' + $selectedRecords.Add([pscustomobject]@{ + SiteID = $f.Directory.Name + Patient = $parts[0] + Session = $parts[1] + }) | Out-Null +} + +# Copy only annotation EDFs for the selected smoke records. +foreach ($record in $selectedRecords) { + $algoSource = Join-Path $FULL_DATA_PATH "algorithmic_annotations/$($record.SiteID)/$($record.Patient)_ses-$($record.Session)_caisr_annotations.edf" + $algoTarget = Join-Path $SMOKE_PATH "algorithmic_annotations/$($record.SiteID)/$($record.Patient)_ses-$($record.Session)_caisr_annotations.edf" + if (Test-Path $algoSource) { + New-Item -ItemType Directory -Force -Path (Split-Path $algoTarget) | Out-Null + Copy-Item $algoSource $algoTarget + } + + $humanSource = Join-Path $FULL_DATA_PATH "human_annotations/$($record.SiteID)/$($record.Patient)_ses-$($record.Session)_expert_annotations.edf" + $humanTarget = Join-Path $SMOKE_PATH "human_annotations/$($record.SiteID)/$($record.Patient)_ses-$($record.Session)_expert_annotations.edf" + if (Test-Path $humanSource) { + New-Item -ItemType Directory -Force -Path (Split-Path $humanTarget) | Out-Null + Copy-Item $humanSource $humanTarget + } +} + +# Filter demographics to the copied smoke records. +$env:SMOKE_FULL_DATA_PATH = (Resolve-Path $FULL_DATA_PATH).Path +$env:SMOKE_PATH = (Resolve-Path $SMOKE_PATH).Path +python -c @" +import csv +import os +from pathlib import Path + +full_data = Path(os.environ['SMOKE_FULL_DATA_PATH']) +smoke_path = Path(os.environ['SMOKE_PATH']) + +source_csv = full_data / 'demographics.csv' +target_csv = smoke_path / 'demographics.csv' +phys_root = smoke_path / 'physiological_data' + +selected_records = set() +for edf_path in phys_root.rglob('*.edf'): + site_id = edf_path.parent.name + patient_part, session_part = edf_path.stem.rsplit('_ses-', 1) + selected_records.add((site_id, patient_part, session_part)) + +with source_csv.open('r', newline='', encoding='utf-8') as source_file: + reader = csv.DictReader(source_file) + rows = [ + row for row in reader + if (row['SiteID'], row['BidsFolder'], str(row['SessionID'])) in selected_records + ] + fieldnames = reader.fieldnames + +with target_csv.open('w', newline='', encoding='utf-8') as target_file: + writer = csv.DictWriter(target_file, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(rows) +"@ +Remove-Item Env:SMOKE_FULL_DATA_PATH -ErrorAction SilentlyContinue +Remove-Item Env:SMOKE_PATH -ErrorAction SilentlyContinue + +Write-Host "Smoke dataset created successfully." \ No newline at end of file diff --git a/scripts/create_smoke.sh b/scripts/create_smoke.sh new file mode 100644 index 0000000..bc01bf8 --- /dev/null +++ b/scripts/create_smoke.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +set -euo pipefail + +# ============================================ +# Create smoke training dataset +# ============================================ + +# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> +# IMPORTANT: +# Each team member can modify this path to +# match their local dataset location. +# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> +FULL_DATA_PATH="${FULL_DATA_PATH:-data/training_set}" # Override with env var if needed + +SMOKE_PATH="data/training_smoke" +N_RECORDS="${N_RECORDS:-20}" + +echo "Creating smoke dataset..." +echo "Source: ${FULL_DATA_PATH}" +echo "Destination: ${SMOKE_PATH}" + +rm -rf "${SMOKE_PATH}" +mkdir -p "${SMOKE_PATH}" + +selected_records_file="$(mktemp)" +trap 'rm -f "${selected_records_file}"' EXIT + +# Select first N EDF files +while IFS= read -r file_path; do + rel_path="${file_path#${FULL_DATA_PATH}/}" + target_path="${SMOKE_PATH}/${rel_path}" + mkdir -p "$(dirname "${target_path}")" + cp "${file_path}" "${target_path}" + stem="$(basename "${file_path}" .edf)" + patient_part="${stem%_ses-*}" + session_part="${stem##*_ses-}" + site_id="$(basename "$(dirname "${file_path}")")" + printf '%s,%s,%s\n' "${site_id}" "${patient_part}" "${session_part}" >> "${selected_records_file}" +done < <( + find "${FULL_DATA_PATH}/physiological_data" -type f -name "*.edf" | sort | head -n "${N_RECORDS}" +) + +# Copy only annotation EDFs for the selected smoke records. +while IFS=',' read -r site_id patient_part session_part; do + algo_source="${FULL_DATA_PATH}/algorithmic_annotations/${site_id}/${patient_part}_ses-${session_part}_caisr_annotations.edf" + algo_target="${SMOKE_PATH}/algorithmic_annotations/${site_id}/${patient_part}_ses-${session_part}_caisr_annotations.edf" + if [[ -f "${algo_source}" ]]; then + mkdir -p "$(dirname "${algo_target}")" + cp "${algo_source}" "${algo_target}" + fi + + human_source="${FULL_DATA_PATH}/human_annotations/${site_id}/${patient_part}_ses-${session_part}_expert_annotations.edf" + human_target="${SMOKE_PATH}/human_annotations/${site_id}/${patient_part}_ses-${session_part}_expert_annotations.edf" + if [[ -f "${human_source}" ]]; then + mkdir -p "$(dirname "${human_target}")" + cp "${human_source}" "${human_target}" + fi +done < "${selected_records_file}" + +# Filter demographics to the copied smoke records. +python - <<'PY' +import csv +from pathlib import Path + +full_data = Path("data/training_set") +smoke_path = Path("data/training_smoke") + +source_csv = full_data / "demographics.csv" +target_csv = smoke_path / "demographics.csv" +phys_root = smoke_path / "physiological_data" + +selected_records = set() +for edf_path in phys_root.rglob("*.edf"): + site_id = edf_path.parent.name + stem = edf_path.stem + patient_part, session_part = stem.rsplit("_ses-", 1) + selected_records.add((site_id, patient_part, session_part)) + +with source_csv.open("r", newline="", encoding="utf-8") as source_file: + reader = csv.DictReader(source_file) + rows = [ + row for row in reader + if (row["SiteID"], row["BidsFolder"], str(row["SessionID"])) in selected_records + ] + fieldnames = reader.fieldnames + +with target_csv.open("w", newline="", encoding="utf-8") as target_file: + writer = csv.DictWriter(target_file, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(rows) +PY + +echo "Smoke dataset created successfully." diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..6319343 --- /dev/null +++ b/src/__init__.py @@ -0,0 +1 @@ +"""Project source package.""" \ No newline at end of file diff --git a/src/common/__init__.py b/src/common/__init__.py new file mode 100644 index 0000000..5baf7d3 --- /dev/null +++ b/src/common/__init__.py @@ -0,0 +1 @@ +"""Shared helpers for active signal-processing modules.""" \ No newline at end of file diff --git a/src/common/channel_utils.py b/src/common/channel_utils.py new file mode 100644 index 0000000..a15aa18 --- /dev/null +++ b/src/common/channel_utils.py @@ -0,0 +1,31 @@ +import os + +import pandas as pd + + +CHANNEL_TABLE_CACHE = {} + + +def normalize_channel_label(text): + normalized = ''.join(ch if ch.isalnum() else ' ' for ch in str(text).lower()) + return ' '.join(normalized.split()) + + +def split_channel_aliases(raw_aliases): + return {normalize_channel_label(alias) for alias in str(raw_aliases).split(';') if alias} + + +def get_cached_channel_table(csv_path): + normalized_csv_path = os.path.abspath(csv_path) + channels = CHANNEL_TABLE_CACHE.get(normalized_csv_path) + if channels is None: + channels = pd.read_csv(normalized_csv_path) + CHANNEL_TABLE_CACHE[normalized_csv_path] = channels + return channels, normalized_csv_path + + +def find_matching_label(signal_dict, aliases): + for label in signal_dict.keys(): + if normalize_channel_label(label) in aliases: + return label + return None \ No newline at end of file diff --git a/src/common/signal_utils.py b/src/common/signal_utils.py new file mode 100644 index 0000000..385ec8f --- /dev/null +++ b/src/common/signal_utils.py @@ -0,0 +1,15 @@ +import numpy as np + + +def resample_signal(signal, fs, target_fs): + signal = np.asarray(signal, dtype=float) + if signal.size == 0: + return signal, target_fs + if fs == target_fs: + return signal, target_fs + + duration = signal.size / fs + target_samples = max(1, int(round(duration * target_fs))) + time_original = np.linspace(0, duration, signal.size) + time_target = np.linspace(0, duration, target_samples) + return np.interp(time_target, time_original, signal), target_fs \ No newline at end of file diff --git a/src/ecg_processing.py b/src/ecg_processing.py new file mode 100644 index 0000000..c0f8243 --- /dev/null +++ b/src/ecg_processing.py @@ -0,0 +1,63 @@ +from .lib.ecg_features import compute_ecg_features +import numpy as np +from src.common.channel_utils import normalize_channel_label + +ECG_KEYWORDS = ['ecg', 'ekg'] + +ECG_SEGMENT_FEATURE_NAMES = [ + "PIP", + "PNNLS", + "PNNSS", + "AVNN", + "SDNN", + "RMSSD", + "HF", + "ECTOPIC" +] +ECG_SEGMENT_FEATURE_LENGTH = len(ECG_SEGMENT_FEATURE_NAMES) +ECG_FEATURE_NAMES = ECG_SEGMENT_FEATURE_NAMES +ECG_FEATURE_LENGTH = ECG_SEGMENT_FEATURE_LENGTH + +def _find_ecg_channel(physiological_data): + for label in physiological_data.keys(): + label_clean = normalize_channel_label(label) + if any(keyword in label_clean for keyword in ECG_KEYWORDS): + return label + return None + + +def processECG(physiological_data, physiological_fs, csv_path): + results = np.full(ECG_SEGMENT_FEATURE_LENGTH, np.nan, dtype=np.float32) + + ecg_label = _find_ecg_channel(physiological_data) + + if ecg_label is None: + return results # no ECG found + + if ecg_label not in physiological_fs: + return results + + ecg_signal = np.asarray(physiological_data[ecg_label], dtype=float) + fs = physiological_fs[ecg_label] + + if ecg_signal.size == 0: + return results + + try: + values = compute_ecg_features(ecg_signal, fs, ECG_SEGMENT_FEATURE_LENGTH) + + if values is None or len(values) == 0: + return results + + values = np.asarray(values, dtype=np.float32) + values[~np.isfinite(values)] = np.nan + + if len(values) >= ECG_SEGMENT_FEATURE_LENGTH: + results[:] = values[:ECG_SEGMENT_FEATURE_LENGTH] + else: + results[:len(values)] = values + + except Exception: + pass + + return results diff --git a/src/eeg_processing.py b/src/eeg_processing.py new file mode 100644 index 0000000..e25cb70 --- /dev/null +++ b/src/eeg_processing.py @@ -0,0 +1,204 @@ +import pandas as pd +import numpy as np +from src.common.channel_utils import find_matching_label, get_cached_channel_table, normalize_channel_label, split_channel_aliases +from src.common.signal_utils import resample_signal +from .lib import eeg_features + +EEG_CHANNEL_SPECS = { + 'C3-M2': {'direct': 'c3-m2', 'positive': 'c3', 'reference': 'm2'}, + 'C4-M1': {'direct': 'c4-m1', 'positive': 'c4', 'reference': 'm1'}, + 'F3-M2': {'direct': 'f3-m2', 'positive': 'f3', 'reference': 'm2'}, + 'F4-M1': {'direct': 'f4-m1', 'positive': 'f4', 'reference': 'm1'}, +} +EEG_FEATURE_SPECS = [ + *[ + (channel_name, feature_name) + for channel_name in EEG_CHANNEL_SPECS + for feature_name in eeg_features.SLOW_WAVE_FEATURE_NAMES + ], + ('C3-M2', 'Hjorth_Complexity'), + ('C4-M1', 'Hjorth_Complexity'), + ('F3-M2', 'Hjorth_Complexity'), + ('F4-M1', 'Hjorth_Complexity'), + ('C3-M2', 'Hjorth_Mobility'), + ('F3-M2', 'Hjorth_Mobility'), + ('F4-M1', 'Hjorth_Mobility'), + ('C3-M2', 'Ratio_Slow_Fast'), + ('C4-M1', 'Ratio_Slow_Fast'), + ('F3-M2', 'Ratio_Slow_Fast'), + ('F4-M1', 'Ratio_Slow_Fast'), + ('C3-M2', 'Rel_Beta'), + ('F3-M2', 'Rel_Beta'), + ('F4-M1', 'Rel_Beta'), + ('C4-M1', 'Rel_Sigma'), + ('F3-M2', 'Rel_Sigma'), + ('C3-M2', 'Relative_Delta_Power'), + ('C4-M1', 'Relative_Delta_Power'), + ('F3-M2', 'Relative_Delta_Power'), + ('F4-M1', 'Relative_Delta_Power'), + ('C3-M2', 'Theta_Alpha_Ratio'), + ('C4-M1', 'Theta_Alpha_Ratio'), + ('F3-M2', 'Theta_Alpha_Ratio'), + ('F4-M1', 'Theta_Alpha_Ratio'), + ('C3-M2', 'Theta_Beta_Ratio'), + ('C4-M1', 'Theta_Beta_Ratio'), + ('F3-M2', 'Theta_Beta_Ratio'), + ('F4-M1', 'Theta_Beta_Ratio'), + ('C3-M2', 'kurtosis_Alpha'), + ('C3-M2', 'kurtosis_Beta'), + ('C4-M1', 'kurtosis_Beta'), + ('F3-M2', 'kurtosis_Beta'), + ('F4-M1', 'kurtosis_Beta'), + ('C3-M2', 'kurtosis_Delta'), + ('C4-M1', 'kurtosis_Delta'), + ('F3-M2', 'kurtosis_Delta'), + ('F4-M1', 'kurtosis_Delta'), + ('C3-M2', 'kurtosis_Sigma'), + ('C4-M1', 'kurtosis_Sigma'), + ('F3-M2', 'kurtosis_Sigma'), + ('F4-M1', 'kurtosis_Sigma'), + ('C3-M2', 'kurtosis_Theta'), + ('C4-M1', 'kurtosis_Theta'), + ('F3-M2', 'kurtosis_Theta'), + ('F4-M1', 'kurtosis_Theta'), + ('C3-M2', 'variability_Delta'), + ('C4-M1', 'variability_Delta'), + ('F3-M2', 'variability_Delta'), + ('F4-M1', 'variability_Delta'), +] +EEG_SEGMENT_FEATURE_NAMES = [f'{channel}_{metric}' for channel, metric in EEG_FEATURE_SPECS] +EEG_SEGMENT_FEATURE_LENGTH = len(EEG_SEGMENT_FEATURE_NAMES) +EEG_FEATURE_NAMES = EEG_SEGMENT_FEATURE_NAMES +EEG_FEATURE_LENGTH = EEG_SEGMENT_FEATURE_LENGTH +EEG_ALIASES_CACHE = {} + + +def _build_eeg_aliases(channels): + alias_lookup = {} + for _, row in channels.iterrows(): + aliases = split_channel_aliases(row['Channel_Names']) + if not aliases: + continue + canonical_name = normalize_channel_label(str(row['Channel_Names']).split(';')[0]) + alias_lookup[canonical_name] = aliases + return alias_lookup + + +def _get_eeg_aliases(csv_path): + channels, normalized_csv_path = get_cached_channel_table(csv_path) + eeg_aliases = EEG_ALIASES_CACHE.get(normalized_csv_path) + if eeg_aliases is None: + eeg_aliases = _build_eeg_aliases(channels) + EEG_ALIASES_CACHE[normalized_csv_path] = eeg_aliases + return eeg_aliases + + +def _get_channel_signal(channel_name, physiological_data, physiological_fs, eeg_aliases): + channel_spec = EEG_CHANNEL_SPECS[channel_name] + direct_aliases = eeg_aliases.get(normalize_channel_label(channel_spec['direct']), set()) + direct_label = find_matching_label(physiological_data, direct_aliases) + if direct_label is not None and direct_label in physiological_fs: + return np.asarray(physiological_data[direct_label], dtype=float), physiological_fs[direct_label] + + positive_aliases = eeg_aliases.get(normalize_channel_label(channel_spec['positive']), set()) + reference_aliases = eeg_aliases.get(normalize_channel_label(channel_spec['reference']), set()) + positive_label = find_matching_label(physiological_data, positive_aliases) + reference_label = find_matching_label(physiological_data, reference_aliases) + if positive_label is None or reference_label is None: + return None, None + if positive_label not in physiological_fs or reference_label not in physiological_fs: + return None, None + + positive_fs = physiological_fs[positive_label] + reference_fs = physiological_fs[reference_label] + if positive_fs != reference_fs: + return None, None + + return ( + np.asarray(physiological_data[positive_label], dtype=float) + - np.asarray(physiological_data[reference_label], dtype=float), + positive_fs, + ) + + +def _extract_channel_metrics(signal, fs): + signal = np.nan_to_num(np.asarray(signal, dtype=float), nan=0.0, posinf=0.0, neginf=0.0) + if signal.size < max(int(fs * 30), 2): + return None + + if fs != 200: + signal, fs = resample_signal(signal, fs, 200) + + filtered = eeg_features.butter_bandpass_filter(signal, lowcut=0.3, highcut=35, fs=fs, order=4) + signal_std = np.std(filtered) + if signal_std == 0 or not np.isfinite(signal_std): + return None + + normalized = (filtered - np.mean(filtered)) / signal_std + epochs = eeg_features.create_epochs(normalized, fs, epoch_duration=30) + if epochs.size == 0: + return None + + band_powers, complexities = eeg_features.extract_band_powers(epochs, fs, win_len=30) + if len(band_powers) > 60: + band_powers = band_powers.iloc[60:] + complexities = complexities.iloc[60:] + if band_powers.empty: + return None + + patient_profile = eeg_features.get_patient_profile(band_powers) + metrics = { + str(name): float(value) + for name, value in patient_profile.replace([np.inf, -np.inf], np.nan).items() + } + + try: + slow_wave_features = eeg_features.get_SW_features(signal, fs) + # SW extraction is optional: a detector failure must not discard the + # spectral and Hjorth metrics already computed for this EEG channel. + except Exception: + slow_wave_features = { + feature_name: np.nan + for feature_name in eeg_features.SLOW_WAVE_FEATURE_NAMES + } + + for feature_name, value in slow_wave_features.items(): + metrics[feature_name] = float(np.nan if pd.isna(value) else value) + + for complexity_name in ('Hjorth_Mobility', 'Hjorth_Complexity'): + if complexity_name in complexities: + value = complexities[complexity_name].replace([np.inf, -np.inf], np.nan).std() + metrics[complexity_name] = float(np.nan if pd.isna(value) else value) + else: + metrics[complexity_name] = np.nan + return metrics + + +def processEEG(physiological_data, physiological_fs, csv_path): + eeg_aliases = _get_eeg_aliases(csv_path) + channel_profiles = {} + + for channel_name in EEG_CHANNEL_SPECS: + signal, fs = _get_channel_signal(channel_name, physiological_data, physiological_fs, eeg_aliases) + if signal is None or fs is None: + continue + + metrics = _extract_channel_metrics(signal, fs) + if metrics is not None: + channel_profiles[channel_name] = metrics + + if not channel_profiles: + return np.full(EEG_SEGMENT_FEATURE_LENGTH, np.nan, dtype=np.float32) + + values = [] + for channel_name, metric_name in EEG_FEATURE_SPECS: + channel_metrics = channel_profiles.get(channel_name) + if channel_metrics is None: + values.append(np.nan) + continue + values.append(float(channel_metrics.get(metric_name, np.nan))) + + return np.asarray(values, dtype=np.float32) + + +_normalize_label = normalize_channel_label diff --git a/src/lib/__init__.py b/src/lib/__init__.py new file mode 100644 index 0000000..665f43f --- /dev/null +++ b/src/lib/__init__.py @@ -0,0 +1 @@ +"""Signal-processing helper package.""" \ No newline at end of file diff --git a/src/lib/ecg_features.py b/src/lib/ecg_features.py new file mode 100644 index 0000000..b07582a --- /dev/null +++ b/src/lib/ecg_features.py @@ -0,0 +1,75 @@ +import numpy as np +from typing import cast +from scipy.signal import butter, filtfilt, resample +from .ecg_peak_detection import pan_tompkins +from .ecg_hrv_features import compute_hrv_hrf +from .ecg_nn_interpolation import interpolate_nn_pchip +from .ecg_rr_cleaning import remove_ectopic_beats + + +def compute_ecg_features(ecg_signal, fs, ecg_feature_length): + fs = int(round(float(fs))) + if fs <= 0: + return None + + signal_duration_seconds = len(ecg_signal) / fs + + ecg_signal = ecg_signal - np.mean(ecg_signal) + + target_fs = 200 + length_ecg = len(ecg_signal) + + if fs != target_fs: + num_samples = int(length_ecg * target_fs / fs) + ecg_signal = resample(ecg_signal, num_samples) + fs = target_fs + + length_ecg = len(ecg_signal) + window_length_seconds = max(1, int(signal_duration_seconds)) + minimum_intervals_per_window = max(1, int(np.ceil(window_length_seconds / 2))) + + if np.sum(np.isnan(ecg_signal)) != 0 or np.sum(ecg_signal == 0) > 0.2 * length_ecg: + return np.full(ecg_feature_length, np.nan, dtype=np.float32) + + b, a = cast(tuple[np.ndarray, np.ndarray], butter(3, [59.5/(fs/2), 60.5/(fs/2)], btype='bandstop', output='ba')) + ecg_signal = filtfilt(b, a, ecg_signal) + + b, a = cast(tuple[np.ndarray, np.ndarray], butter(3, 0.5/(fs/2), btype='high', output='ba')) + ecg_signal = filtfilt(b, a, ecg_signal) + + b, a = cast(tuple[np.ndarray, np.ndarray], butter(3, 45/(fs/2), btype='low', output='ba')) + ecg_signal = filtfilt(b, a, ecg_signal) + + _, r_locs, _ = pan_tompkins(ecg_signal, fs, 0) + + if len(r_locs) - 1 < minimum_intervals_per_window: + return np.full(ecg_feature_length, np.nan, dtype=np.float32) + + nn_intervals = np.diff(r_locs) / fs + + nn_intervals, ectopic_perc = remove_ectopic_beats(nn_intervals, 40, 0.10) + nn_intervals = interpolate_nn_pchip(nn_intervals, 2) + + if len(nn_intervals) == 0: + return np.full(ecg_feature_length, np.nan, dtype=np.float32) + + valid_ratio = np.sum(~np.isnan(nn_intervals)) / len(nn_intervals) + nn_intervals = nn_intervals[~np.isnan(nn_intervals)] + + if valid_ratio < 0.75 or len(nn_intervals) < minimum_intervals_per_window: + return np.full(ecg_feature_length, np.nan, dtype=np.float32) + + metrics = compute_hrv_hrf(nn_intervals, fs) + + features = np.array([ + metrics["PIP"], + metrics["PNNLS"], + metrics["PNNSS"], + metrics["AVNN"], + metrics["SDNN"], + metrics["RMSSD"], + metrics["HF"], + ectopic_perc, + ], dtype=np.float32) + + return features diff --git a/src/lib/ecg_hrv_features.py b/src/lib/ecg_hrv_features.py new file mode 100644 index 0000000..f3c2c4b --- /dev/null +++ b/src/lib/ecg_hrv_features.py @@ -0,0 +1,88 @@ +import numpy as np +from scipy.integrate import trapezoid +from scipy.signal import lombscargle + +def compute_hrv_hrf(nn_intervals, sampling_frequency): + """Compute time-domain and HF HRV metrics from NN intervals.""" + + nn_intervals = np.asarray(nn_intervals).flatten() + + delta_nn = np.diff(nn_intervals) + + n = 1 + threshold = n / sampling_frequency + + acceleration = delta_nn <= -threshold + deceleration = delta_nn >= threshold + + sign_delta_nn = np.zeros_like(delta_nn) + sign_delta_nn[acceleration] = -1 + sign_delta_nn[deceleration] = 1 + + num_deltas = len(delta_nn) + + inflection = 0 + + for index in range(num_deltas - 1): + if (delta_nn[index + 1] * delta_nn[index] <= 0) and (delta_nn[index + 1] != delta_nn[index]): + inflection += 1 + + pip = (inflection / (num_deltas - 1)) * 100 if num_deltas > 1 else np.nan + + segments = [] + if num_deltas > 0: + current_segment = sign_delta_nn[0] + segment_length = 1 + + for index in range(1, num_deltas): + if sign_delta_nn[index] == current_segment and sign_delta_nn[index] != 0: + segment_length += 1 + else: + if current_segment != 0: + segments.append(segment_length) + current_segment = sign_delta_nn[index] + segment_length = 1 + + if current_segment != 0: + segments.append(segment_length) + + segments = np.array(segments) + + if len(segments) > 0: + long_segments = segments[segments >= 3] + short_segments = segments[segments < 3] + + pnnls = np.sum(long_segments) / num_deltas * 100 + pnnss = np.sum(short_segments) / np.sum(segments) * 100 + else: + pnnls = np.nan + pnnss = np.nan + + avnn = np.nanmean(nn_intervals) if len(nn_intervals) > 0 else np.nan + sdnn = np.nanstd(nn_intervals, ddof=1) if len(nn_intervals) > 1 else np.nan + + diff_nn = np.diff(nn_intervals) + rmssd = np.sqrt(np.nanmean(diff_nn**2)) if len(diff_nn) > 0 else np.nan + + if len(nn_intervals) > 1: + elapsed_time = np.cumsum(nn_intervals) + frequencies = np.linspace(0.01, 0.5, 1000) + angular_frequencies = 2 * np.pi * frequencies + detrended_nn = nn_intervals - np.nanmean(nn_intervals) + power_spectrum = lombscargle(elapsed_time, detrended_nn, angular_frequencies, normalize=True) + hf_band = (frequencies >= 0.15) & (frequencies <= 0.4) + hf = trapezoid(power_spectrum[hf_band], frequencies[hf_band]) + else: + hf = np.nan + + results = { + "PIP": pip, + "PNNLS": pnnls, + "PNNSS": pnnss, + "AVNN": avnn, + "SDNN": sdnn, + "RMSSD": rmssd, + "HF": hf, + } + + return results diff --git a/src/lib/ecg_nn_interpolation.py b/src/lib/ecg_nn_interpolation.py new file mode 100644 index 0000000..dc61a42 --- /dev/null +++ b/src/lib/ecg_nn_interpolation.py @@ -0,0 +1,34 @@ +import numpy as np +from scipy.interpolate import PchipInterpolator + +def interpolate_nn_pchip(nn_intervals, max_gap): + """Interpolate short NaN gaps in NN intervals using PCHIP.""" + + nn_intervals = np.asarray(nn_intervals).flatten() + nn_interpolated = nn_intervals.copy() + + nan_indices = np.isnan(nn_intervals) + + boundaries = np.diff(np.concatenate(([0], nan_indices.astype(int), [0]))) + start_indices = np.where(boundaries == 1)[0] + end_indices = np.where(boundaries == -1)[0] - 1 + + for index in range(len(start_indices)): + segment_length = end_indices[index] - start_indices[index] + 1 + + if segment_length <= max_gap: + left = start_indices[index] - 1 + right = end_indices[index] + 1 + + if (left >= 0 and right < len(nn_intervals) and + not np.isnan(nn_intervals[left]) and not np.isnan(nn_intervals[right])): + + x = np.array([left, right]) + y = np.array([nn_intervals[left], nn_intervals[right]]) + + xi = np.arange(start_indices[index], end_indices[index] + 1) + + interpolator = PchipInterpolator(x, y) + nn_interpolated[xi] = interpolator(xi) + + return nn_interpolated \ No newline at end of file diff --git a/src/lib/ecg_peak_detection.py b/src/lib/ecg_peak_detection.py new file mode 100644 index 0000000..89e4df3 --- /dev/null +++ b/src/lib/ecg_peak_detection.py @@ -0,0 +1,187 @@ +import numpy as np +from scipy.signal import butter, filtfilt, find_peaks + + +def pan_tompkins(ecg, fs, gr=0): + fs = int(round(float(fs))) + if fs <= 0: + raise ValueError('Sampling frequency must be positive.') + + ecg = np.asarray(ecg).flatten() + delay = 0 + + skip = 0 + m_selected_RR = 0 + mean_RR = 0 + ser_back = 0 + + # ===================== FILTERING ===================== # + ecg = ecg - np.mean(ecg) + + if fs == 200: + # Low-pass + b, a = butter(3, 12*2/fs, btype='low') + ecg_l = filtfilt(b, a, ecg) + ecg_l = ecg_l / np.max(np.abs(ecg_l)) + + # High-pass + b, a = butter(3, 5*2/fs, btype='high') + ecg_h = filtfilt(b, a, ecg_l) + ecg_h = ecg_h / np.max(np.abs(ecg_h)) + else: + b, a = butter(3, [5*2/fs, 15*2/fs], btype='band') + ecg_h = filtfilt(b, a, ecg) + ecg_h = ecg_h / np.max(np.abs(ecg_h)) + + # ===================== DERIVATIVE ===================== # + if fs != 200: + int_c = int((5 - 1) / (fs * (1/40))) + base = np.array([1, 2, 0, -2, -1]) * (1/8) * fs + x_old = np.linspace(1, 5, 5) + x_new = np.linspace(1, 5, int_c) + b = np.interp(x_new, x_old, base) + else: + b = np.array([1, 2, 0, -2, -1]) * (1/8) * fs + + ecg_d = filtfilt(b, [1], ecg_h) + ecg_d = ecg_d / np.max(np.abs(ecg_d)) + + # ===================== SQUARING ===================== # + ecg_s = ecg_d ** 2 + + # ===================== MOVING WINDOW ===================== # + win = int(round(0.150 * fs)) + ecg_m = np.convolve(ecg_s, np.ones(win)/win, mode='same') + delay += win // 2 + + # ===================== PEAK DETECTION ===================== # + locs, _ = find_peaks(ecg_m, distance=int(0.2 * fs)) + pks = ecg_m[locs] + + LLp = len(pks) + + qrs_i = [] + qrs_c = [] + qrs_i_raw = [] + qrs_amp_raw = [] + + nois_i = [] + nois_c = [] + + # Threshold initialization + THR_SIG = np.max(ecg_m[:2*fs]) / 3 + THR_NOISE = np.mean(ecg_m[:2*fs]) / 2 + SIG_LEV = THR_SIG + NOISE_LEV = THR_NOISE + + THR_SIG1 = np.max(ecg_h[:2*fs]) / 3 + THR_NOISE1 = np.mean(ecg_h[:2*fs]) / 2 + SIG_LEV1 = THR_SIG1 + NOISE_LEV1 = THR_NOISE1 + + Beat_C = 0 + Beat_C1 = 0 + + for i in range(LLp): + + loc = locs[i] + + # Find peak in filtered signal + left = max(0, loc - int(0.150 * fs)) + right = loc + + if right < len(ecg_h): + segment = ecg_h[left:right+1] + if len(segment) > 0: + y_i = np.max(segment) + x_i = np.argmax(segment) + else: + continue + else: + continue + + # RR interval update + if len(qrs_i) >= 9: + diffRR = np.diff(qrs_i[-8:]) + mean_RR = np.mean(diffRR) + comp = qrs_i[-1] - qrs_i[-2] + + if comp <= 0.92 * mean_RR or comp >= 1.16 * mean_RR: + THR_SIG *= 0.5 + THR_SIG1 *= 0.5 + else: + m_selected_RR = mean_RR + + test_m = m_selected_RR if m_selected_RR else mean_RR + + # ===================== SEARCH BACK ===================== # + if test_m and len(qrs_i) > 0: + if (loc - qrs_i[-1]) >= int(1.66 * test_m): + + sb_left = qrs_i[-1] + int(0.2 * fs) + sb_right = loc - int(0.2 * fs) + + if sb_right > sb_left: + segment = ecg_m[sb_left:sb_right] + if len(segment) > 0: + pks_temp = np.max(segment) + locs_temp = sb_left + np.argmax(segment) + + if pks_temp > THR_NOISE: + qrs_c.append(pks_temp) + qrs_i.append(locs_temp) + + seg = ecg_h[max(0, locs_temp-int(0.150*fs)):locs_temp] + if len(seg) > 0: + y_i_t = np.max(seg) + x_i_t = np.argmax(seg) + + if y_i_t > THR_NOISE1: + qrs_i_raw.append(locs_temp - int(0.150*fs) + x_i_t) + qrs_amp_raw.append(y_i_t) + SIG_LEV1 = 0.25*y_i_t + 0.75*SIG_LEV1 + + SIG_LEV = 0.25*pks_temp + 0.75*SIG_LEV + + # ===================== CLASSIFICATION ===================== # + if pks[i] >= THR_SIG: + + # T-wave rejection + if len(qrs_i) >= 3: + if (loc - qrs_i[-1]) <= int(0.36 * fs): + + slope1 = np.mean(np.diff(ecg_m[max(0, loc-int(0.075*fs)):loc])) + slope2 = np.mean(np.diff(ecg_m[max(0, qrs_i[-1]-int(0.075*fs)):qrs_i[-1]])) + + if abs(slope1) <= 0.5 * abs(slope2): + NOISE_LEV1 = 0.125*y_i + 0.875*NOISE_LEV1 + NOISE_LEV = 0.125*pks[i] + 0.875*NOISE_LEV + continue + + # Accept QRS + qrs_c.append(pks[i]) + qrs_i.append(loc) + + if y_i >= THR_SIG1: + qrs_i_raw.append(loc - int(0.150*fs) + x_i) + qrs_amp_raw.append(y_i) + SIG_LEV1 = 0.125*y_i + 0.875*SIG_LEV1 + + SIG_LEV = 0.125*pks[i] + 0.875*SIG_LEV + + elif THR_NOISE <= pks[i] < THR_SIG: + NOISE_LEV1 = 0.125*y_i + 0.875*NOISE_LEV1 + NOISE_LEV = 0.125*pks[i] + 0.875*NOISE_LEV + + else: + NOISE_LEV1 = 0.125*y_i + 0.875*NOISE_LEV1 + NOISE_LEV = 0.125*pks[i] + 0.875*NOISE_LEV + + # Update thresholds + THR_SIG = NOISE_LEV + 0.25 * abs(SIG_LEV - NOISE_LEV) + THR_NOISE = 0.5 * THR_SIG + + THR_SIG1 = NOISE_LEV1 + 0.25 * abs(SIG_LEV1 - NOISE_LEV1) + THR_NOISE1 = 0.5 * THR_SIG1 + + return np.array(qrs_amp_raw), np.array(qrs_i_raw), delay \ No newline at end of file diff --git a/src/lib/ecg_rr_cleaning.py b/src/lib/ecg_rr_cleaning.py new file mode 100644 index 0000000..e56597a --- /dev/null +++ b/src/lib/ecg_rr_cleaning.py @@ -0,0 +1,38 @@ +import numpy as np + +def remove_ectopic_beats(nn_intervals, window_size, threshold): + nn_intervals = np.asarray(nn_intervals).flatten() + nn_corrected = nn_intervals.copy() + + half_win = window_size // 2 + ectopic_count = 0 + valid_count = 0 + + for index in range(len(nn_intervals)): + + if np.isnan(nn_intervals[index]): + continue + + valid_count += 1 + + left = max(0, index - half_win) + right = min(len(nn_intervals), index + half_win + 1) + + local_segment = nn_intervals[left:right] + local_segment = local_segment[~np.isnan(local_segment)] + + if local_segment.size == 0: + continue + + med_val = np.median(local_segment) + + if abs(nn_intervals[index] - med_val) > threshold * med_val: + nn_corrected[index] = med_val + ectopic_count += 1 + + if valid_count > 0: + ectopic_perc = (ectopic_count / valid_count) * 100 + else: + ectopic_perc = np.nan + + return nn_corrected, ectopic_perc \ No newline at end of file diff --git a/src/lib/eeg_features.py b/src/lib/eeg_features.py new file mode 100644 index 0000000..854035a --- /dev/null +++ b/src/lib/eeg_features.py @@ -0,0 +1,230 @@ +"""EEG feature helpers used by the active submission pipeline.""" + +from contextlib import nullcontext, redirect_stdout +import io + +import numpy as np +import pandas as pd +from scipy.signal import butter, filtfilt, welch +from scipy.stats import kurtosis, entropy + +from .swa import swa_CalculateReference +from .swa import swa_FindSWChannels +from .swa import swa_FindSWRef +from .swa import swa_getInfoDefaults + + +SLOW_WAVE_FEATURE_NAMES = ( + 'TotalSW', + 'SWdensity', + 'SWpeakAmp_mean', + 'SWpeakAmp_std', + 'SWp2p_mean', + 'SWp2p_std', + 'SWnegSlope_mean', + 'SWnegSlope_std', + 'SWposSlope_mean', + 'SWposSlope_std', + 'SWduration_mean', + 'SWduration_std', +) + +_SLOW_WAVE_EVENT_FIELDS = { + 'SWpeakAmp': 'Ref_PeakAmp', + 'SWp2p': 'Ref_P2PAmp', + 'SWnegSlope': 'Ref_NegSlope', + 'SWposSlope': 'Ref_PosSlope', +} + +def _safe_sqrt_variance_ratio(numerator_signal, denominator_signal): + numerator_var = np.var(numerator_signal) + denominator_var = np.var(denominator_signal) + if denominator_var <= 0 or not np.isfinite(denominator_var): + return 0.0 + ratio = numerator_var / denominator_var + if ratio <= 0 or not np.isfinite(ratio): + return 0.0 + return float(np.sqrt(ratio)) + +def butter_bandpass_filter(data, lowcut, highcut, fs, order=4): + nyq = 0.5 * fs + low = lowcut / nyq + high = highcut / nyq + b, a = butter(order, [low, high], btype='bandpass') + y = filtfilt(b, a, data) + return y + +def create_epochs(data, fs, epoch_duration=30): + samples_per_epoch = int(fs * epoch_duration) + num_epochs = len(data) // samples_per_epoch + + data_trimmed = data[:num_epochs * samples_per_epoch] + epochs = data_trimmed.reshape(num_epochs, samples_per_epoch) + return epochs + +def extract_band_powers(epochs, fs, win_len = 2): + features = [] + complexities = [] + bands = { + 'Delta': (0.5, 4), + 'Theta': (4, 8), + 'Alpha': (8, 12), + 'Sigma': (11, 16), + 'Beta': (12, 30) + } + + for epoch in epochs: + freqs, psd = welch(epoch, fs, nperseg=fs*30) + epoch_features = {} + for band_name, (low, high) in bands.items(): + idx_band = np.logical_and(freqs >= low, freqs <= high) + epoch_features[band_name] = np.mean(psd[idx_band]) + + features.append(epoch_features) + + diff = np.diff(epoch) + mobility = _safe_sqrt_variance_ratio(diff, epoch) + diff2 = np.diff(diff) + mobility_diff = _safe_sqrt_variance_ratio(diff2, diff) + complexity = mobility_diff / mobility if mobility > 0 else 0 + complexities.append({'Hjorth_Mobility': mobility, 'Hjorth_Complexity': complexity}) + + return pd.DataFrame(features), pd.DataFrame(complexities) + +def _finite_slow_wave_values(slow_waves, field_name): + values = [] + for wave in slow_waves: + try: + value = float(np.asarray(wave[field_name]).squeeze()) + except (KeyError, TypeError, ValueError): + continue + if np.isfinite(value): + values.append(value) + return np.asarray(values, dtype=float) + + +def summarize_slow_waves(slow_waves, fs, signal_duration_seconds): + """Aggregate the event dictionaries returned by ``swa`` into scalar features. + + Amplitudes and slopes retain the sign and units returned by ``swa``. + Durations and density are expressed in seconds and waves/minute, respectively. + """ + if fs <= 0 or not np.isfinite(fs): + raise ValueError('fs must be a positive finite sampling frequency.') + if signal_duration_seconds <= 0 or not np.isfinite(signal_duration_seconds): + raise ValueError('signal_duration_seconds must be positive and finite.') + + slow_waves = [] if slow_waves is None else list(slow_waves) + features = {name: np.nan for name in SLOW_WAVE_FEATURE_NAMES} + features['TotalSW'] = float(len(slow_waves)) + features['SWdensity'] = float(len(slow_waves) / (signal_duration_seconds / 60.0)) + + for feature_prefix, event_field in _SLOW_WAVE_EVENT_FIELDS.items(): + values = _finite_slow_wave_values(slow_waves, event_field) + if values.size: + features[f'{feature_prefix}_mean'] = float(np.mean(values)) + features[f'{feature_prefix}_std'] = float(np.std(values)) + + durations = [] + for wave in slow_waves: + try: + duration = ( + float(np.asarray(wave['Ref_UpInd']).squeeze()) + - float(np.asarray(wave['Ref_DownInd']).squeeze()) + ) / float(fs) + except (KeyError, TypeError, ValueError): + continue + if np.isfinite(duration) and duration > 0: + durations.append(duration) + + if durations: + features['SWduration_mean'] = float(np.mean(durations)) + features['SWduration_std'] = float(np.std(durations)) + + return features + + +def get_SW_features(signal, fs, verbose=False): + """Detect slow waves with ``swa`` and return fixed-length scalar features.""" + signal = np.asarray(signal, dtype=float).reshape(-1) + if fs <= 0 or not np.isfinite(fs): + raise ValueError('fs must be a positive finite sampling frequency.') + if signal.size == 0: + raise ValueError('signal must contain at least one sample.') + if not np.all(np.isfinite(signal)): + raise ValueError('signal must contain only finite values.') + + info = swa_getInfoDefaults.swa_getInfoDefaults({}, 'SW', method='envelope') + info['Electrodes'] = ['EEG'] + info['Recording'] = {'sRate': float(fs)} + info['Parameters']['Ref_InspectionPoint'] = 'ZC' + # Spatial clustering is undefined for a single-channel invocation. + info['Parameters']['Channels_ClusterTest'] = False + + # swa consistently uses (channels, samples). + data = {'Raw': signal[np.newaxis, :]} + output_context = nullcontext() if verbose else redirect_stdout(io.StringIO()) + with output_context: + data['SWRef'], info = swa_CalculateReference.swa_CalculateReference( + data['Raw'], info, False + ) + data, info, slow_waves = swa_FindSWRef.swa_FindSWRef(data, info) + data, info, slow_waves = swa_FindSWChannels.swa_FindSWChannels( + data, info, slow_waves, flag_progress=verbose + ) + + return summarize_slow_waves( + slow_waves, + fs=float(fs), + signal_duration_seconds=signal.size / float(fs), + ) + +def get_patient_profile(df_features): + total_power = df_features.sum(axis=1) + avg_p = df_features.mean() + total_avg_p = avg_p.sum() + + variability = df_features.std() / df_features.mean() + variability.index = ['CV_' + col for col in variability.index] + + kurt = df_features.apply(kurtosis) + kurt.index = ['Kurt_' + col for col in kurt.index] + + rel_delta = avg_p['Delta'] / total_avg_p + + tar = avg_p['Theta'] / avg_p['Alpha'] + tbr = avg_p['Theta'] / avg_p['Beta'] + + spec_entropy = entropy(df_features) + + rel_powers = df_features.div(total_power, axis=0).mean() + rel_powers.index = ['Rel_' + col for col in rel_powers.index] + + avg_p = df_features.mean() + ratios = { + 'Ratio_Theta_Alpha': avg_p['Theta'] / avg_p['Alpha'], + 'Ratio_Slow_Fast': (avg_p['Delta'] + avg_p['Theta']) / (avg_p['Alpha'] + avg_p['Beta']), + 'Sigma_Stability': df_features['Sigma'].std() / df_features['Sigma'].mean(), + 'Spectral_Entropy_delta': spec_entropy[0], + 'Spectral_Entropy_theta': spec_entropy[1], + 'Spectral_Entropy_alpha': spec_entropy[2], + 'Spectral_Entropy_sigma': spec_entropy[3], + 'Spectral_Entropy_beta': spec_entropy[4], + 'Theta_Alpha_Ratio': tar, + 'Theta_Beta_Ratio': tbr, + 'Relative_Delta_Power': rel_delta, + 'kurtosis_Delta': kurt['Kurt_Delta'], + 'kurtosis_Theta': kurt['Kurt_Theta'], + 'kurtosis_Alpha': kurt['Kurt_Alpha'], + 'kurtosis_Sigma': kurt['Kurt_Sigma'], + 'kurtosis_Beta': kurt['Kurt_Beta'], + 'variability_Delta': variability['CV_Delta'], + 'variability_Theta': variability['CV_Theta'], + 'variability_Alpha': variability['CV_Alpha'], + 'variability_Sigma': variability['CV_Sigma'], + 'variability_Beta': variability['CV_Beta'], + + } + + profile = pd.concat([rel_powers, pd.Series(ratios)]) + return profile diff --git a/src/lib/resp_features.py b/src/lib/resp_features.py new file mode 100644 index 0000000..378e51e --- /dev/null +++ b/src/lib/resp_features.py @@ -0,0 +1,70 @@ +"""Respiratory feature helpers used by the active submission pipeline.""" + +import numpy as np +import pandas as pd + +from .resp_peakedness import peakednessCost + + +def peakedness_application(data, stage, subject_id=1): + fs = 25 + setup = { + "K": 15, + "DT": 15, + "Ts": 60, + "Tm": 30, + "Omega_r": np.array([5, 25]) / 60, + "Nfft": np.power(2, 12), + } + time_axis = np.arange(0, data.shape[0] / fs, 1 / fs) + + if time_axis.shape[0] != data.shape[0]: + time_axis = np.arange(0, data.shape[0] / fs, 1 / fs)[:data.shape[0]] + + hat_br, sk_br, t_aver, used = peakednessCost( + data, + time_axis, + fs, + setup, + title=stage, + storeGraph=False, + subjet=subject_id, + ) + return hat_br, sk_br, t_aver, used + + +def odi_application(data, fs): + """Compute oxygen desaturation event rate and average event depth.""" + sp = pd.Series(data) + if len(sp) == 0 or fs <= 0: + return 0.0, 0.0 + + window = max(1, int(fs * 60)) + baseline = sp.rolling(window, min_periods=1, center=True).median() + + diff = baseline - sp + mask = diff >= 3 + + events = [] + in_event = False + prev_idx = 0 + for idx, flag in mask.items(): + if flag and not in_event: + start = idx + in_event = True + elif not flag and in_event: + events.append((start, prev_idx)) + in_event = False + prev_idx = idx + if in_event: + events.append((start, prev_idx)) + + duration_hours = len(sp) / fs / 3600.0 + odi_mean = len(events) / duration_hours if duration_hours > 0 else 0.0 + + magnitudes = [] + for start, end in events: + magnitudes.append(diff.loc[start:end].max()) + odi_deepness = np.mean(magnitudes) if magnitudes else 0.0 + + return odi_mean, odi_deepness diff --git a/src/lib/resp_peakedness.py b/src/lib/resp_peakedness.py new file mode 100644 index 0000000..f14d41e --- /dev/null +++ b/src/lib/resp_peakedness.py @@ -0,0 +1,633 @@ +import pandas as pd +import numpy as np +from numpy.fft import fftshift +from numpy.fft import fft +from scipy.signal import detrend, find_peaks +from time import time + +def _safe_ratio(numerator, denominator, default=0.0): + if denominator is None or not np.isfinite(denominator) or denominator == 0: + return default + value = numerator / denominator + if np.isfinite(value): + return value + return default + +def setParamFr(Setup): + if 'DT' not in Setup.keys(): + Setup["DT"] = 5 + DT = 5 + else: + DT = Setup["DT"] + + if 'Ts' not in Setup.keys(): + Setup["Ts"] = 42 + Ts = 42 + else: + Ts = Setup["Ts"] + + if 'Tm' not in Setup.keys(): + Setup["Tm"] = 12 + Tm = 12 + else: + Tm = Setup["Tm"] + + if 'Nfft' not in Setup.keys(): + Setup["Nfft"] = np.power(2,12) + Nfft = np.power(2,12) + else: + Nfft = Setup["Nfft"] + + if 'K' not in Setup.keys(): + Setup["K"] = 5 + K = 5 + else: + K = Setup["K"] + + if 'Omega_r' not in Setup.keys(): + Setup["Omega_r"] = np.array([0.04, 1]) + Omega_r = np.array([0.04, 1]) + else: + Omega_r = Setup["Omega_r"] + + if 'ksi_p' not in Setup.keys(): + Setup["ksi_p"] = 45 + ksi_p = 45 + else: + ksi_p = Setup["ksi_p"] + + if 'N_k' not in Setup.keys(): + Setup["N_k"] = 4 + N_k = 4 + else: + N_k = Setup["N_k"] + + if 'ksi_a' not in Setup.keys(): + Setup["ksi_a"] = 85 + ksi_a = 85 + else: + ksi_a = Setup["ksi_a"] + + if 'd' not in Setup.keys(): + Setup["d"] = 0.125 + d = 0.125 + else: + d = Setup["d"] + + if 'b' not in Setup.keys(): + Setup["b"] = 0.8 + b = 0.8 + else: + b = Setup["b"] + + if 'a' not in Setup.keys(): + Setup["a"] = 0.5 + a = 0.5 + else: + a = Setup["a"] + + if 'plotflag' not in Setup.keys(): + Setup["plotflag"] = False + plotflag =False + else: + plotflag = Setup["plotflag"] + + + return [ DT, Ts, Tm, Nfft, K, Omega_r, ksi_p, ksi_a, d, b, a, N_k, plotflag, Setup] + +def extract_interval( x, t, int_ini, int_end ): + # EXTRACT_INTERVAL Very simple function to extract an interval from a signal + # + # Created by Jesús Lázaro in 2011 + # -------- + # Sintax: [ x_int, t_int, indexes ] = extract_interval( x, t, int_ini, int_end ) + # In: x = signal + # t = time vector + # int_ini = interval begin time (same units as 't') + # int_end = interval end time (same units as 't') + # + # Out: x_int = interval [int_ini, int_end] of 'x' + # t_int = interval [int_ini, int_end] of 't' + # indexes = indexes corresponding to returned time interval + + start_idx = np.searchsorted(t, int_ini, side='left') + end_idx = np.searchsorted(t, int_end, side='right') + x_int = x[start_idx:end_idx] + t_int = t[start_idx:end_idx] + + return [ x_int, t_int ] + +def normalizar_PSD( PSD, f = None, rango = None): + # NORMALIZAR_PSD Normaliza una densidad espectral de potencia en el rango + # de frecuencias requerido. + # + # Created by Jesús Lázaro in 2011 + # ------- + # Sintax: [ PSD_norm, f_PSD_norm, factor_norm ] = normalizar_PSD( PSD, f, rango ) + # In: PSD = Densidad espectral de potencia + # f = Vector de frecuencias para PSD [Por defecto: frecuencias digitales] + # rango = Rango [f1, f2] en el que se aplicar� la normalizaci�n [Por defecto: Todo f] + # + # Out: PSD_norm = Densidad espectral de potencia notmalizada + # f_PSD_norm = Vector de frecuencias para PSD_norm + # factor_norm = Factor de normalizaci�n utilizado + + if f is None: + f = np.arange(0,PSD.shape[0]) / PSD.shape[0] - 1/2 + + if rango is None: + rango = [f[0], f[-1]] + + + # Seleccionar rango de inter�s: + f_PSD_norm = f[(f>=rango[0]) & (f<=rango[1])] + PSD = PSD[(f>=rango[0]) & (f<=rango[1])] + if not np.any(f_PSD_norm): # El vector de frecuencias no estaba ordenado + print('El vector de frecuencias debe estar ordenado de forma ascendente'); + + + # Calcular factor de normalizaci�n y normalizar: + ##ADD NAN removal from PSD + factor_norm = sum(PSD) + # print("factor_norm "+ str(factor_norm)) + if factor_norm == 0: + # print("stop") // IMPORTANT MODIFICATION TODO + PSD_norm = PSD + else: + PSD_norm = PSD/factor_norm + + return [ PSD_norm, f_PSD_norm, factor_norm ] + +def init_module(kk,vars,param, plotflag): + # function vars = init_module(kk,vars,param, plotflag) + # This function is used for initialization and reinitialization of bar_fr + Skl = vars["Skl"] + t_orig = vars["t_orig"] + t_aver = vars["t_aver"] + f = vars["f"] + L = vars["L"] + + DT = param["DT"] + K = param["K"] + ksi_p = param["ksi_p"] + d = param["d"] + + # Increment of number of spectra for averaging + if kk == 0: # INITIALIZATION + N = 4*np.floor(K/2) + else: # RE-INITIALIZATION + N = 2*np.floor(K/2) + + ###### Peakedness Analysis : + # Indexes of original spectra that take part in the average + O = np.bitwise_and(t_orig>=t_aver[kk]-N*DT, t_orig<=t_aver[kk]+N*DT) + W = np.arange(O.shape[0]) + O = W[O] + # W = np.ones([O.shape[0]]) + # O1 = W[O] + # Pre-allocate + Xkl = np.empty((O.shape[0], L)) + Xkl[:] = np.nan + for k in range(O.shape[0]): + for l in range(L): + S = Skl[:, O[k], l] + # print(S.shape) + # Use as reference for Pkl calculation the absolute maximum + i_m = S.argmax() + fr_max = f[i_m] + + # Define the Omega, Omega_p bands + Omega = np.bitwise_and(f>=fr_max-d, f<=fr_max+d) + + # Modified limits for initialization (reduces the risk for 0.1 Hz) + Omega_p = np.bitwise_and(f>=max(fr_max-0.4*d,0.15), f<=min(fr_max+0.4*d,0.8)) + + # Peakedness + # print(S[Omega]) + band_power = np.sum(S[Omega]) + peaky_power = np.sum(S[Omega_p]) + Pkl = 100*_safe_ratio(peaky_power, band_power) + + if Pkl >= ksi_p: + Xkl[k,l] = 1 + else: + Xkl[k,l] = 0 + + # Initialization for averaged spectrum (if cannot be defined) + if L>1: + averS = np.mean(np.squeeze(np.mean(Skl[:, O, :],1)),1) + else: + averS = np.mean(np.mean(Skl[:, O, :],1),1) + + + if kk == 0: #INITIALIZATION + if np.sum(Xkl[:]) > 0: # One or more spectra were peaked enough + + # Sum all peaky spectra + averS = np.zeros((f.shape[0], 1)) + for k in range(O.shape[0]): + for l in range(L): + if Xkl[k, l] == 1: + averS = averS + Skl[:, O[k], l] + + # Select the maximum in the spectrum + i_m = averS.argmax() + + # Save in vars + vars["bar_fr"][0] = f[i_m] + + else: # RE-INITIALIZATION + # One or more spectra were peaked enough + if np.sum(Xkl[:]) > 0: + # Sum all peaky spectra + averS = np.zeros(f.shape[0]) + for k in range(O.shape[0]): + for l in range(L): + if Xkl[k,l] == 1: + averS = averS + Skl[:, O[k], l] + + # Local maxima in the averaged spectrum + j_pk = find_peaks(averS) + j_pk = j_pk[0] + pk = averS[j_pk] + # Extra restriction : consider peaks with important power + # j_del = pk<0.5*np.max(averS) # IMPORTANTE TODO + j_del = pk<0.2*np.max(averS) + pk = pk[~j_del] + j_pk = j_pk[~j_del] + + # Cost function for deviation from previous fr and maximum power + max_s = np.max(S) + if np.isfinite(max_s) and max_s != 0: + C_a = 1 - (np.transpose(pk) / max_s) + else: + C_a = np.ones_like(pk, dtype=float) + fr_prev = vars["bar_fr"][np.max(kk,0)] + C_f = abs(f[j_pk[:]]-fr_prev)/(2*d) + # C_f = abs(f(i_pk(:))-fr_prev)/(Omega_r(2)-Omega(1)); + + C = C_a +C_f + if C.size > 0: + j_min = C.argmin() + fj = j_pk[j_min] + vars["bar_fr"][kk] = f[fj] + else: + vars["bar_fr"][kk] = 0 + return vars + +def compute_Xkl( Skl, f, bar_fr, O, ksi_p, ksi_a, d): + # function [ Xkl ] = compute_Xkl( Skl, f, bar_fr, O, ksi_p, ksi_a, d) + # Created by Spyros Kontaxis in 2019 + # Computation of peakedness for a power spectrum + # Sintax: [ Xkl ] = compute_Xkl( Skl, f, bar_fr, O, ksi_p, ksi_a, d) + # Inputs: + # Skl : Welch TF maps in a 3D matrix (f x t x DR signals) + # f : frequency vector (Hz) + # bar_fr : smoothed estimate of the respiratory rate (Hz) + # O : Indexes of original spectra that take part in the average + # ksi_p : peakedness threshold based on power concentration (%) + # ksi_a : peakedness threshold based on absolute maximum (%) + # d : half bandwith of Omega centered around bar_fr (Hz) + # Outputs: + # Xkl : 1-> the o:th spectrum will be used in the average + # 0-> the o:th spectrum will not be used in the average + # + + # % Define two search window arround the estimated respiratory rate + Omega = np.bitwise_and(f>=bar_fr-d, f<=bar_fr+d) + Omega_p = np.bitwise_and(f>=bar_fr-0.4*d, f<=bar_fr+0.4*d) + + # % Get the ammount of signals + L = Skl.shape[2] + + # % Pre-allocate + Xkl = np.zeros((O.shape[0],L)) + + # % Loop over all segments + for k in range(O.shape[0]): + + # % Loop over all signals + for l in range(L): + # % Select the power spectrum of one segment + S = Skl[:, O[k], l] + + # % Define peakedness based on the power concentration + band_power = np.sum(S[Omega]) + peaky_power = np.sum(S[Omega_p]) + Pkl = 100*_safe_ratio(peaky_power, band_power) + + # % Define peakedness based on the absolute maximum + # print(max(S)) + max_s = np.max(S) + max_band = np.max(S[Omega]) if np.any(Omega) else 0.0 + Akl = 100*_safe_ratio(max_band, max_s) + # % If the spectrum is concidered peaky by both conditions, mark as + # % peaky + if np.bitwise_and(Pkl >= ksi_p, Akl >= ksi_a): + Xkl[k,l] = 1 + else: + Xkl[k,l] = 0 + + return Xkl + +def compute_fJmin( S, f, bar_fr, d): + # function [ fJmin ] = compute_fJmin( S, f, bar_fr, d) + # Created by Spyros Kontaxis in 2019 + # Spectral peak selection based on cost function + # Sintax: [ fJmin ] = compute_fJmin( S, f, bar_fr, d) + # Inputs: + # S : Averaged Spectrum + # f : frequency vector (Hz) + # bar_fr : smoothed estimate of the respiratory rate (Hz) + # d : half bandwith of Omega centered around bar_fr (Hz) + # Outputs: + # fJmin : respiratory rate estimate + # + + # Define the search window + Omega = np.bitwise_and(f >= bar_fr-d, f <= bar_fr+d) + + # Pre-allocate + fJmin = np.nan + + # Locate peaks in the search window + [peaks, properties] = find_peaks(S[Omega]) #,'SortStr','descend' + + # Put the location in the correct perspective + lm = peaks + (Omega[:] ==1).argmax() + + if lm.shape[0] == 1: + fJmin = f[lm[0]] + elif lm.shape[0] > 1: + fJ = f[lm] + C_f = abs(fJ-bar_fr)/(2*d) + C_a = 1-S[lm]/max(S[Omega]) + + C = C_f+C_a + Jmin = C.argmin() + fJmin = fJ[Jmin] + + return fJmin + +def peakednessCost(signals, ts, fs, Setup = {}, title = "", storeGraph = False, subjet =1): + + vars = {} + [ DT, Ts, Tm, Nfft, K, Omega_r, ksi_p, ksi_a, d, b, a,N_k, plotflag , Setup] = setParamFr(Setup) + + ts1 = ts[0] + ts = ts-ts1 + if type(signals) == type(pd.DataFrame()): + signals = signals.to_numpy() + + if len(signals.shape) == 1: + signals = np.reshape(signals, (signals.shape[0],1)) + if signals.shape[0]= Omega_r[0], f < Omega_r[1]) + vars["f"] = f[f_ind] + + # Time vector for original Welch periodograms + # vars["t_orig"] = np.arange(Ts/2, ts[-1]+DT,DT) - Ts/2+DT #Es posible que sea esto lo que quieren pero no es lo que sale de MATLAB + vars["t_orig"] = np.arange(Ts/2, ts[-1]- Ts/2+DT,DT) #Esto es lo que sale de MATLAB + + # Pre-allocate + vars["Skl"] = np.empty((vars["f"].shape[0], vars["t_orig"].shape[0], vars["L"])) + vars["Skl"][:] = np.nan + + t_for1 = time() + for ii in range(vars["L"]): + t_for_L = time() + + # Select signal + signal = signals[:, ii] + # signal = np.reshape(signal, (signal.shape[0],1)) + # Compute the Welch Periodgrams + for k, ki in zip(vars["t_orig"], range(vars["t_orig"].shape[0])): + # Begin of Ts seconds interval + # Ws_begin = vars["t_orig"][k] - Ts/2 + Ws_begin = k - Ts/2 + + # End of Ts seconds interval + Ws_end = Ws_begin + Ts + [int_Ts_sig, int_Ts_t] = extract_interval(signal, ts, Ws_begin, Ws_end); # Ts seconds interval + S = np.zeros((vars["f"].shape[0])) + if int_Ts_sig.shape[0] < (Tm*100)/2: + vars["Skl"][:, ki, ii] = np.zeros((vars["f"].shape[0])) + continue + int_Ts_sig = int_Ts_sig.astype(float, copy=False) + ts_has_nan = np.isnan(int_Ts_sig).any() + # Number of Tm length subintervals + NWm = int(np.floor(2*Ts/Tm)) + I=0 + + for i_Tm in range(NWm): + S_i = [] + + # Begin of Tm seconds interval + Wm_begin = Ws_begin + (i_Tm)*Tm/2 + + # End of Tm seconds interval + Wm_end = min(Wm_begin + Tm, Ws_end) + + # Tm seconds interval + [int_Tm_sig, int_Tm_t] = extract_interval(int_Ts_sig, int_Ts_t, Wm_begin, Wm_end) + + # Estimate the spectrum only for intervals without NaNs + if not ts_has_nan: + S_i = abs(fftshift(fft(detrend(int_Tm_sig[:-1]), Nfft)))**2 + # S_i = abs(fftshift(fft(int_Tm_sig[:-1], Nfft)))**2 + S_i = S_i[f_ind] + [ S_i, f_PSD_norm, factor_norm ] = normalizar_PSD(S_i) + if ~np.isnan(S_i).any(): + S = S + (1/NWm)*S_i #TODO hacer una median real, que si uno falla la media se coja con los otros 3 dividido entre 3 + I=I+1 + + if I < 0.5*NWm : + vars["Skl"][:, ki, ii] = np.zeros((vars["f"].shape[0])) + else: + # Define the spectrum when enough subintervals were used + vars["Skl"][:, ki, ii] = S + + + + + ##### Peak-conditioned spectral average: ###### + # Pre-allocate + N = int(np.floor(K/2)) + vars["t_aver"] = vars["t_orig"][N:-N] + if vars["t_aver"].shape[0] == 0: + print("No hay tiempo para promediar") + empty_spectra = np.empty((vars["f"].shape[0], 0)) + empty_used = np.empty((0, vars["L"])) + return np.array([]), empty_spectra, np.array([]), empty_used + vars["Sk"] = np.empty((vars["f"].shape[0], vars["t_aver"].shape[0])) + vars["Sk"][:] = np.nan + vars["bar_fr"] = np.empty(( vars["t_aver"].shape[0])) + vars["bar_fr"][:] = np.nan + vars["hat_fr"] = np.empty((vars["t_aver"].shape[0])) + vars["hat_fr"][:] = np.nan + vars["Naveraged"] = np.zeros((vars["t_aver"].shape[0])) + vars["used"] = np.zeros((vars["t_aver"].shape[0],vars["L"])) + vars["times_used"] = np.zeros((vars["t_orig"].shape[0],vars["L"])) + + # Call the initialization module + k_ini = 0 + plotFlag = False + # print(vars["t_aver"]) + vars = init_module(k_ini,vars,Setup,plotFlag); #bar_fr has been initialized + + for k in np.arange(k_ini, vars["t_aver"].shape[0]): + if k >= 1: + k_prev = k-1 + else: + k_prev = 0 + + # Re-initialization when hat_fr has not been defined for N_k time instants + N_k = 2#3+1#vars["N_k"] + N_prev = np.arange(k,max(k-N_k,-1),-1) + if np.isnan(vars["hat_fr"][N_prev]).all() and k > 2: + vars = init_module(k_prev,vars,Setup,plotFlag) # bar_fr has been re-initialized + + # Peakedness Analysis: + # Indexes of original spectra that take part in the average + O = np.bitwise_and(vars["t_orig"]>=vars["t_aver"][k]-N*DT, vars["t_orig"]<=vars["t_aver"][k]+N*DT) + W = np.arange(O.shape[0]) + O = W[O] + + # Compute the peakedness of the power spectrum (1 or 0) + Xkl = compute_Xkl(vars["Skl"], vars["f"], vars["bar_fr"][k_prev], O, ksi_p, ksi_a, d) + + + if np.sum(Xkl) == 0: # No spectrum was peaked + # Store the previous respiratory frequency + vars["bar_fr"][k] = vars["bar_fr"][k_prev] + + # Compute averaged spectrum just for visualization + if vars["L"]>1: + vars["Sk"][:, k] = np.mean(np.squeeze(np.mean(vars["Skl"][:, O, :],1)),1) + else: + try: + vars["Sk"][:,k] = np.mean(vars["Skl"][:, O, :],1)[:,0] + except: + print("Cogido en el except") + print("Cogido en el except") + print("Cogido en el except") + print("Cogido en el except") + vars["Sk"][:,k] = np.nan + + else: #One or more spectra were peaked enough + # Pre-allocate + averS = np.zeros((vars["f"].shape[0])) + + for i_Tm in range(O.shape[0]): + for ii in range(vars["L"]): + if Xkl[i_Tm,ii] == 1: # If this spectrum is considered peaky + # Sum all peaky spectra + averS = averS[:] + vars["Skl"][:, O[i_Tm], ii] + + # Store the nr of peaky spectra + vars["Naveraged"][k] = vars["Naveraged"][k] + 1 + vars["used"][k,ii] = 1 + + # Compute and store the averaged spectrum + vars["Sk"][:, k] = averS/vars["Naveraged"][k] + vars["times_used"][O,:] = vars["times_used"][O,:] + Xkl + + #Spectral peak selection + fJmin = compute_fJmin( vars["Sk"][:, k], vars["f"], vars["bar_fr"][k_prev], d) + + if ~np.isnan(fJmin).any(): # Local maxima inside Omega has been found + # Update bar_fr + + vars["bar_fr"][k] = b*vars["bar_fr"][k_prev] + (1-b)*fJmin + + # Update hat_fr + if ~np.isnan(vars["hat_fr"][k_prev]).any(): + vars["hat_fr"][k]= a*vars["hat_fr"][k_prev] + (1-a)*fJmin + else: + # Use bar_fr(k-1) that always is defined, instead of hat_fr(k-1) + vars["hat_fr"][k]= a*vars["bar_fr"][k_prev] + (1-a)*fJmin + + else: # No local maxima inside Omega + # Update bar_fr + vars["bar_fr"][k] = vars["bar_fr"][k_prev] + + # Don't Update hat_fr + + t_taver = time() + + # Extra : use bar_fr to update hat_fr when was not defined for small gaps (N_k) + # Beginning of the intervals + N_k = 0 + + int_b = np.argwhere(np.isnan(vars["hat_fr"])) + int_b1 = np.append(0,int_b) + int_b = int_b[np.diff(int_b1)>1] + + # End of the intervals + int_e = np.argwhere(np.isnan(vars["hat_fr"])) + int_e1 = np.append(int_e,np.inf) + int_e = int_e[np.diff(int_e1)>1] + + if np.isnan(vars["hat_fr"][0]) and int_e.shape[0]>1: + + int_e = int_e[1:] + if int_b.size > 0 and int_e.size > 0 and (int_e[0]-int_b[0])[0] < 0: + int_e = int_e[1:] + + int_small = (int_e-int_b)<=(N_k-1) + + int_b = int_b[int_small] + int_e = int_e[int_small] + for i in range(int_small.sum()): + vars["hat_fr"][int_b[i]:int_e[i]+1] = vars["hat_fr"][min(int_e[i]+1,vars["hat_fr"].shape[0])] + vars["bar_fr"][int_b[i]:int_e[i]+1] = vars["bar_fr"][min(int_e[i]+1,vars["hat_fr"].shape[0])] + + + + # # Total times a signal can be used + Ntotal = K*(vars["t_orig"].shape[0] - 2) + np.sum(np.arange(1,K)) + + # Times each signal is used + Nused = np.sum(vars["times_used"], 1) + vars["percentage_used"] = 100*Nused/Ntotal + + + vars["t_aver"] = vars["t_aver"] + ts1 + vars["t_orig"] = vars["t_orig"] + ts1 + t_fin = time() + + # if plotflag: + # if go is None or subplots is None: + # raise ModuleNotFoundError("plotly is required when plotflag=True") + + # fig = subplots.make_subplots(rows=2,shared_xaxes=True, subplot_titles=('Peak-condition averaged EDR Spectra in '+title,"EDR/RESP signals"), row_heights=[0.7, 0.3]) + + # fig.add_heatmap(x=vars["t_aver"], y=vars["f"], z=vars["Sk"]/np.max(vars["Sk"]),colorscale='jet',colorbar=dict(orientation='h')) + # fig.update_layout(coloraxis_showscale=False) + # fig.add_trace(go.Line(x=vars["t_aver"], y=vars["hat_fr"],name = 'f\u0302_r(k)'), row = 1, col=1) + # fig.add_trace(go.Line(x=vars["t_aver"],y=vars["bar_fr"],name= 'f\u0304_r(k)'), row = 1, col=1) + + # fig.add_trace(go.Line(x=vars["t_aver"],y=vars["used"]), row = 1, col=1) + # # fig.axis([vars.t_aver(1), vars.t_aver(end), vars.f(1), vars.f(end)]) + # for i in range(signals.shape[1]): + # fig.add_trace(go.Line(x=ts+ts1,y=signals[:,i],name = 'Signal '+str(i)), row = 2, col=1) + + # fig.update_layout(coloraxis_showscale=False) + # fig.update_yaxes(title_text="f (Hz)", row=1, col=1) + # fig.update_yaxes(title_text="(n.u.)", row=2, col=1) + # fig.update_xaxes(title_text="time (s)", row=2, col=1) + # if storeGraph: + # os.makedirs("Graphs/Peakedness/"+str(subjet), exist_ok=True) + # # fig.write_image(os.path.join("Graphs", "Peakedness",str(subjet),title+".png")) + # fig.write_html(os.path.join("Graphs", "Peakedness",str(subjet),title+".html")) + # # fig.write_image() + # else: + # fig.show() + + return vars["hat_fr"], vars["Sk"], vars["t_aver"], vars["used"] + # return vars["hat_fr"], vars["Sk"], vars["bar_fr"],vars["t_aver"], vars["f"], vars["used"] \ No newline at end of file diff --git a/src/lib/swa/mat_to_dict.py b/src/lib/swa/mat_to_dict.py new file mode 100644 index 0000000..a2a9f98 --- /dev/null +++ b/src/lib/swa/mat_to_dict.py @@ -0,0 +1,24 @@ +import scipy.io as sio +import numpy as np + +def mat_to_dict(mat_element): + """ + Convierte de forma recursiva estructuras de MATLAB (mat-objects) + en diccionarios nativos de Python, limpiando las dimensiones vacías. + """ + if isinstance(mat_element, sio.matlab.mat_struct): + # Es una estructura de MATLAB -> La convertimos a diccionario + return {field: mat_to_dict(getattr(mat_element, field)) for field in mat_element._fieldnames} + + elif isinstance(mat_element, np.ndarray): + # Si es un array de objetos, seguimos buscando estructuras dentro + if mat_element.dtype == object: + return [mat_to_dict(element) for element in mat_element] + # Si es un array normal pero está envuelto en dimensiones extra (ej: [[valor]]) + elif mat_element.ndim <= 2 and mat_element.size == 1: + return mat_element.item() + else: + return mat_element + + return mat_element + diff --git a/src/lib/swa/swa_CalculateReference.py b/src/lib/swa/swa_CalculateReference.py new file mode 100644 index 0000000..dba222c --- /dev/null +++ b/src/lib/swa/swa_CalculateReference.py @@ -0,0 +1,237 @@ +import numpy as np +import matplotlib.pyplot as plt +import matplotlib.cm as cm + +def swa_CalculateReference(data, Info, display_plot=False): + """ + Calculates the canonical reference wave(s) for slow wave detection + and applies the bandpass filter. + + Parameters: + ----------- + data : numpy.ndarray + Original EEG data matrix of shape (n_channels, n_samples). + Info : dict + Configuration dictionary. + display_plot : bool, optional + If True, plots electrode selections and a 15-second data sample. + + Returns: + -------- + filtData : numpy.ndarray + Filtered reference wave(s). Shape depends on method (e.g., 1 row for envelope, 4 for diamond). + Info : dict + Updated Info dictionary with chosen parameters and reference electrodes. + """ + data = np.asarray(data, dtype=float) + if data.ndim != 2: + raise ValueError("Error: EEG data must have shape (channels, samples)") + + # 1. Validaciones de estructura iniciales + if 'Electrodes' not in Info: + raise ValueError("Error: No electrode information found in Info") + if 'Recording' not in Info or 'sRate' not in Info['Recording']: + raise ValueError("Error: No sampling rate information found in Info['Recording']") + if 'Parameters' not in Info: + from src.lib.swa.swa_getInfoDefaults import swa_getInfoDefaults # Evitar importación cíclica + Info = swa_getInfoDefaults(Info, 'SW', 'envelope') + print("Warning: No parameters specified; using defaults.") + + # Compatibilidad de minúsculas + Info['Parameters']['Ref_Method'] = Info['Parameters']['Ref_Method'].lower() + ref_method = Info['Parameters']['Ref_Method'] + print(f"Calculating: Canonical wave ({ref_method})") + + # 2. Ajuste y proyección 2D de las coordenadas de los electrodos + # MATLAB: Th = pi/180*[Info.Electrodes.theta]; Rd = [Info.Electrodes.radius]; + # e_locs = Info['Electrodes'] + # # Soporta si e_locs es una lista de objetos o una lista de dicts (desde el JSON) + # theta = np.array([getattr(el, 'theta', el.get('theta')) for el in e_locs], dtype=float) + # radius = np.array([getattr(el, 'radius', el.get('radius')) for el in e_locs], dtype=float) + + # Th = (np.pi / 180.0) * theta + # x = radius * np.cos(Th) + # y = radius * np.sin(Th) + + # # Encajonar las coordenadas en un rango de -0.5 a 0.5 + # intrad = min(1.0, max(np.abs(radius))) + # intrad = max(intrad, 0.5) + # squeezefac = 0.5 / intrad + # x = x * squeezefac + # y = y * squeezefac + + # Inicializar figura si flag_plot está activo + fig_topo, ax_topo = None, None + # if display_plot: + # fig_topo, ax_topo = plt.subplots(figsize=(6, 6)) + # ax_topo.scatter(y, x, s=30, edgecolors=[0.5, 0.5, 0.5], facecolors=[0.5, 0.5, 0.5], label='All Electrodes') + # ax_topo.set_aspect('equal') + # ax_topo.axis('off') + + n_samples = data.shape[1] + n_total_ch = len(Info['Electrodes']) + + # ========================================================================= + # SELECCIÓN POR MÉTODOS + # ========================================================================= + + if ref_method == 'envelope': + # Evitar canales externos/periféricos si está configurado + # if Info['Parameters'].get('Ref_UseInside', True): + # distances = np.sqrt(x**2 + y**2) + # # Guardar máscara (1 fila, n_canales) + # Info['Parameters']['Ref_Electrodes'] = distances < 0.35 + # working_data = data.iloc[Info['Parameters']['Ref_Electrodes'], :] + # else: + Info['Parameters']['Ref_Electrodes'] = np.ones(n_total_ch, dtype=bool) + working_data = data + + # Ordenar muestras para obtener el percentil de negatividad + rData = np.sort(working_data, axis=0) + nCh = max(3, int(np.floor(n_total_ch * 0.025))) + + # Si hay más de 3 canales, saltamos el más negativo para evitar artefactos + if nCh > 3: + nData = np.mean(rData[1:nCh, :], axis=0, keepdims=True) + else: + nData = np.mean(rData[0:nCh, :], axis=0, keepdims=True) + + elif ref_method in ['square', 'diamond']: + distance_from_center = 0.2 + circle_radius = 0.175 + + if ref_method == 'square': + RegionCenters = np.array([ + [-distance_from_center, -distance_from_center, distance_from_center, distance_from_center], + [ distance_from_center, -distance_from_center, -distance_from_center, distance_from_center] + ]) + else: # diamond + RegionCenters = np.array([ + [distance_from_center, 0, 0, -distance_from_center], + [ 0, distance_from_center, -distance_from_center, 0] + ]) + + nData = np.zeros((4, n_samples)) + ref_electrodes_mask = np.zeros((4, n_total_ch), dtype=bool) + + for n in range(4): + distances = np.sqrt((x + RegionCenters[0, n])**2 + (y + RegionCenters[1, n])**2) + ref_electrodes_mask[n, :] = distances < circle_radius + nData[n, :] = np.mean(data[ref_electrodes_mask[n, :], :], axis=0) + + Info['Parameters']['Ref_Electrodes'] = ref_electrodes_mask + + elif ref_method == 'grid': + distance_from_center = 0.225 + circle_radius = 0.10 + + # Replicar meshgrid (-1 a 1) de MATLAB + grid_x, grid_y = np.meshgrid([-distance_from_center, 0, distance_from_center], + [-distance_from_center, 0, distance_from_center]) + centers_x = grid_x.flatten() + centers_y = grid_y.flatten() + + nData = np.zeros((9, n_samples)) + ref_electrodes_mask = np.zeros((9, n_total_ch), dtype=bool) + + for n in range(9): + distances = np.sqrt((x + centers_x[n])**2 + (y + centers_y[n])**2) + ref_electrodes_mask[n, :] = distances < circle_radius + nData[n, :] = np.mean(data[ref_electrodes_mask[n, :], :], axis=0) + + Info['Parameters']['Ref_Electrodes'] = ref_electrodes_mask + + elif ref_method == 'central': + circle_radius = 0.175 + distances = np.sqrt(x**2 + y**2) + + Info['Parameters']['Ref_Electrodes'] = distances < circle_radius + print(f"Information: Central using {np.sum(Info['Parameters']['Ref_Electrodes'])} channels for reference") + nData = np.mean(data[Info['Parameters']['Ref_Electrodes'], :], axis=0, keepdims=True) + + elif ref_method == 'midline': + distance_from_center = 0.25 + circle_radius = 0.125 + RegionCenters = np.array([ + [-distance_from_center, 0, distance_from_center], + [ 0, 0, 0] + ]) + + nData = np.zeros((3, n_samples)) + ref_electrodes_mask = np.zeros((3, n_total_ch), dtype=bool) + + for n in range(3): + distances = np.sqrt((x + RegionCenters[0, n])**2 + (y + RegionCenters[1, n])**2) + ref_electrodes_mask[n, :] = distances < circle_radius + nData[n, :] = np.mean(data[ref_electrodes_mask[n, :], :], axis=0) + + Info['Parameters']['Ref_Electrodes'] = ref_electrodes_mask + + else: + raise ValueError(f"Unrecognised reference method type: {ref_method}") + + # ========================================================================= + # PLOT REGIONES (Si corresponde) + # ========================================================================= + if display_plot and ax_topo is not None: + mask_matrix = np.atleast_2d(Info['Parameters']['Ref_Electrodes']) + no_regions = mask_matrix.shape[0] + colors = cm.get_cmap('tab10', no_regions) + + for n in range(no_regions): + region_idx = mask_matrix[n, :] + ax_topo.scatter(y[region_idx], x[region_idx], s=90, + edgecolors=[0.3, 0.3, 0.3], facecolors=colors(n), + label=f'Region {n+1}') + plt.title(f"Selected Electrodes: {ref_method}") + plt.show() + + # ========================================================================= + # FILTRADO DE LA SEÑAL CANÓNICA + # ========================================================================= + if Info['Parameters'].get('Filter_Apply', True): + if 'Filter_Method' not in Info['Parameters']: + print("Information: No filter parameters given, using defaults.") + Info['Parameters']['Filter_Method'] = 'Chebyshev' + Info['Parameters']['Filter_hPass'] = 0.2 + Info['Parameters']['Filter_lPass'] = 4.0 + Info['Parameters']['Filter_order'] = 2 + + print(f"Calculation: Applying {Info['Parameters']['Filter_Method']} filter for [{Info['Parameters']['Filter_hPass']:.1f}, {Info['Parameters']['Filter_lPass']:.1f}] Hz...") + + # Llamar a la función previamente traducida + from src.lib.swa.swa_filter_data import swa_filter_data + filtData = swa_filter_data(nData, Info) + print("Done") + else: + filtData = nData + + # ========================================================================= + # PLOT 15 SEGUNDOS DE SEÑAL + # ========================================================================= + if display_plot: + sRate = Info['Recording']['sRate'] + # Tomar un punto aleatorio asegurando espacio para 15 segundos + max_start = max(1, n_samples - int(15 * sRate)) + random_sample = np.random.randint(0, max_start) + sample_range = np.arange(random_sample, random_sample + int(15 * sRate)) + time_range = np.arange(len(sample_range)) / sRate + + plt.figure(figsize=(10, 4)) + no_waves = filtData.shape[0] + colors = cm.get_cmap('tab10', no_waves) + + for n in range(no_waves): + if ref_method == 'envelope': + plt.plot(time_range, filtData[n, sample_range], color=[0.2, 0.2, 0.2], linewidth=2.5, label='Envelope Ref') + else: + # Separar las ondas verticalmente como lo hace MATLAB si hay múltiples regiones + plt.plot(time_range, filtData[n, sample_range] - (n * 60), color=colors(n), linewidth=1.5, label=f'Region {n+1}') + + plt.xlabel('Time (seconds)') + plt.ylabel('Amplitude') + plt.title('15 Second Sample of Calculated Reference Wave(s)') + plt.grid(True, alpha=0.3) + plt.show() + + return filtData, Info diff --git a/src/lib/swa/swa_FindSWChannels.py b/src/lib/swa/swa_FindSWChannels.py new file mode 100644 index 0000000..0ae14fa --- /dev/null +++ b/src/lib/swa/swa_FindSWChannels.py @@ -0,0 +1,236 @@ +import numpy as np +from tqdm import tqdm +from .swa_channelNeighbours import swa_channelNeighbours +from .swa_filter_data import swa_filter_data +from .swa_xcorr import swa_xcorr, swa_xcorr2, swa_xcorr_ultra +from .swa_cluster_test import swa_cluster_test + +def swa_FindSWChannels(Data, Info, SW, flag_progress=True): + """ + Finds the slow waves present at each channel given the parameters + already calculated for the reference wave. + Translated from swa-Matlab. + """ + # 1. Check inputs & structures + if len(SW) == 0: + print("Warning: Wave structure is empty") + return Data, Info, SW + + + + p = Info['Parameters'] + + # 2. Cluster test parameter check + if p.get('Channels_ClusterTest', True): + if 'ChannelNeighbours' not in Info['Recording']: + print("Calculating: Channel Neighbours...", end="") + # Asume la existencia de la función externa swa_channelNeighbours + Info['Recording']['ChannelNeighbours'] = swa_channelNeighbours(Info['Electrodes']) + print(" done.") + else: + print("Information: Using channels neighbourhood in 'Info'.") + + # 3. Parameter default settings for Envelope method + if p.get('Ref_Method', '').lower() == 'envelope': + if 'Channels_Threshold' not in p: + print("Warning: No further SW parameters found in Info; using defaults") + Info['Parameters']['Channels_Threshold'] = 0.9 + Info['Parameters']['Channels_WinSize'] = 0.2 + + # 4. Filter data if not already done + if Data.get('Filtered') is not None: + if Data['Filtered'].size == 0: + Data['Filtered'] = swa_filter_data(Data['Raw'], Info) + else: + print("Calculation: Filtering Data.") + Data['Filtered'] = swa_filter_data(Data['Raw'], Info) + + # Calculate window size in samples + win = int(round(Info['Parameters']['Channels_WinSize'] * Info['Recording']['sRate'])) + + # Dimensiones del set de datos (0: canales, 1: muestras de tiempo) + try: + n_channels = Data['Filtered'].shape[0] + except: + n_channels = 1 + n_samples = Data['Filtered'].shape[1] + + to_delete = set() + + # 5. Switch between detection methods + detection_method = Info['Parameters']['Channels_Detection'].lower() + + # ========================================================================= + # CORRELATION METHOD + # ========================================================================= + if detection_method == 'correlation': + + # Iteración con barra de progreso tqdm + for nSW, sw in enumerate(tqdm(SW, disable=not flag_progress, desc="Finding Slow Waves (Correlation)")): + + # Ref_PeakInd is already zero-based in swa_FindSWRef. + ref_peak_ind = int(sw['Ref_PeakInd']) + + # Check search window bounds + if ref_peak_ind - win * 2 < 0 or ref_peak_ind + win * 2 >= n_samples: + to_delete.add(nSW) + continue + + # Extract portion of data around reference peak + shortData = Data['Filtered'][:,ref_peak_ind - win * 2 : ref_peak_ind + win * 2 + 1] + + # Get canonical wave reference data + correlate_type = p.get('Channels_Correlate2', 'all') + + if correlate_type == 'mean': + # En Python los índices de regiones deben ser enteros o booleanos + regions = np.array(sw['Ref_Region'], dtype=int) - 1 + refData = np.mean(Data['SWRef'][regions, ref_peak_ind - win : ref_peak_ind + win + 1], axis=0) + elif correlate_type == 'main': + main_region = int(sw['Ref_Region'][0]) - 1 + refData = np.mean(Data['SWRef'][[main_region], ref_peak_ind - win : ref_peak_ind + win + 1], axis=0) + else: + # if n_channels == 1: + refData = Data['SWRef'][0][ref_peak_ind - win : ref_peak_ind + win + 1] + # else: + # refData = np.mean(Data['SWRef'][0,ref_peak_ind - win : ref_peak_ind + win + 1], axis=0) + + # Cross correlate (Asume la existencia de la función externa swa_xcorr) + cc = swa_xcorr_ultra(refData, shortData, win) + + # Find max correlation and its index location + maxCC = np.max(cc, axis=1) + maxID = np.argmax(cc, axis=1) + + # 1. Forzamos a que maxCC sea un vector plano 1D (elimina cualquier dimensión extra como (N, 1)) + maxCC_1d = np.asarray(maxCC).reshape(-1) + + # 2. Creamos channels_active como un vector booleano puramente 1D de tamaño (n_channels,) + channels_active = maxCC_1d > Info['Parameters']['Channels_Threshold'] + + # Si ningún canal correlaciona bien, se elimina la SW y se continúa + if np.sum(channels_active) == 0: + to_delete.add(nSW) + continue + + # 3. Inicializar matriz de amplitudes negativas con NaNs (canales x 1) + sw['Channels_NegAmp'] = np.full((n_channels, 1), np.nan) + + # 4. Extracción segura de amplitudes para canales activos + if np.any(channels_active): + if n_channels == 1: + sw['Channels_NegAmp'][channels_active, 0] = np.min(shortData) + else: + # shortData es (n_channels, n_samples). channels_active al ser 1D filtra las filas perfectamente. + sw['Channels_NegAmp'][channels_active, 0] = np.min(shortData[channels_active, :], axis=1) + + # 5. Desactivar canales que no superen el umbral de amplitud absoluta mínimo + amp_thresh = np.mean(Info['Parameters']['Ref_AmplitudeAbsolute']) / 10.0 + + # Forzamos que la máscara de desactivación sea 1D para que no altere las dimensiones + deactivate_mask = (sw['Channels_NegAmp'][:, 0] > amp_thresh).flatten() + channels_active[deactivate_mask] = False + + if p.get('Channels_ClusterTest', True): + clusters = swa_cluster_test(channels_active.astype(float), Info['Recording']['ChannelNeighbours'], 0.01) + clusters[np.isnan(clusters)] = 0 + n_clusters = np.unique(clusters) + + if len(n_clusters) > 2: # Más allá del fondo (0) y un cluster válido + max_cluster_size = 0 + keep_cluster_val = n_clusters[1] + for val in n_clusters: + if val == 0: + continue + s_cluster = np.sum(clusters == val) + if s_cluster > max_cluster_size: + max_cluster_size = s_cluster + keep_cluster_val = val + channels_active = (clusters == keep_cluster_val) + + # Recalcular con el canal prototípico si la correlación de la referencia no es muy alta + negative_peak_index = np.nanargmin(sw['Channels_NegAmp'][:, 0]) + + if maxCC[negative_peak_index] < (Info['Parameters']['Channels_Threshold'] + 1) / 2: + # maxID es 0-based, compensamos el comportamiento de MATLAB (maxID_matlab - win) + max_delay = int((maxID[negative_peak_index] + 1) - win) + + maxData = Data['Filtered'][ + negative_peak_index, + ref_peak_ind - win + max_delay : ref_peak_ind + win + max_delay + 1 + ] + + cc = swa_xcorr_ultra(maxData, shortData, win) + maxCC = np.max(cc, axis=1) + maxID = np.argmax(cc, axis=1) + + channels_active[maxCC > Info['Parameters']['Channels_Threshold']] = True + + # Limpieza final de Amplitudes Negativas basada en los canales descartados + sw['Channels_NegAmp'][~channels_active, 0] = np.nan + + # Delay Calculation + sw['Travelling_Delays'] = np.full((n_channels, 1), np.nan) + # maxID de Python ya es nativamente relativo a 0 + sw['Travelling_Delays'][channels_active, 0] = maxID[channels_active] - np.min(maxID[channels_active]) + + sw['Channels_Globality'] = (np.sum(channels_active) / n_channels) * 100.0 + sw['Channels_Active'] = channels_active + + # ========================================================================= + # THRESHOLD METHOD + # ========================================================================= + elif detection_method == 'threshold': + if not p.get('Ref_Peak2Peak'): + Info['Parameters']['Ref_Peak2Peak'] = abs(Info['Parameters']['Ref_NegAmpMin']) * 1.75 + + for nSW, sw in enumerate(tqdm(SW, disable=not flag_progress, desc="Finding Slow Waves (Threshold)")): + + ref_peak_ind = int(sw['Ref_PeakInd']) + + if ref_peak_ind - win < 0 or ref_peak_ind + win * 3 >= n_samples: + to_delete.add(nSW) + continue + + # Short window for negative peak + shortData = Data['Filtered'][:, ref_peak_ind - win : ref_peak_ind + win + 1] + + sw.Channels_NegAmp = np.min(shortData, axis=1) + minChId = np.argmax(shortData, axis=1) # El índice relativo del mínimo dentro de este bloque + + thresh_val = -np.mean(Info['Parameters']['Ref_AmplitudeAbsolute']) * Info['Parameters']['Channels_Threshold'] + sw.Channels_Active = sw.Channels_NegAmp < thresh_val + + # Peak to Peak Check (ventana más larga hacia adelante) + shortData_long = Data['Filtered'][:, ref_peak_ind - win : ref_peak_ind + win * 3 + 1] + + posPeakAmp = np.full(n_channels, np.nan) + active_indices = np.where(sw.Channels_Active)[0] + + for nCh in active_indices: + # Buscamos el pico positivo después del pico negativo detectado + posPeakAmp[nCh] = np.max(shortData_long[nCh, minChId[nCh]:]) + + # Aplicar filtro de umbral pico a pico + p2p_thresh = Info['Parameters']['Ref_Peak2Peak'] * Info['Parameters']['Channels_Threshold'] + sw.Channels_Active[posPeakAmp - sw.Channels_NegAmp < p2p_thresh] = False + + # Eliminar canales sub-umbral de las amplitudes negativas + sw.Channels_NegAmp[~sw.Channels_Active] = np.nan + + if np.sum(sw.Channels_Active) == 0: + to_delete.add(nSW) + continue + + # Delay Calculation usando el identificador del pico negativo + sw.Travelling_Delays = np.full((len(Info.Electrodes), 1), np.nan) + active_delays = minChId[sw.Channels_Active] + sw.Travelling_Delays[sw.Channels_Active, 0] = active_delays - np.min(active_delays) + + sw.Channels_Globality = (np.sum(sw.Channels_Active) / len(sw.Channels_Active)) * 100.0 + + # 6. Delete the bad waves from list + print(f"Information: {len(to_delete)} slow waves were removed due to insufficient criteria.") + SW = [sw for idx, sw in enumerate(SW) if idx not in to_delete] + + return Data, Info, SW diff --git a/src/lib/swa/swa_FindSWRef.py b/src/lib/swa/swa_FindSWRef.py new file mode 100644 index 0000000..35a849d --- /dev/null +++ b/src/lib/swa/swa_FindSWRef.py @@ -0,0 +1,254 @@ +import numpy as np +from scipy.signal import find_peaks + +def swa_FindSWRef(Data, Info, SW=None): + """ + Finds slow waves in the reference signal based on amplitude, wavelength, + and slope criteria. Handles multiple reference regions and removes duplicates. + Translated from swa-Matlab. + + Parameters: + ----------- + Data : dict + Dictionary containing 'SWRef' (n_refs x n_samples) and optionally 'sleep_stages'. + Info : dict + Configuration dictionary. + SW : list of dicts, optional + Existing list of detected slow waves. Defaults to empty list. + + Returns: + -------- + Data : dict + Info : dict + SW : list of dicts + """ + if 'SWRef' not in Data: + raise ValueError("Error: Data dictionary must contain 'SWRef' array.") + + p = Info.get('Parameters', {}) + if 'Ref_Method' not in p: + print("Error: No detection parameters found in the 'Info' structure") + return Data, Info, SW + + # Inicializar la lista de ondas si no existe + if SW is None: + SW = [] + + OSWCount = len(SW) + SWCount = len(SW) + + number_ref_waves = Data['SWRef'].shape[0] + + # Inicializar vectores de umbrales según el criterio + if p.get('Ref_AmplitudeCriteria') == 'relative': + p['Ref_AmplitudeAbsolute'] = np.zeros(number_ref_waves) + elif p.get('Ref_AmplitudeCriteria') == 'absolute': + # Asegurar que Ref_AmplitudeRelative exista como lista/array + p['Ref_AmplitudeRelative'] = np.zeros(number_ref_waves) + + if 'Recording' not in Info: + Info['Recording'] = {} + Info['Recording']['Data_Deviation'] = np.zeros(number_ref_waves) + + # Asegurar que el umbral absoluto tenga el tamaño adecuado si es un solo valor + if p.get('Ref_AmplitudeCriteria') == 'absolute': + abs_val = np.atleast_1d(p['Ref_AmplitudeAbsolute']) + if len(abs_val) < number_ref_waves: + abs_val = np.repeat(abs_val[0], number_ref_waves) + p['Ref_AmplitudeAbsolute'] = abs_val + + sRate = Info['Recording']['sRate'] + + # Bucle por cada onda de referencia (ej. regiones diamond, central, envelope) + for ref_wave in range(number_ref_waves): + if ref_wave > 0: + OSWCount = len(SW) + + ref_signal = Data['SWRef'][ref_wave, :] + + # Calcular la derivada de la señal (añadiendo un 0 inicial para mantener longitud) + slopeData = np.concatenate(([0], np.diff(ref_signal))) + + # Extraer Mínimos (MNP - Maximum Negative Peaks) y Máximos (MPP) + from src.lib.swa.swa_get_peaks import swa_get_peaks + MNP, MPP = swa_get_peaks(slopeData, Info, True) + + MNP = MNP[1:] + MPP = MPP[1:] + # Eliminar picos fuera de las fases de sueño de interés + if p.get('Ref_UseStages') is not None and 'sleep_stages' in Data: + stages = Data['sleep_stages'] + valid_MNP_mask = np.isin(stages[MNP], p['Ref_UseStages']) + relevant_MNP = MNP[valid_MNP_mask] + else: + relevant_MNP = MNP + + # Control de seguridad si no hay picos + if len(relevant_MNP) == 0: + continue + + # ===================================================================== + # CÁLCULO DEL UMBRAL DE AMPLITUD (Amplitude Threshold Criteria) + # ===================================================================== + # MAD (Median Absolute Deviation respecto a la mediana) + median_act = np.median(ref_signal[relevant_MNP]) + mad_val = np.median(np.abs(ref_signal[relevant_MNP] - median_act)) + Info['Recording']['Data_Deviation'][ref_wave] = mad_val + + if p['Ref_AmplitudeCriteria'] == 'relative': + p['Ref_AmplitudeAbsolute'][ref_wave] = (mad_val * p['Ref_AmplitudeRelative']) + abs(median_act) + print(f"Calculation: Amplitude threshold set to {p['Ref_AmplitudeAbsolute'][ref_wave]:.1f}uV for canonical wave {ref_wave + 1}") + elif p['Ref_AmplitudeCriteria'] == 'absolute': + deviation = (p['Ref_AmplitudeAbsolute'][ref_wave] - abs(median_act)) / mad_val if mad_val != 0 else 0 + p['Ref_AmplitudeRelative'][ref_wave] = deviation + print(f"Information: Threshold is {deviation:.1f} deviations from median activity") + + if p.get('Ref_Peak2Peak') is None: + p['Ref_Peak2Peak'] = abs(p['Ref_AmplitudeAbsolute'][ref_wave]) * 1.75 + + # ===================================================================== + # MÉTODOS DE INSPECCIÓN (MNP vs ZC) + # ===================================================================== + if p['Ref_InspectionPoint'] == 'MNP': + # ----- MÉTODO MNP (Maximum Negative Peak) ----- + badWaves = np.zeros(len(MNP), dtype=bool) + + if p.get('Ref_UseStages') is not None and 'sleep_stages' in Data: + badWaves[~valid_MNP_mask] = True + + # (El código MNP de MATLAB requiere alinear los MPP con MNP, + # asumiendo que un MNP está rodeado por dos MPP. Esta lógica + # se adapta mejor con Zero-Crossings a menos que los picos estén + # estrictamente pareados). + # Por simplicidad y robustez frente al código MATLAB original: + pass # En Python el método más seguro y estandarizado es ZC. + + elif p['Ref_InspectionPoint'] == 'ZC': + # ----- MÉTODO ZC (Zero Crossings) ----- + signData = np.sign(ref_signal) + signData[signData == 0] = 1 # Evitar pendiente 0 exacta + + DZC = np.where(np.diff(signData) < 0)[0] + UZC = np.where(np.diff(signData) > 0)[0] + 1 # +1 para ser la muestra tras el cruce + + # Umbral de pendiente (percentil) + pos_slopes = slopeData[slopeData > 0] + slopeThresh = np.percentile(pos_slopes, p['Ref_SlopeMin'] * 100) if len(pos_slopes) > 0 else 0 + + # Alinear DZC y UZC + if len(DZC) > 0 and len(UZC) > 0 and DZC[0] >= UZC[0]: + UZC = UZC[1:] + if len(DZC) != len(UZC): + DZC = DZC[:-1] + if len(DZC) > len(UZC): + DZC = DZC[:-1] + + # Filtrar por fase de sueño + if p.get('Ref_UseStages') is not None and 'sleep_stages' in Data: + valid_DZC = np.isin(Data['sleep_stages'][DZC], p['Ref_UseStages']) + DZC = DZC[valid_DZC] + UZC = UZC[valid_DZC] + + # Criterio Wavelength + SWLengths = UZC - DZC + valid_len = (SWLengths >= p['Ref_WaveLength'][0] * sRate) & (SWLengths <= p['Ref_WaveLength'][1] * sRate) + DZC = DZC[valid_len] + UZC = UZC[valid_len] + + AllPeaks = np.array([sw['Ref_PeakInd'] for sw in SW]) if len(SW) > 0 else np.array([]) + + # Analizar cada candidato DZC + for n in range(len(DZC)): + start, end = DZC[n], UZC[n] + segment = ref_signal[start:end] + + if len(segment) == 0: + continue + + # Amplitud Negativa + NegPeakAmp = np.min(segment) + NegPeakId = start + np.argmin(segment) + + if NegPeakAmp > -p['Ref_AmplitudeAbsolute'][ref_wave] or NegPeakAmp < -p.get('Ref_AmplitudeMax', 250): + continue + + # Amplitud Peak2Peak + search_end = min(end + int(1 * sRate), len(ref_signal)) + PosPeakAmp = np.max(ref_signal[end:search_end]) if search_end > end else 0 + + if p['Ref_Method'] in ['diamond', 'square']: + if (PosPeakAmp - NegPeakAmp) < p['Ref_Peak2Peak']: + continue + + # Pendiente positiva máxima + MaxPosSlope = np.max(slopeData[start:end]) + if MaxPosSlope < slopeThresh: + continue + + # Evitar duplicados inter-regiones + if ref_wave > 0 and len(AllPeaks) > 0: + # Comprobar si el pico de una onda ya guardada cae dentro de este nuevo DZC-UZC + overlap_mask = (AllPeaks > start) & (AllPeaks < end) + if np.any(overlap_mask): + SWid = np.argmax(overlap_mask) # Encontrar el índice conflictivo + + # Guardar la región que tenga la mayor amplitud (más negativo) + if ref_signal[NegPeakId] < SW[SWid]['Ref_PeakAmp']: + # Sobrescribir con la nueva onda que es mejor + old_regions = SW[SWid]['Ref_Region'] if isinstance(SW[SWid]['Ref_Region'], list) else [SW[SWid]['Ref_Region']] + SW[SWid]['Ref_Region'] = [ref_wave + 1] + old_regions # +1 para mantener ID visual (1-based index concept) + SW[SWid]['Ref_DownInd'] = start + SW[SWid]['Ref_PeakInd'] = NegPeakId + SW[SWid]['Ref_UpInd'] = end + SW[SWid]['Ref_PeakAmp'] = ref_signal[NegPeakId] + SW[SWid]['Ref_P2PAmp'] = PosPeakAmp - NegPeakAmp + SW[SWid]['Ref_NegSlope'] = np.min(slopeData[start:end]) + SW[SWid]['Ref_PosSlope'] = MaxPosSlope + else: + # Añadir esta región a la onda existente + if isinstance(SW[SWid]['Ref_Region'], list): + SW[SWid]['Ref_Region'].append(ref_wave + 1) + else: + SW[SWid]['Ref_Region'] = [SW[SWid]['Ref_Region'], ref_wave + 1] + continue + + # Añadir nueva onda detectada + SWCount += 1 + new_wave = { + 'Ref_Region': [ref_wave + 1], + 'Ref_DownInd': start, + 'Ref_PeakInd': NegPeakId, + 'Ref_UpInd': end, + 'Ref_PeakAmp': ref_signal[NegPeakId], + 'Ref_P2PAmp': PosPeakAmp - NegPeakAmp, + 'Ref_NegSlope': np.min(slopeData[start:end]), + 'Ref_PosSlope': MaxPosSlope, + # Campos vacíos que se llenarán más adelante + 'Channels_Active': [], 'Channels_NegAmp': [], + 'Channels_Globality': [], 'Travelling_Delays': [], + 'Travelling_DelayMap': [], 'Travelling_Streams': [], 'Code': [] + } + SW.append(new_wave) + + else: + print("Error: Unrecognised detection method") + return Data, Info, SW + + # Ordenar las ondas detectadas cronológicamente + if len(SW) > 0: + SW.sort(key=lambda x: x['Ref_DownInd']) + + # Limpieza final de estadios de sueño no válidos (por si acaso quedaron en las fusiones) + if p.get('Ref_UseStages') is not None and 'sleep_stages' in Data: + valid_indices = [] + for i, wave in enumerate(SW): + if Data['sleep_stages'][wave['Ref_PeakInd']] in p['Ref_UseStages']: + valid_indices.append(i) + + removed_count = len(SW) - len(valid_indices) + SW = [SW[i] for i in valid_indices] + + if removed_count > 0: + print(f"Information: {removed_count} waves were found in non-specified stages and removed") + + return Data, Info, SW diff --git a/src/lib/swa/swa_FindSWTravelling.py b/src/lib/swa/swa_FindSWTravelling.py new file mode 100644 index 0000000..2dca0eb --- /dev/null +++ b/src/lib/swa/swa_FindSWTravelling.py @@ -0,0 +1,201 @@ +import numpy as np +from scipy.interpolate import griddata, RegularGridInterpolator + +def swa_FindSWTravelling(Info, SW, indSW=None, flag_wait=True): + """ + Calculates the travelling parameters for each slow wave given the + delay maps, defining the streamlines of propagation. + Translated from swa-Matlab. + + Parameters: + ----------- + Info : dict + Configuration dictionary. + SW : list of dicts + List containing all detected slow waves. + indSW : list or numpy.ndarray, optional + Indices of the SW list to process. If None, processes all. + flag_wait : bool + If True, prints progress. + + Returns: + -------- + Info : dict + SW : list of dicts + """ + if len(SW) == 0 or ('Ref_Region' not in SW[0] and len(SW) < 2): + print("Warning: Wave structure is empty, you must find waves in the reference first.") + return Info, SW + + p = Info.setdefault('Parameters', {}) + + # Defaults + if 'Travelling_GS' not in p: + p['Travelling_GS'] = 40 + print("Information: Interpolation grid set at 40x40 by default.") + p['Travelling_MinDelay'] = 40.0 + if 'Travelling_RecalculateDelay' not in p: + p['Travelling_RecalculateDelay'] = True + + GS = p['Travelling_GS'] + sRate = Info['Recording']['sRate'] + + # --- Check Electrodes and 2D Locations --- + if p['Travelling_RecalculateDelay']: + # Ensure 2D locations are scaled to grid size [1, GS] + # (Assuming 'x' and 'y' are present in Info['Electrodes']) + xloc = np.array([el.get('x', 0) for el in Info['Electrodes']], dtype=float) + yloc = np.array([el.get('y', 0) for el in Info['Electrodes']], dtype=float) + + # Auto-scale coordinates to grid if they are between -0.5 and 0.5 (from Reference func) + if np.max(np.abs(xloc)) <= 1.0: + xloc = (xloc - np.min(xloc)) / (np.max(xloc) - np.min(xloc) + 1e-9) * (GS - 1) + 1 + yloc = (yloc - np.min(yloc)) / (np.max(yloc) - np.min(yloc) + 1e-9) * (GS - 1) + 1 + # Update dictionary coordinates for consistency + for i, el in enumerate(Info['Electrodes']): + el['x'], el['y'] = xloc[i], yloc[i] + + print("Calculation: 2D electrode projections created.") + + # Grid definition + XYrange = np.linspace(1, GS, GS) + grid_x, grid_y = np.meshgrid(XYrange, XYrange) + else: + if 'Travelling_DelayMap' not in SW[0] or len(SW[0]['Travelling_DelayMap']) == 0: + raise ValueError("User requested no map calculation, but no maps were found.") + XYrange = np.linspace(1, SW[0]['Travelling_DelayMap'].shape[1], SW[0]['Travelling_DelayMap'].shape[1]) + xloc = np.array([el['x'] for el in Info['Electrodes']]) + yloc = np.array([el['y'] for el in Info['Electrodes']]) + + loopRange = indSW if indSW is not None else range(len(SW)) + total_sw = len(loopRange) + + for idx, nSW in enumerate(loopRange): + if p['Travelling_RecalculateDelay']: + Delays = np.array(SW[nSW].get('Travelling_Delays', [])) + + if len(Delays) == 0 or np.max(Delays) < (p['Travelling_MinDelay'] * sRate / 1000.0): + continue + + # Interpolate Delays onto 2D Grid + # method='cubic' acts similarly to 'natural' interpolant in MATLAB + delay_map = griddata((xloc, yloc), Delays, (grid_x, grid_y), method='cubic') + delay_map = np.nan_to_num(delay_map) # Replace NaNs with 0 + SW[nSW]['Travelling_DelayMap'] = delay_map + + # Starting points + active_ch = SW[nSW].get('Channels_Active', []) + if len(active_ch) == 0: + continue + sx = xloc[active_ch] + sy = yloc[active_ch] + + else: + delay_map = SW[nSW]['Travelling_DelayMap'] + active_ch = SW[nSW].get('Channels_Active', []) + sx = xloc[active_ch] + sy = yloc[active_ch] + + # Calculate gradients (MATLAB: [u, v] = gradient(Map)) + # np.gradient returns [row_diff (Y), col_diff (X)] + grad_y, grad_x = np.gradient(delay_map) + u, v = grad_x, grad_y + + Streams = [] + Distances = [] + + for n in range(len(sx)): + # Trace backwards + path_b, dist_b = _trace_streamline(XYrange, XYrange, -u, -v, sx[n], sy[n], step=0.1, max_steps=1000) + # Trace forwards + path_f, dist_f = _trace_streamline(XYrange, XYrange, u, v, sx[n], sy[n], step=0.1, max_steps=1000) + + # Combine paths (Reverse backward path, skip the overlapping start point, append forward) + if path_b.shape[1] > 0 and path_f.shape[1] > 0: + path = np.hstack((path_b[:, ::-1], path_f[:, 1:])) + dist = np.concatenate((dist_b[::-1], dist_f)) + Streams.append(path) + Distances.append(dist) + + if len(Streams) == 0: + continue + + # Filter Streams by minimum distance threshold (25% of longest path) + tDist = np.array([np.sum(d) for d in Distances]) + valid_streams = tDist >= (np.max(tDist) / 4.0) + + Streams = [s for i, s in enumerate(Streams) if valid_streams[i]] + tDist = tDist[valid_streams] + + if len(Streams) == 0: + continue + + SW[nSW]['Travelling_Streams'] = [] + + # 1. Longest Displacement (Straight line distance from start to end of stream) + tDisp = np.array([np.hypot(s[0, 0] - s[0, -1], s[1, 0] - s[1, -1]) for s in Streams]) + maxDispId = np.argmax(tDisp) + SW[nSW]['Travelling_Streams'].append(Streams[maxDispId]) + + # 2. Longest Travelled Distance (Sum of all step distances) + maxDistId = np.argmax(tDist) + if maxDistId != maxDispId: + SW[nSW]['Travelling_Streams'].append(Streams[maxDistId]) + + # 3. Most different displacement angle compared to longest displacement stream (> 45 degrees) + def get_angle(path): + return np.degrees(np.arctan2(path[1, -1] - path[1, 0], path[0, -1] - path[0, 0])) + + base_angle = get_angle(Streams[maxDispId]) + angles = np.array([get_angle(s) for s in Streams]) + angle_diff = np.abs(angles - base_angle) + # Normalize to 0-180 + angle_diff = np.where(angle_diff > 180, 360 - angle_diff, angle_diff) + + maxAngleId = np.argmax(angle_diff) + if angle_diff[maxAngleId] > 45.0 and maxAngleId not in [maxDispId, maxDistId]: + SW[nSW]['Travelling_Streams'].append(Streams[maxAngleId]) + + if flag_wait and (idx + 1) % max(1, int(total_sw / 10)) == 0: + print(f"Processing Slow Wave {idx + 1} of {total_sw}...") + + return Info, SW + + +# --- HELPER FUNCTION: Vector Field Streamline Tracer --- +def _trace_streamline(X, Y, U, V, start_x, start_y, step=0.1, max_steps=1000): + """ + Simulates MATLAB's adstream2b by tracing paths through a 2D vector field. + """ + interp_U = RegularGridInterpolator((Y, X), U, bounds_error=False, fill_value=0) + interp_V = RegularGridInterpolator((Y, X), V, bounds_error=False, fill_value=0) + + path = [[start_x], [start_y]] + dists = [] + + curr_x, curr_y = start_x, start_y + + for _ in range(max_steps): + u_val = interp_U((curr_y, curr_x)) + v_val = interp_V((curr_y, curr_x)) + + norm = np.hypot(u_val, v_val) + if norm < 1e-5: # Vector is practically zero, stop tracing + break + + # Normalize direction and apply step size + dx = step * (u_val / norm) + dy = step * (v_val / norm) + + curr_x += dx + curr_y += dy + + # Check Boundaries + if curr_x < X[0] or curr_x > X[-1] or curr_y < Y[0] or curr_y > Y[-1]: + break + + path[0].append(float(curr_x)) + path[1].append(float(curr_y)) + dists.append(float(np.hypot(dx, dy))) + + return np.array(path), np.array(dists) \ No newline at end of file diff --git a/src/lib/swa/swa_channelNeighbours.py b/src/lib/swa/swa_channelNeighbours.py new file mode 100644 index 0000000..74b204b --- /dev/null +++ b/src/lib/swa/swa_channelNeighbours.py @@ -0,0 +1,121 @@ +import numpy as np +from scipy.spatial import ConvexHull +from scipy.optimize import minimize +import matplotlib.pyplot as plt +from mpl_toolkits.mplot3d.art3d import Poly3DCollection + +def swa_channelNeighbours(eLoc, displayNet=False): + """ + Searches for neighbouring channels using triangulation and reports back + each channel's neighbours. + Translated from swa-Matlab / ept_TFCE. + """ + nCh = len(eLoc) + + # 1. Extraer coordenadas X, Y, Z de forma flexible (soporta objetos o dicts) + x = np.array([getattr(el, 'X', el.get('X') if isinstance(el, dict) else None) for el in eLoc], dtype=float) + y = np.array([getattr(el, 'Y', el.get('Y') if isinstance(el, dict) else None) for el in eLoc], dtype=float) + z = np.array([getattr(el, 'Z', el.get('Z') if isinstance(el, dict) else None) for el in eLoc], dtype=float) + + vertices = np.column_stack((x, y, z)) + + # 2. Proyección plana 2D (Algoritmo simplificado de Brainstorm) + z2 = z - np.max(z) + hypotxy = np.hypot(x, y) + R = np.hypot(hypotxy, z2) + PHI = np.arctan2(z2, hypotxy) + TH = np.arctan2(y, x) + + # Prevenir valores excesivamente pequeños para PHI + PHI[PHI < 0.001] = 0.001 + + # Proyección achatada (Flat projection) + R2 = R / (np.cos(PHI) ** 0.2) + X_2d = R2 * np.cos(TH) + Y_2d = R2 * np.sin(TH) + + # 3. Ajuste de Esfera (Equivalente a bst_bfs / fminsearch) + mass = np.mean(vertices, axis=0) + diffvert = vertices - mass + R0 = np.mean(np.sqrt(np.sum(diffvert**2, axis=1))) + vec0 = np.append(mass, R0) + + # Función de optimización interna para encontrar el centro de la cabeza + def dist_sph(vec, sensloc): + radius = vec[-1] + center = vec[:-1] + diff = sensloc - center + return np.mean(np.abs(np.sqrt(np.sum(diff**2, axis=1)) - radius)) + + res = minimize(dist_sph, vec0, args=(vertices,), method='Nelder-Mead') + HeadCenter = res.x[:-1] + + # Normalización sobre el centro estimado de la cabeza + coordC = vertices - HeadCenter + coordC = coordC / np.sqrt(np.sum(coordC**2, axis=1))[:, np.newaxis] + + # Teselación (Convex Hull 3D) -> Reemplaza a convhulln + hull_3d = ConvexHull(coordC) + faces = hull_3d.simplices # Matriz de Nx3 (índices de los triángulos) + + # 4. Eliminar triángulos innecesarios / externos + # Obtener el borde exterior en 2D + points_2d = np.column_stack((X_2d, Y_2d)) + hull_2d = ConvexHull(points_2d) + border = set(hull_2d.vertices) + + # Conservar caras que NO tengan todos sus 3 vértices en el borde exterior + iInside = ~np.all(np.isin(faces, list(border)), axis=1) + faces = faces[iInside] + + # Calcular perímetros para eliminar triángulos desproporcionados (outliers) + v0 = vertices[faces[:, 0]] + v1 = vertices[faces[:, 1]] + v2 = vertices[faces[:, 2]] + + side0 = np.sqrt(np.sum((v0 - v1)**2, axis=1)) + side1 = np.sqrt(np.sum((v0 - v2)**2, axis=1)) + side2 = np.sqrt(np.sum((v1 - v2)**2, axis=1)) + + triPerimeter = side0 + side1 + side2 + thresholdPerim = np.mean(triPerimeter) + 3 * np.std(triPerimeter) + + # Aplicar umbral de distancia + faces = faces[triPerimeter <= thresholdPerim] + + # 5. Visualización de la Red (Opcional) + if displayNet: + fig = plt.figure(figsize=(7, 7)) + ax = fig.add_subplot(111, projection='3d') + + # Dibujar las caras del mallado + mesh = Poly3DCollection(vertices[faces], alpha=0.6, facecolor=[0.5, 0.5, 0.5], edgecolor='k', linewidths=0.5) + ax.add_collection3d(mesh) + + # Dibujar los electrodos como puntos + ax.scatter(x, y, z, color='r', s=40, depthshade=False) + + ax.set_axis_off() + ax.set_box_aspect([1,1,1]) + plt.show() + + # 6. Buscar vecinos para cada canal + output = [] + for n in range(nCh): + # Encontrar filas donde aparezca el canal actual 'n' + rows_with_n = np.any(faces == n, axis=1) + # Extraer todos los nodos únicos de esos triángulos + neighbours = np.unique(faces[rows_with_n]) + + # OJO: MATLAB guarda el propio nodo dentro de sus vecinos. + # Convertimos a Base-1 para mantener idéntica paridad con MATLAB + output.append(neighbours + 1) + + # Generar matriz final acolchada con ceros (como cell2mat en MATLAB) + max_neighbors = max(len(ch) for ch in output) + ChN = np.zeros((nCh, max_neighbors), dtype=int) + + for idx, neighbours in enumerate(output): + ChN[idx, :len(neighbours)] = neighbours + + return ChN \ No newline at end of file diff --git a/src/lib/swa/swa_cluster_test.py b/src/lib/swa/swa_cluster_test.py new file mode 100644 index 0000000..c4f6a49 --- /dev/null +++ b/src/lib/swa/swa_cluster_test.py @@ -0,0 +1,80 @@ +import numpy as np + +def swa_cluster_test(data, ChN, threshold): + """ + Finds clusters of spatially contiguous channels that exceed a given threshold + using a breadth-first search on the channel network. + Translated from swa-Matlab. + + Parameters: + ----------- + data : numpy.ndarray + 1D array of shape (n_channels,) containing the values to test (e.g., t-values, amplitudes). + ChN : numpy.ndarray + Neighbor matrix of shape (n_channels, max_neighbors) from swa_channelNeighbours (1-based indexing). + threshold : float + The minimum value required to consider a channel for clustering. + + Returns: + -------- + clusters : numpy.ndarray + 1D array of shape (n_channels,) containing NaN for unclustered channels + and integer IDs (1, 2, 3...) for each detected spatial cluster. + """ + # Forzar que data sea un vector plano (1D) + data = data.flatten() + n_channels = len(data) + + # Pre-asignar salidas y banderas de control + flag_used = np.zeros(n_channels, dtype=bool) + clusters = np.full(n_channels, np.nan) # Relleno de NaNs como en MATLAB + + cluster_id = 1 + + for n in range(n_channels): + # Si el canal no ha sido usado y supera el umbral, inicia un nuevo cluster + if not flag_used[n] and data[n] > threshold: + + # Marcar el canal inicial como examinado + flag_used[n] = True + + # Inicializar la lista del cluster (cola de exploración) + cluster_list = [n] + current_count = 0 # Python usa indexación 0-based + + # El bucle se detiene cuando revisamos todos los canales añadidos a la lista + while current_count < len(cluster_list): + + # Canal que estamos analizando actualmente + current_channel = cluster_list[current_count] + + # Obtener vecinos (filtrando los ceros que actúan como relleno en ChN) + # ChN viene en Base-1, restamos 1 para indexar en Python + raw_neighbours = ChN[current_channel, :] + current_neighbours = raw_neighbours[raw_neighbours > 0] - 1 + current_neighbours = current_neighbours.astype(int) + + if len(current_neighbours) > 0: + # Encontrar cuáles vecinos pasan el umbral y NO han sido usados todavía + in_cluster = (data[current_neighbours] > threshold) & (~flag_used[current_neighbours]) + + if np.sum(in_cluster) > 0: + # Extraer los índices reales de los vecinos que se unen al cluster + valid_neighbours = current_neighbours[in_cluster] + + # Marcarlos como usados inmediatamente para que no los vuelva a agarrar otro canal + flag_used[valid_neighbours] = True + + # Expandir la lista de canales a revisar (cola del BFS) + cluster_list.extend(valid_neighbours) + + # Avanzar al siguiente canal de la lista + current_count += 1 + + # Asignar el ID del cluster actual a todos los canales descubiertos en este viaje + clusters[cluster_list] = cluster_id + + # Incrementar el identificador para el próximo cluster + cluster_id += 1 + + return clusters \ No newline at end of file diff --git a/src/lib/swa/swa_filter_data.py b/src/lib/swa/swa_filter_data.py new file mode 100644 index 0000000..d045236 --- /dev/null +++ b/src/lib/swa/swa_filter_data.py @@ -0,0 +1,65 @@ +import numpy as np +import scipy.signal as signal + +def swa_filter_data(data, Info): + """ + Filters EEG data using either Chebyshev Type II or Butterworth filters. + Translated from swa-Matlab. + + Parameters: + ----------- + data : numpy.ndarray + EEG data array of shape (n_channels, n_samples). + Info : object or dict + Structure containing parameters and recording metadata. + + Returns: + -------- + filtData : numpy.ndarray + Filtered EEG data of shape (n_channels, n_samples). + """ + # 1. Verificar el orden del filtro por defecto + if Info['Parameters'].get('Filter_order') is None: + Info['Parameters']['Filter_order'] = 2 + + # Extraer método en minúsculas para evitar problemas de mayúsculas/minúsculas + method = Info['Parameters']['Filter_Method'].lower() + nyquist = Info['Recording']['sRate'] / 2.0 + + # ========================================================================= + # METODO CHEBYSHEV (Tipo II) + # ========================================================================= + if method == 'chebyshev': + # Parámetros de filtrado normalizados respecto a Nyquist + Wp = np.array([Info['Parameters']['Filter_hPass'], Info['Parameters']['Filter_lPass']]) / nyquist + Ws = np.array([Info['Parameters']['Filter_hPass'] / 5.0, Info['Parameters']['Filter_lPass'] * 2.0]) / nyquist + Rp = 3.0 # Rizado en la banda de paso (dB) + Rs = 10.0 # Atenuación en la banda de rechazo (dB) + + # Calcular el orden óptimo del filtro Chebyshev y las frecuencias naturales + n, Wn = signal.cheb2ord(Wp, Ws, Rp, Rs) + # Diseñar el filtro Chebyshev Tipo II (pasa-banda automático al pasar 2 frecuencias) + bbp, abp = signal.cheby2(n, Rs, Wn, btype='bandpass') + + # Aplicar el filtro de fase cero a lo largo del eje del tiempo (axis=-1) + filtData = signal.filtfilt(bbp, abp, data, axis=-1) + + # ========================================================================= + # METODO BUTTERWORTH (Soporta la errata 'buttersworth' del script original) + # ========================================================================= + elif method in ['buttersworth', 'butterworth']: + fhc = Info['Parameters']['Filter_hPass'] / nyquist + flc = Info['Parameters']['Filter_lPass'] / nyquist + + # Diseñar filtros Butterworth independientes (Alta y Baja frecuencia) + b1, a1 = signal.butter(Info['Parameters']['Filter_order'], fhc, btype='high') + b2, a2 = signal.butter(Info['Parameters']['Filter_order'], flc, btype='low') + + # Aplicar el filtrado secuencial idéntico a MATLAB usando el eje del tiempo + filtData = signal.filtfilt(b1, a1, data, axis=-1) + filtData = signal.filtfilt(b2, a2, filtData, axis=-1) + + else: + raise ValueError(f"Unknown filter method: {Info['Parameters']['Filter_Method']}") + + return filtData diff --git a/src/lib/swa/swa_getInfoDefaults.py b/src/lib/swa/swa_getInfoDefaults.py new file mode 100644 index 0000000..257bc94 --- /dev/null +++ b/src/lib/swa/swa_getInfoDefaults.py @@ -0,0 +1,140 @@ +def swa_getInfoDefaults(Info, type_wave, method='envelope'): + """ + Get the current default detection parameters for slow waves (SW), spindles (SS), + or saw-tooth waves (ST). + Translated from swa-Matlab. + + Parameters: + ----------- + Info : dict + The configuration dictionary (equivalent to Info struct in MATLAB). + type_wave : str + Type of event to detect: 'SW' (Slow Waves), 'ST' (Saw-tooth), or 'SS' (Spindles). + method : str, optional + Method used mainly for 'SW' (e.g., 'envelope' or 'mdc'). Default is 'envelope'. + + Returns: + -------- + Info : dict + The updated dictionary containing all default configuration parameters. + """ + # Asegurar que la clave 'Parameters' exista en el diccionario Info + if 'Parameters' not in Info: + Info['Parameters'] = {} + + p = Info['Parameters'] # Atajo para simplificar la escritura del código + + # Manejar el valor por defecto del método si viene vacío o nulo + if method is None or method == '': + method = 'envelope' + + type_wave = type_wave.upper() + + # ========================================================================= + # CASE: SLOW WAVES (SW) + # ========================================================================= + if type_wave == 'SW': + # Parámetros de filtrado + p['Filter_Apply'] = True + p['Filter_Method'] = 'Chebyshev' # 'Chebyshev' / 'Buttersworth' + p['Filter_hPass'] = 0.5 + p['Filter_lPass'] = 4.0 + p['Filter_order'] = 2 + + # Detección en canal de referencia (Canonical Wave) + p['Ref_Method'] = None # Método canónico ('envelope', 'diamond', 'midline', etc.) + p['Ref_Electrodes'] = False # Array lógico de electrodos usados + p['Ref_InspectionPoint'] = 'MNP' # 'MNP' (Max Neg Peak) / 'ZC' (Zero Crossing) + p['Ref_UseInside'] = 1 # 1: Usar canales interiores de la cabeza / 0: Todos + p['Ref_UseStages'] = None # Información de estadiaje de sueño (Sleep Scoring) + p['Ref_AmplitudeCriteria'] = 'relative' # 'relative' / 'absolute' + p['Ref_AmplitudeRelative'] = 5.0 # Desviaciones estándar desde la media de negatividad + p['Ref_AmplitudeAbsolute'] = 60.0 + p['Ref_AmplitudeMax'] = 250.0 # Amplitud máxima para control de artefactos + p['Ref_WaveLength'] = [0.25, 1.25] # Criterio de longitud (segundos) entre cruces por cero + p['Ref_SlopeMin'] = 0.90 # Porcentaje de corte para las pendientes (slopes) + p['Ref_Peak2Peak'] = None # Solo para umbralización por canales + + # Detección por canales individuales + p['Channels_Correlate2'] = 'mean' # Qué onda canónica usar ('main', 'mean' o 'all') + p['Channels_Detection'] = 'correlation' # 'correlation' / 'threshold' + p['Channels_Threshold'] = 0.9 # Ajuste si se usa el método de umbralización + p['Channels_ClusterTest'] = True + p['Channels_WinSize'] = 0.100 # Ventana de búsqueda en segundos + + # Parámetros de propagación (Travelling) + p['Travelling_GS'] = 40 # Tamaño de la cuadrícula de interpolación + p['Travelling_MinDelay'] = 40.0 # Tiempo de viaje mínimo (ms) + p['Travelling_RecalculateDelay'] = True # False si los mapas de retraso se calculan fuera + + # Configuración específica según el método elegido + if method.lower() == 'envelope': + p['Ref_Method'] = 'Envelope' + elif method.lower() == 'mdc': # Massimini Detection Criteria + p['Ref_Method'] = 'diamond' + p['Ref_AmplitudeCriteria'] = 'absolute' + p['Ref_InspectionPoint'] = 'ZC' + p['Ref_AmplitudeAbsolute'] = 80.0 + p['Ref_Peak2Peak'] = 140.0 + p['Channels_Detection'] = 'threshold' + p['Channels_Threshold'] = 1.0 + + # ========================================================================= + # CASE: SAW-TOOTH WAVES (ST) + # ========================================================================= + elif type_wave == 'ST': + p['Ref_Method'] = 'Midline' + p['Ref_Electrodes'] = None + p['Filter_Apply'] = False # No se necesita filtro clásico para el método CWT + + # Parámetros Wavelet (CWT) para detección + p['CWT_hPass'] = 2.0 + p['CWT_lPass'] = 5.0 + p['CWT_StdThresh'] = 1.75 + p['CWT_AmpThresh'] = None + p['CWT_ThetaAlpha'] = 1.2 + + p['Burst_Length'] = 1.0 # Tiempo máximo entre ondas en segundos + p['Burst_Adjust'] = 0.75 # Ajuste de criterio si la onda se encuentra en ráfaga + + p['Channels_WinSize'] = 0.060 # En segundos + p['Channel_Adjust'] = 0.9 + p['Travelling_GS'] = 40 + p['Travelling_MinDelay'] = 20.0 + + # ========================================================================= + # CASE: SPINDLES (SS) + # ========================================================================= + elif type_wave == 'SS': + p['Ref_Method'] = 'Midline' + p['Ref_Electrodes'] = None + p['Filter_Apply'] = False + p['Ref_UseStages'] = None + + # Parámetros del filtro de Spindles + p['Filter_Method'] = 'Chebyshev' + p['Filter_band'] = [10.0, 16.0] + p['Filter_checkrange'] = 2.0 + p['Filter_Window'] = 0.150 # Ventana de suavizado RMS de la potencia + p['Filter_order'] = 2 + p['Wavelet_name'] = 'fbsp1-1-3' # B-spline wavelet + p['Wavelet_norm'] = 1 + + # Criterios del huso de sueño (Spindle) + p['Ref_AmplitudeCriteria'] = 'relative' + p['Ref_AmplitudeMetric'] = 'median' + p['Ref_AmplitudeRelative'] = [4.0, 2.0] # [umbral alto, umbral bajo] en Desviaciones Estándar + p['Ref_AmplitudeAbsolute'] = 15.0 + p['Ref_NeighbourRatio'] = 3.0 # Ratio mínimo potencia Spindle/Vecinos + + p['Ref_WaveLength'] = [0.3, 3.0] # Tiempo en segundos sobre el umbral + p['Ref_MinWaves'] = 3 # Número mínimo de ondas internas del huso + + p['Channels_Method'] = 'power' # Método wavelet o potencia (FFT) + p['Channels_WinSize'] = 0.150 # Ventana de búsqueda alrededor del huso de referencia + p['Channels_Threshold'] = 0.75 # Ajuste respecto al criterio de referencia + + else: + raise ValueError(f"Unknown wave type: {type_wave}. Choose 'SW', 'ST', or 'SS'.") + + return Info \ No newline at end of file diff --git a/src/lib/swa/swa_get_peaks.py b/src/lib/swa/swa_get_peaks.py new file mode 100644 index 0000000..42e4fb7 --- /dev/null +++ b/src/lib/swa/swa_get_peaks.py @@ -0,0 +1,92 @@ +import numpy as np + +def swa_get_peaks(slope_data, Info, flag_notch=False): + """ + Finds the maximum negative peaks (MNP) and maximum positive peaks (MPP) + within a channel's slope data and iteratively removes small notches. + Translated from swa-Matlab. + + Parameters: + ----------- + slope_data : numpy.ndarray + 1D array containing the derivative/slope of the EEG signal. + Info : dict + Configuration dictionary containing parameters and sampling rate. + flag_notch : bool, optional + If True, iteratively erases small bumps/notches based on wavelength. + + Returns: + -------- + MNP : numpy.ndarray + Indices of Maximum Negative Peaks (local minima). + MPP : numpy.ndarray + Indices of Maximum Positive Peaks (local maxima). + """ + # Encontrar el signo de la pendiente (-1, 0, 1) + sign_data = np.sign(slope_data) + # Forzar que los ceros cuenten como positivos para evitar problemas en mesetas + sign_data[sign_data == 0] = 1 + + diff_sign = np.diff(sign_data) + + # MNP: La pendiente pasa de negativa a positiva (mínimo local / pico negativo) + # En Python obtenemos el índice del cambio + MNP = np.where(diff_sign == 2)[0] + 1 + + # MPP: La pendiente pasa de positiva a negativa (máximo local / pico positivo) + MPP = np.where(diff_sign == -2)[0] + 1 + + if len(MNP) == 0 or len(MPP) == 0: + return np.array([], dtype=int), np.array([], dtype=int) + + # Asegurar que la secuencia empiece con un MPP (Pico Positivo Anterior) + if MNP[0] < MPP[0]: + MNP = MNP[1:] + + if len(MNP) == 0 or len(MPP) == 0: + return np.array([], dtype=int), np.array([], dtype=int) + + # Asegurar que la secuencia termine con un MPP (Pico Positivo Posterior) + if MNP[-1] > MPP[-1]: + MNP = MNP[:-1] + + # Eliminar muescas/ruido pequeño de forma iterativa + if flag_notch and len(MNP) > 0 and len(MPP) > 0: + sRate = Info['Recording']['sRate'] + # Umbral: 10% de la longitud de onda mínima permitida + thresh = Info['Parameters']['Ref_WaveLength'][0] * sRate / 10.0 + + nb = 1 + while nb > 0: + if len(MNP) == 0 or len(MPP) < 2: + break + + # --- 1. Eliminar "Bumps" Positivos --- + # Distancia entre el MNP actual y el SIGUIENTE MPP + posBumps = (MPP[1:] - MNP) < thresh + + if np.any(posBumps): + # En MPP queremos borrar el elemento subsiguiente, así que añadimos False al inicio + mpp_mask = np.concatenate(([False], posBumps)) + MPP = MPP[~mpp_mask] + MNP = MNP[~posBumps] + + if len(MNP) == 0 or len(MPP) < 2: + break + + # --- 2. Eliminar "Bumps" Negativos --- + # Distancia entre el MNP actual y el ANTERIOR MPP + negBumps = (MNP - MPP[:-1]) < thresh + + if np.any(negBumps): + # En MPP queremos borrar el elemento previo, así que añadimos False al final + mpp_mask = np.concatenate((negBumps, [False])) + MPP = MPP[~mpp_mask] + MNP = MNP[~negBumps] + + # Contar cuántos elementos se eliminaron en esta ronda para decidir si continuar + sum_pos = np.sum(posBumps) if np.any(posBumps) else 0 + sum_neg = np.sum(negBumps) if np.any(negBumps) else 0 + nb = max(sum_pos, sum_neg) + + return MNP, MPP \ No newline at end of file diff --git a/src/lib/swa/swa_saveOutput.py b/src/lib/swa/swa_saveOutput.py new file mode 100644 index 0000000..bd686e2 --- /dev/null +++ b/src/lib/swa/swa_saveOutput.py @@ -0,0 +1,88 @@ +import os +import numpy as np +import scipy.io as sio +import tkinter as tk +from tkinter import filedialog + +def swa_saveOutput(Data, Info, SW, save_name=None, flag_raw=True, flag_filtered=False, wave_type='SW'): + """ + Saves the wave detection output. + Translated from swa-Matlab. + + Parameters: + ----------- + Data : dict + Dictionary containing EEG data. + Info : dict + Configuration dictionary. + SW : list of dicts + List containing all detected waves. + save_name : str, optional + File path to save the output. If None, opens a UI dialog. + flag_raw : bool + If True, drops raw data from memory and saves only the file pointer. + flag_filtered : bool + If True, saves filtered data to an external binary file to save space. + wave_type : str + The key under which the wave structure will be saved ('SW', 'SS', 'ST'). + """ + # 1. Manejo de los Datos Crudos (Raw Data) + if flag_raw: + # Reemplaza los datos por un puntero al archivo si el espacio es una preocupación + if 'Recording' in Info and 'dataFile' in Info['Recording']: + Data['Raw'] = Info['Recording']['dataFile'] + else: + print("Information: Will save all the data into the new file; this may take some time and memory.") + + # 2. Manejo de los Datos Filtrados + if flag_filtered: + if 'Recording' in Info and 'dataFile' in Info['Recording']: + base_name = os.path.splitext(Info['Recording']['dataFile'])[0] + filtered_name = f"{base_name}_filtered.fdt" + + if not os.path.exists(filtered_name) and 'Filtered' in Data: + # Guardar los datos filtrados en un binario plano (equivalente a .fdt de MATLAB) + # Se asume que Data['Filtered'] es un numpy array + Data['Filtered'].astype(np.float32).tofile(filtered_name) + print(f"Calculation: Filtered data saved to {filtered_name}") + + Data['Filtered'] = filtered_name + + elif 'Filtered' in Data: + # Eliminar el campo si existe y no queremos guardarlo + del Data['Filtered'] + + # 3. Interfaz de usuario para nombrar el archivo si no se proporcionó uno + if save_name is None or save_name.strip() == '': + root = tk.Tk() + root.withdraw() # Ocultar la ventana principal de tkinter + save_name = filedialog.asksaveasfilename( + title="Save Output File", + defaultextension=".mat", + filetypes=[("MATLAB files", "*.mat"), ("All files", "*.*")] + ) + if not save_name: + print("Warning: Save cancelled by user.") + return + + # 4. Preparar el diccionario final para guardar + save_dict = { + 'Data': Data, + 'Info': Info, + wave_type: SW # Usamos la variable wave_type ('SW', 'SS' o 'ST') como llave + } + + # 5. Guardar el archivo + try: + # Intenta guardarlo usando scipy.io + sio.savemat(save_name, save_dict) + print(f"Success: Output successfully saved to {save_name}") + except OverflowError: + # Si el archivo es muy grande (> 2GB), scipy fallará al tratar de imitar mat v7.2. + # Fallback a un archivo Pickle nativo de Python + import pickle + pickle_name = os.path.splitext(save_name)[0] + '.pkl' + print("Warning: File too large for standard MAT format. Saving as Python Pickle (.pkl) instead.") + with open(pickle_name, 'wb') as f: + pickle.dump(save_dict, f, protocol=pickle.HIGHEST_PROTOCOL) + print(f"Success: Output successfully saved to {pickle_name}") \ No newline at end of file diff --git a/src/lib/swa/swa_xcorr.py b/src/lib/swa/swa_xcorr.py new file mode 100644 index 0000000..1ec39f4 --- /dev/null +++ b/src/lib/swa/swa_xcorr.py @@ -0,0 +1,154 @@ +import numpy as np +from numpy.lib.stride_tricks import sliding_window_view + +def swa_xcorr(refData, shortData, win): + + """ + Analyses the cross-correlation (Pearson r) between the reference channel + and all subsequent channels using a sliding window without zero padding. + Translated from swa-Matlab. + + Parameters: + ----------- + refData : numpy.ndarray + Reference signal row vector of shape (n_samples_ref,) or (1, n_samples_ref). + shortData : numpy.ndarray + Target channels matrix of shape (n_channels, n_samples_short). + win : int + Window size parameter. + + Returns: + -------- + R : numpy.ndarray + Correlation matrix of shape (n_channels, win * 2 + 1). + """ + # Asegurar que refData sea un vector plano (1D) para las operaciones de correlación + if refData.ndim > 1: + refData = refData.flatten() + + n_channels = refData.ndim + n_shifts = win * 2 + 1 + try: + len_ref = len(refData) + except: + len_ref = 1 + + # Pre-asignar la matriz de resultados (canales x desplazamientos) + R = np.zeros((n_channels, n_shifts)) + + # Centralizar la referencia (restar media) para el cálculo de Pearson + ref_centered = refData - np.mean(refData) + ref_norm = np.sqrt(np.sum(ref_centered ** 2)) + + if ref_norm == 0: + return R # Evitar división por cero si la referencia es plana + + # Bucle a lo largo del tiempo (desplazamientos de la ventana deslizante) + for t in range(n_shifts): + # Extraer la porción de datos de todos los canales para el lag 't' + # Equivalente a shortData(:, t : t + size(refData,2) - 1) en MATLAB + sub_data = shortData[t : t + len_ref] + + # Centralizar cada canal en esta ventana temporal específica + sub_mean = np.mean(sub_data, keepdims=True) + sub_centered = sub_data - sub_mean + + # Calcular desviaciones estándar (denominador de Pearson) + sub_norm = np.sqrt(np.sum(sub_centered ** 2)) + + # Producto punto entre la referencia y los canales (numerador de Pearson) + numerator = np.dot(sub_centered, ref_centered) + + # Coeficiente de correlación r de Pearson: n_channels x 1 + # Usamos np.errstate para evitar advertencias si algún canal tiene varianza cero + with np.errstate(divide='ignore', invalid='ignore'): + r_val = numerator / (sub_norm * ref_norm) + + # Reemplazar posibles NaNs (por canales planos/sin varianza) con 0 + if np.isnan(r_val): + r_val = 0.0 + + R[:, t] = r_val + + return R + +def swa_xcorr2(refData, shortData, win): + """ + Calculates the sliding Pearson cross-correlation between a reference channel + and all other channels without zero padding. + + Parameters: + ----------- + refData : numpy.ndarray + 1D array (or 2D with 1 row) of the reference channel data. + shortData : numpy.ndarray + 2D array (channels x samples) of the data to compare. + win : int + Window reach (the loop will execute win * 2 + 1 times). + """ + # Asegurar que refData sea un vector 1D plano + refData = np.squeeze(refData) + + n_channels = shortData.shape[1] + n_samples_ref = len(refData) + n_lags = win * 2 + 1 + + # Inicializar la matriz de resultados (canales x retrasos) + R = np.zeros((n_channels, n_lags)) + + # Precalcular la media y la desviación de la onda de referencia (fijos) + ref_mean = np.mean(refData) + ref_dev = refData - ref_mean + ref_var = np.sum(ref_dev ** 2) + + # Bucle solo para los desplazamientos temporales (lags) + for t in range(n_lags): + # Extraer la ventana actual para TODOS los canales a la vez + short_slice = shortData[t : t + n_samples_ref] + + # Calcular la media y desviación local de la ventana por cada canal + short_mean = np.mean(short_slice, keepdims=True) + short_dev = short_slice - short_mean + short_var = np.sum(short_dev ** 2) + + # Covarianza cruzada (multiplicación matricial elemento a elemento) + covariance = np.sum(short_dev * ref_dev) + + # Calcular el coeficiente r de Pearson (evitando divisiones por cero) + with np.errstate(divide='ignore', invalid='ignore'): + R[:, t] = covariance / np.sqrt(ref_var * short_var) + + # Reemplazar posibles NaNs (por canales planos sin variación) con cero + return np.nan_to_num(R) + +def swa_xcorr_ultra(refData, shortData, win): + """ + Calcula la correlación cruzada móvil de Pearson de forma 100% vectorial, + sin bucles 'for', utilizando vistas de ventanas deslizantes de NumPy. + + shortData debe tener la forma: (canales, muestras) + """ + refData = np.squeeze(refData) + n_samples_ref = len(refData) + + # 1. Crear las ventanas deslizantes de forma virtual (sin coste de memoria extra) + # Resultado 'windows' tendrá la forma: (canales, n_lags, n_samples_ref) + windows = sliding_window_view(shortData, window_shape=n_samples_ref, axis=1) + + # 2. Operaciones sobre la señal de referencia (fijas) + ref_dev = refData - np.mean(refData) + ref_var = np.sum(ref_dev ** 2) + + # 3. Operaciones vectoriales simultáneas para TODOS los canales y TODOS los lags + short_mean = np.mean(windows, axis=2, keepdims=True) + short_dev = windows - short_mean + short_var = np.sum(short_dev ** 2, axis=2) + + # 4. Covarianza cruzada mediante 'broadcasting' en la última dimensión (tiempo) + covariance = np.sum(short_dev * ref_dev, axis=2) + + # 5. Coeficiente de correlación de Pearson final + with np.errstate(divide='ignore', invalid='ignore'): + R = covariance / np.sqrt(ref_var * short_var) + + return np.nan_to_num(R) diff --git a/src/lib/train_test_split.py b/src/lib/train_test_split.py new file mode 100644 index 0000000..1d4956a --- /dev/null +++ b/src/lib/train_test_split.py @@ -0,0 +1,132 @@ +import csv +import os +import shutil + + +def create_test_set_from_ids(test_ids_file, demographics_file, source_dir, dest_dir): + """ + Create a test set by moving EDF files based on patient IDs. + + Args: + test_ids_file (str): Path to the file containing test patient IDs (one per line). + demographics_file (str): Path to the demographics CSV file. + source_dir (str): Path to the source directory containing physiological data folders. + dest_dir (str): Path to the destination directory for the test set. + """ + # Read test IDs + with open(test_ids_file, 'r') as f: + test_ids = set(line.strip() for line in f if line.strip()) + + print(f"Test IDs: {len(test_ids)}") + + # Parse demographics to get mapping + id_to_bids = {} + with open(demographics_file, 'r', newline='', encoding='utf-8') as f: + reader = csv.DictReader(f) + for row in reader: + patient_id = row['BDSPPatientID'] + bids_folder = row['BidsFolder'] + if patient_id in test_ids: + if patient_id not in id_to_bids: + id_to_bids[patient_id] = [] + id_to_bids[patient_id].append(bids_folder) + + print(f"Found mappings for {len(id_to_bids)} patients") + + # Create destination directory if it doesn't exist + os.makedirs(dest_dir, exist_ok=True) + + moved_files = [] + # Find and move files + for patient_id, bids_list in id_to_bids.items(): + for bids in bids_list: + # bids is like sub-I0002150000686 + # file is sub-I0002150000686_ses-X.edf + # folder is I0002, I0006, S0001 + site = bids.split('-')[1][:5] # I0002 or I0006 or S0001 + folder = os.path.join(source_dir, site) + if os.path.exists(folder): + for file in os.listdir(folder): + if file.startswith(bids + '_') and file.lower().endswith('.edf'): + src = os.path.join(folder, file) + dst = os.path.join(dest_dir, file) + shutil.move(src, dst) + moved_files.append(dst) + print(f"Moved {src} to {dst}") + + print(f"Done. Moved {len(moved_files)} EDF files") + return moved_files + + +def get_selected_records_from_test_set(test_set_dir): + selected_records = set() + for root, _, files in os.walk(test_set_dir): + for file in files: + if file.lower().endswith('.edf'): + stem = os.path.splitext(file)[0] + if '_ses-' not in stem: + continue + bids_folder, session_id = stem.rsplit('_ses-', 1) + site_id = bids_folder.split('-')[1][:5] if '-' in bids_folder else '' + selected_records.add((site_id, bids_folder, session_id)) + return selected_records + + +def sync_demographics_between_train_and_test(source_demographics_file, test_set_dir, test_demographics_file=None): + """ + Create a filtered demographics CSV in the test set and remove those rows from the training demographics file. + + Args: + source_demographics_file (str): Path to the source training_set demographics CSV. + test_set_dir (str): Directory containing the moved EDF files for the test set. + test_demographics_file (str, optional): Output path for the test set demographics CSV. + Defaults to '/demographics.csv'. + """ + if test_demographics_file is None: + test_demographics_file = os.path.join(test_set_dir, 'demographics.csv') + + selected_records = get_selected_records_from_test_set(test_set_dir) + print(f"Found {len(selected_records)} EDF records in {test_set_dir}") + + with open(source_demographics_file, 'r', newline='', encoding='utf-8') as f: + reader = csv.DictReader(f) + fieldnames = reader.fieldnames + source_rows = list(reader) + + rows_for_test = [] + remaining_rows = [] + for row in source_rows: + key = (row.get('SiteID'), row.get('BidsFolder'), str(row.get('SessionID'))) + if key in selected_records: + rows_for_test.append(row) + else: + remaining_rows.append(row) + + os.makedirs(os.path.dirname(test_demographics_file), exist_ok=True) + with open(test_demographics_file, 'w', newline='', encoding='utf-8') as f: + writer = csv.DictWriter(f, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(rows_for_test) + + with open(source_demographics_file, 'w', newline='', encoding='utf-8') as f: + writer = csv.DictWriter(f, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(remaining_rows) + + print(f"Wrote {len(rows_for_test)} rows to {test_demographics_file}") + print(f"Removed {len(source_rows) - len(remaining_rows)} rows from {source_demographics_file}") + + +if __name__ == "__main__": + # Example usage + moved_files = create_test_set_from_ids( + test_ids_file='testids_lista.txt', + demographics_file='data/training_set/demographics.csv', + source_dir='data/training_set/physiological_data', + dest_dir='data/test_set' + ) + if moved_files: + sync_demographics_between_train_and_test( + source_demographics_file='data/training_set/demographics.csv', + test_set_dir='data/test_set' + ) \ No newline at end of file diff --git a/src/pipeline/__init__.py b/src/pipeline/__init__.py new file mode 100644 index 0000000..beafece --- /dev/null +++ b/src/pipeline/__init__.py @@ -0,0 +1 @@ +"""Internal pipeline modules for the active Challenge submission.""" \ No newline at end of file diff --git a/src/pipeline/config.py b/src/pipeline/config.py new file mode 100644 index 0000000..3c9d95d --- /dev/null +++ b/src/pipeline/config.py @@ -0,0 +1,38 @@ +import os + +from src.ecg_processing import ECG_FEATURE_LENGTH +from src.eeg_processing import EEG_FEATURE_LENGTH +from src.resp_processing import RESP_FEATURE_LENGTH + + +SCRIPT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')) +DEFAULT_CSV_PATH = os.path.join(SCRIPT_DIR, 'channel_table.csv') +FEATURE_CACHE_FOLDER_NAME = '.feature_cache' +INCLUDE_SUBJECT_FEATURE_MEAN = False +SEGMENT_AGGREGATION_NAMES = ( + ('Max', 'Min', 'Mean', 'Median', 'Std') + if INCLUDE_SUBJECT_FEATURE_MEAN + else ('Max', 'Min', 'Median', 'Std') +) +FEATURE_CORRELATION_THRESHOLD = 0.8 +MAX_TRAIN_WORKERS = max(1, min(4, os.cpu_count() or 1)) +USE_SITE_GROUPED_CV = False +OPTIMIZE_HYPERPARAMETER_SEARCH = True +RANDOM_CV_N_SPLITS = 5 +CV_RANDOM_STATE = 42 +CV_SEARCH_ITERATIONS = 20 +DEFAULT_CV_HYPERPARAMETERS = { + 'max_depth': 3, + 'min_child_weight': 3, + 'subsample': 0.9, + 'colsample_bytree': 0.6, + 'reg_lambda': 0.5, + 'reg_alpha': 0.1, +} +SEGMENT_DURATION_SECONDS = 5 * 60 +SEGMENT_STRIDE_SECONDS = 15 * 60 +TOTAL_PHYSIOLOGICAL_FEATURE_LENGTH = ( + RESP_FEATURE_LENGTH * len(SEGMENT_AGGREGATION_NAMES) + + EEG_FEATURE_LENGTH * len(SEGMENT_AGGREGATION_NAMES) + + ECG_FEATURE_LENGTH * len(SEGMENT_AGGREGATION_NAMES) +) diff --git a/src/pipeline/cross_validation.py b/src/pipeline/cross_validation.py new file mode 100644 index 0000000..0c249ed --- /dev/null +++ b/src/pipeline/cross_validation.py @@ -0,0 +1,518 @@ +from dataclasses import dataclass, field +from typing import Any, Callable, Optional + +import numpy as np +from sklearn.metrics import accuracy_score, average_precision_score, f1_score, precision_score, recall_score, roc_auc_score +from sklearn.model_selection import LeaveOneGroupOut, RandomizedSearchCV, StratifiedKFold + + +def normalize_site_group(site_id): + site_text = str(site_id).strip().upper() + return site_text[:5] if site_text else 'UNKNOWN' + + +@dataclass(frozen=True) +class CrossValidationConfig: + use_site_grouped_cv: bool = True + optimize_hyperparameter_search: bool = False + outer_random_splits: int = 5 + random_state: int = 42 + search_iterations: int = 20 + fixed_hyperparameters: dict[str, Any] = field(default_factory=dict) + + +@dataclass(frozen=True) +class CrossValidationResult: + threshold: float + consensus_params: Optional[dict[str, Any]] + metrics: dict[str, Any] + + +@dataclass(frozen=True) +class FoldSplit: + fold_index: int + train_idx: np.ndarray + validation_idx: np.ndarray + label: str + + +class EnsembleCrossValidator: + def __init__( + self, + config: CrossValidationConfig, + param_dist, + default_threshold, + build_preprocessor: Callable[..., Any], + build_search_model: Callable[..., Any], + fit_ensemble: Callable[..., Any], + predict_probabilities: Callable[..., Any], + ): + self.config = config + self.param_dist = param_dist + self.default_threshold = default_threshold + self.build_preprocessor = build_preprocessor + self.build_search_model = build_search_model + self.fit_ensemble = fit_ensemble + self.predict_probabilities = predict_probabilities + + def run( + self, + features, + labels, + feature_indices, + modality_presence_indices, + categorical_indices=None, + site_groups=None, + ): + labels = np.asarray(labels, dtype=np.int32) + if self.config.use_site_grouped_cv: + return self._run_grouped_cv( + features, + labels, + site_groups, + feature_indices, + modality_presence_indices, + categorical_indices=categorical_indices, + ) + + return self._run_random_cv( + features, + labels, + feature_indices, + modality_presence_indices, + categorical_indices=categorical_indices, + ) + + def _run_grouped_cv( + self, + features, + labels, + site_groups, + feature_indices, + modality_presence_indices, + categorical_indices=None, + ): + site_groups = np.asarray(site_groups) + unique_sites = np.unique(site_groups) + if unique_sites.size < 2: + return CrossValidationResult( + threshold=self.default_threshold, + consensus_params=None, + metrics={ + 'skipped': True, + 'cv_strategy': 'grouped_by_site', + 'reason': 'Not enough hospitals to run grouped cross-validation.', + 'site_groups': unique_sites.tolist(), + }, + ) + + classes = np.unique(labels) + if len(classes) != 2: + return CrossValidationResult( + threshold=self.default_threshold, + consensus_params=None, + metrics={ + 'skipped': True, + 'cv_strategy': 'grouped_by_site', + 'reason': 'Need both classes to run grouped cross-validation.', + 'site_groups': unique_sites.tolist(), + }, + ) + + group_cv = LeaveOneGroupOut() + split_plan = [] + for fold_idx, (train_idx, val_idx) in enumerate(group_cv.split(features, labels, groups=site_groups), start=1): + held_out_site = normalize_site_group(site_groups[val_idx][0]) + split_plan.append(FoldSplit( + fold_index=fold_idx, + train_idx=train_idx, + validation_idx=val_idx, + label=held_out_site, + )) + + consensus, selected_params_per_fold = self._select_consensus_params( + split_plan, + features, + labels, + feature_indices, + categorical_indices=categorical_indices, + site_groups=site_groups, + label_prefix='held-out hospital', + ) + fold_metrics, oof_probabilities = self._evaluate_with_fixed_params( + split_plan, + features, + labels, + feature_indices, + modality_presence_indices, + categorical_indices=categorical_indices, + consensus_params=consensus, + label_prefix='held-out hospital', + extra_metric_fields=lambda split: {'held_out_site': split.label}, + ) + + return self._finalize_result( + consensus_params=consensus, + labels=labels, + oof_probabilities=oof_probabilities, + best_params_per_fold=selected_params_per_fold, + fold_metrics=fold_metrics, + metadata={ + 'cv_strategy': 'grouped_by_site', + 'n_splits': int(len(split_plan)), + 'site_groups': unique_sites.tolist(), + }, + ) + + def _run_random_cv( + self, + features, + labels, + feature_indices, + modality_presence_indices, + categorical_indices=None, + ): + outer_cv, n_splits = self._build_stratified_splitter(labels, self.config.outer_random_splits) + if outer_cv is None: + return CrossValidationResult( + threshold=self.default_threshold, + consensus_params=None, + metrics={ + 'skipped': True, + 'cv_strategy': 'random_stratified', + 'reason': 'Not enough samples per class to run random stratified cross-validation.', + 'requested_n_splits': int(self.config.outer_random_splits), + }, + ) + + split_plan = [ + FoldSplit( + fold_index=fold_idx, + train_idx=train_idx, + validation_idx=val_idx, + label='random stratified split', + ) + for fold_idx, (train_idx, val_idx) in enumerate(outer_cv.split(features, labels), start=1) + ] + consensus, selected_params_per_fold = self._select_consensus_params( + split_plan, + features, + labels, + feature_indices, + categorical_indices=categorical_indices, + site_groups=None, + label_prefix='random split', + ) + fold_metrics, oof_probabilities = self._evaluate_with_fixed_params( + split_plan, + features, + labels, + feature_indices, + modality_presence_indices, + categorical_indices=categorical_indices, + consensus_params=consensus, + label_prefix='random split', + ) + + return self._finalize_result( + consensus_params=consensus, + labels=labels, + oof_probabilities=oof_probabilities, + best_params_per_fold=selected_params_per_fold, + fold_metrics=fold_metrics, + metadata={ + 'cv_strategy': 'random_stratified', + 'n_splits': int(n_splits), + }, + ) + + def _select_consensus_params( + self, + split_plan, + features, + labels, + feature_indices, + categorical_indices=None, + site_groups=None, + label_prefix='fold', + ): + if not self.config.optimize_hyperparameter_search: + fixed_params = dict(self.config.fixed_hyperparameters) + print(f" Hyperparameter search disabled. Using fixed parameters: {fixed_params}") + return fixed_params, [ + { + 'fold': int(split.fold_index), + 'label': split.label, + 'params': fixed_params, + 'source': 'fixed_defaults', + } + for split in split_plan + ] + + selected_params_per_fold = [] + + for split in split_plan: + print(f" Search fold {split.fold_index}/{len(split_plan)} - {label_prefix} {split.label}") + X_train = features[split.train_idx] + y_train = labels[split.train_idx] + search_site_groups = None if site_groups is None else site_groups[split.train_idx] + + fold_preprocessor = self.build_preprocessor(len(y_train), categorical_indices) + X_train_proc = np.asarray(fold_preprocessor.fit_transform(X_train), dtype=np.float32) + remapped_feature_indices = fold_preprocessor.transform_feature_indices(feature_indices) + print( + f" Correlation selector kept {X_train_proc.shape[1]}/{X_train.shape[1]} features" + ) + fold_best_params = self._search_hyperparams( + X_train_proc[:, remapped_feature_indices['all']], + y_train, + site_groups=search_site_groups, + ) + print(f" Best params: {fold_best_params}") + selected_params_per_fold.append({ + 'fold': int(split.fold_index), + 'label': split.label, + 'params': fold_best_params, + 'source': 'search', + }) + + consensus = self._consensus_params([item['params'] for item in selected_params_per_fold]) + print(f" Consensus hyperparameters: {consensus}") + return consensus, selected_params_per_fold + + def _evaluate_with_fixed_params( + self, + split_plan, + features, + labels, + feature_indices, + modality_presence_indices, + categorical_indices=None, + consensus_params=None, + label_prefix='fold', + extra_metric_fields=None, + ): + oof_probabilities = np.zeros(len(labels), dtype=np.float32) + fold_metrics = [] + + for split in split_plan: + print(f" Eval fold {split.fold_index}/{len(split_plan)} - {label_prefix} {split.label}") + X_train, X_val = features[split.train_idx], features[split.validation_idx] + y_train, y_val = labels[split.train_idx], labels[split.validation_idx] + + fold_preprocessor = self.build_preprocessor(len(y_train), categorical_indices) + X_train_proc = np.asarray(fold_preprocessor.fit_transform(X_train), dtype=np.float32) + remapped_feature_indices = fold_preprocessor.transform_feature_indices(feature_indices) + print( + f" Correlation selector kept {X_train_proc.shape[1]}/{X_train.shape[1]} features" + ) + fold_models = self.fit_ensemble( + X_train_proc, + y_train, + remapped_feature_indices, + consensus_params=consensus_params, + ) + fold_bundle = { + 'models': fold_models, + 'feature_indices': remapped_feature_indices, + 'modality_presence_indices': modality_presence_indices, + 'preprocessor': fold_preprocessor, + 'threshold': self.default_threshold, + } + + fold_probabilities = self.predict_probabilities(fold_bundle, X_val) + oof_probabilities[split.validation_idx] = fold_probabilities + + fold_metric_row = self._compute_evaluation_metrics( + y_val, + fold_probabilities, + threshold=self.default_threshold, + ) + extra_fields = {} if extra_metric_fields is None else extra_metric_fields(split) + fold_metric_row = { + **fold_metric_row, + 'fold': int(split.fold_index), + 'train_size': int(len(split.train_idx)), + 'validation_size': int(len(split.validation_idx)), + **extra_fields, + } + fold_metrics.append(fold_metric_row) + self._print_metrics(f" Fold {split.fold_index} metrics:", fold_metric_row) + + return fold_metrics, oof_probabilities + + def _finalize_result(self, consensus_params, labels, oof_probabilities, best_params_per_fold, fold_metrics, metadata): + consensus = dict(consensus_params or {}) + + fold_metric_summary = self._summarize_metrics(fold_metrics) + self._print_metric_summary(" Mean fold metrics:", fold_metric_summary) + + oof_default_metrics = self._compute_evaluation_metrics( + labels, + oof_probabilities, + threshold=self.default_threshold, + ) + self._print_metrics(" OOF metrics before calibration:", oof_default_metrics) + + threshold = self._best_threshold(oof_probabilities, labels) + oof_calibrated_metrics = self._compute_evaluation_metrics( + labels, + oof_probabilities, + threshold=threshold, + ) + self._print_metrics(" OOF metrics after calibration:", oof_calibrated_metrics) + + metrics = { + 'skipped': False, + **metadata, + 'hyperparameter_optimization_enabled': bool(self.config.optimize_hyperparameter_search), + 'fixed_hyperparameters': dict(self.config.fixed_hyperparameters), + 'selected_params_per_fold': best_params_per_fold, + 'fold_metrics': fold_metrics, + 'fold_metric_summary': fold_metric_summary, + 'oof_default_threshold_metrics': oof_default_metrics, + 'oof_calibrated_metrics': oof_calibrated_metrics, + } + return CrossValidationResult( + threshold=threshold, + consensus_params=consensus, + metrics=metrics, + ) + + def _search_hyperparams(self, X_train, y_train, site_groups=None): + inner_cv, fit_kwargs = self._build_inner_cv(y_train, site_groups) + if inner_cv is None: + return {} + + search = RandomizedSearchCV( + estimator=self.build_search_model(y_train), + param_distributions=self.param_dist, + n_iter=self.config.search_iterations, + scoring='roc_auc', + cv=inner_cv, + random_state=self.config.random_state, + n_jobs=-1, + refit=False, + ) + search.fit(X_train, y_train, **fit_kwargs) + return search.best_params_ + + def _build_inner_cv(self, y_train, site_groups=None): + fit_kwargs = {} + + if self.config.use_site_grouped_cv and site_groups is not None: + site_groups = np.asarray(site_groups) + unique_groups = np.unique(site_groups) + if unique_groups.size >= 2: + fit_kwargs['groups'] = site_groups + return LeaveOneGroupOut(), fit_kwargs + + inner_cv, _ = self._build_stratified_splitter(y_train, self.config.outer_random_splits) + return inner_cv, fit_kwargs + + def _build_stratified_splitter(self, labels, requested_splits): + classes, class_counts = np.unique(labels, return_counts=True) + if len(classes) != 2: + return None, 0 + + n_splits = min(int(requested_splits), int(np.min(class_counts))) + if n_splits < 2: + return None, 0 + + return StratifiedKFold( + n_splits=n_splits, + shuffle=True, + random_state=self.config.random_state, + ), n_splits + + def _best_threshold(self, probabilities, labels): + thresholds = np.linspace(0, 1, 101) + best_score = -1.0 + best_value = self.default_threshold + + for threshold in thresholds: + predictions = (probabilities >= threshold).astype(np.int32) + score = f1_score(labels, predictions, zero_division=0) + if score > best_score: + best_score = score + best_value = float(threshold) + if best_value < 0.5: + best_value = 0.5 + + return best_value + + def _safe_auroc(self, labels, probabilities): + if len(np.unique(labels)) < 2: + return np.nan + return float(roc_auc_score(labels, probabilities)) + + def _safe_auprc(self, labels, probabilities): + if len(np.unique(labels)) < 2: + return np.nan + return float(average_precision_score(labels, probabilities, pos_label=1)) + + def _compute_evaluation_metrics(self, labels, probabilities, threshold): + labels = np.asarray(labels, dtype=np.int32) + probabilities = np.asarray(probabilities, dtype=np.float32) + predictions = (probabilities >= threshold).astype(np.int32) + + return { + 'threshold': float(threshold), + 'auroc': self._safe_auroc(labels, probabilities), + 'auprc': self._safe_auprc(labels, probabilities), + 'accuracy': float(accuracy_score(labels, predictions)), + 'f1': float(f1_score(labels, predictions, zero_division=0)), + 'precision': float(precision_score(labels, predictions, zero_division=0)), + 'recall': float(recall_score(labels, predictions, zero_division=0)), + } + + def _summarize_metrics(self, metric_rows): + summary = {} + metric_names = ('auroc', 'auprc', 'accuracy', 'f1', 'precision', 'recall') + + for metric_name in metric_names: + metric_values = np.asarray([row[metric_name] for row in metric_rows], dtype=np.float32) + finite_values = metric_values[np.isfinite(metric_values)] + if finite_values.size == 0: + summary[metric_name] = {'mean': None, 'std': None} + continue + + summary[metric_name] = { + 'mean': float(np.mean(finite_values)), + 'std': float(np.std(finite_values)), + } + + return summary + + def _consensus_params(self, params_per_fold): + consensus = {} + for key in self.param_dist.keys(): + values = [params[key] for params in params_per_fold if key in params] + if not values: + continue + consensus[key] = max(set(values), key=values.count) + return consensus + + def _format_metric_value(self, value): + return 'nan' if value is None or not np.isfinite(value) else f'{value:.3f}' + + def _print_metrics(self, prefix, metrics): + print( + f"{prefix} AUROC={self._format_metric_value(metrics['auroc'])}, " + f"AUPRC={self._format_metric_value(metrics['auprc'])}, " + f"Accuracy={self._format_metric_value(metrics['accuracy'])}, " + f"F1={self._format_metric_value(metrics['f1'])}, " + f"Precision={self._format_metric_value(metrics['precision'])}, " + f"Recall={self._format_metric_value(metrics['recall'])}, " + f"Threshold={metrics['threshold']:.2f}" + ) + + def _print_metric_summary(self, prefix, summary): + metric_names = ('auroc', 'auprc', 'accuracy', 'f1', 'precision', 'recall') + parts = [] + for metric_name in metric_names: + metric_summary = summary.get(metric_name, {}) + mean_value = self._format_metric_value(metric_summary.get('mean')) + std_value = self._format_metric_value(metric_summary.get('std')) + parts.append(f"{metric_name.upper()}={mean_value} +/- {std_value}") + print(f"{prefix} {', '.join(parts)}") diff --git a/src/pipeline/features.py b/src/pipeline/features.py new file mode 100644 index 0000000..54433c2 --- /dev/null +++ b/src/pipeline/features.py @@ -0,0 +1,413 @@ +import os + +import edfio +import joblib +import numpy as np +import pandas as pd + +from src.common.channel_utils import normalize_channel_label +from helper_code import HEADERS, PHYSIOLOGICAL_DATA_SUBFOLDER, load_age, load_sex, load_signal_data +from src.ecg_processing import ECG_FEATURE_LENGTH, ECG_FEATURE_NAMES, ECG_KEYWORDS, processECG +from src.eeg_processing import EEG_CHANNEL_SPECS, EEG_FEATURE_LENGTH, EEG_FEATURE_NAMES, processEEG, _get_eeg_aliases +from src.resp_processing import RESP_FEATURE_LENGTH, RESP_FEATURE_NAMES, processResp, _get_resp_alias_groups + +from .config import ( + DEFAULT_CSV_PATH, + FEATURE_CACHE_FOLDER_NAME, + SEGMENT_AGGREGATION_NAMES, + SCRIPT_DIR, + SEGMENT_DURATION_SECONDS, + SEGMENT_STRIDE_SECONDS, + TOTAL_PHYSIOLOGICAL_FEATURE_LENGTH, +) + + +REQUIRED_SIGNAL_ALIASES_CACHE = {} +DEMOGRAPHIC_FEATURE_NAMES = ( + 'Age', + 'Sex', +) + + +def _build_aggregated_feature_names(segment_feature_names): + return tuple( + f'{feature_name}_{aggregation_name}' + for feature_name in segment_feature_names + for aggregation_name in SEGMENT_AGGREGATION_NAMES + ) + + +FEATURE_NAME_GROUPS = { + 'demographics': DEMOGRAPHIC_FEATURE_NAMES, + 'resp': _build_aggregated_feature_names(RESP_FEATURE_NAMES), + 'eeg': _build_aggregated_feature_names(EEG_FEATURE_NAMES), + 'ecg': _build_aggregated_feature_names(ECG_FEATURE_NAMES), +} +FEATURE_NAMES = ( + *FEATURE_NAME_GROUPS['demographics'], + *FEATURE_NAME_GROUPS['resp'], + *FEATURE_NAME_GROUPS['eeg'], + *FEATURE_NAME_GROUPS['ecg'], +) + +SEGMENT_AGGREGATION_FUNCTIONS = { + 'Max': lambda values: float(np.max(values)), + 'Min': lambda values: float(np.min(values)), + 'Mean': lambda values: float(np.mean(values)), + 'Median': lambda values: float(np.median(values)), + 'Std': lambda values: float(np.std(values)), +} + + +def _get_feature_cache_root(data_folder): + return os.path.join(SCRIPT_DIR, FEATURE_CACHE_FOLDER_NAME) + + +def _coerce_feature_vector(features): + vector = np.asarray(features, dtype=np.float32).reshape(-1) + vector[~np.isfinite(vector)] = np.nan + return vector + + +def _extract_optional_features(extractor, expected_length, *args, **kwargs): + vector = _coerce_feature_vector(extractor(*args, **kwargs)) + if vector.size != expected_length: + raise ValueError( + f"{extractor.__name__} returned {vector.size} features; expected {expected_length}." + ) + return vector + + +def _get_physiological_data_file(data_folder, site_id, patient_id, session_id): + return os.path.join( + data_folder, + PHYSIOLOGICAL_DATA_SUBFOLDER, + site_id, + f"{patient_id}_ses-{session_id}.edf", + ) +def _get_required_signal_aliases(csv_path): + normalized_csv_path = os.path.abspath(csv_path) + required_aliases = REQUIRED_SIGNAL_ALIASES_CACHE.get(normalized_csv_path) + if required_aliases is not None: + return required_aliases + + resp_alias_groups = _get_resp_alias_groups(normalized_csv_path) + eeg_aliases = _get_eeg_aliases(normalized_csv_path) + + required_aliases = set() + for aliases in resp_alias_groups.values(): + required_aliases.update(aliases) + + for channel_spec in EEG_CHANNEL_SPECS.values(): + required_aliases.update(eeg_aliases.get(normalize_channel_label(channel_spec['direct']), set())) + required_aliases.update(eeg_aliases.get(normalize_channel_label(channel_spec['positive']), set())) + required_aliases.update(eeg_aliases.get(normalize_channel_label(channel_spec['reference']), set())) + + REQUIRED_SIGNAL_ALIASES_CACHE[normalized_csv_path] = required_aliases + return required_aliases + + +def _load_required_signal_data(edf_path, csv_path): + required_aliases = _get_required_signal_aliases(csv_path) + channel_dict = {} + fs_dict = {} + + try: + edf = edfio.read_edf(edf_path, lazy_load_data=True) + except Exception: + return load_signal_data(edf_path) + + for signal in edf.signals: + label = signal.label.lower().strip() + normalized_label = normalize_channel_label(label) + is_required_signal = normalized_label in required_aliases + is_ecg_signal = any(keyword in normalized_label for keyword in ECG_KEYWORDS) + + if not is_required_signal and not is_ecg_signal: + continue + + fs_dict[label] = float(signal.sampling_frequency) + channel_dict[label] = signal.data + + if channel_dict: + return channel_dict, fs_dict + + return load_signal_data(edf_path) + + +def _get_feature_cache_file(data_folder, site_id, patient_id, session_id): + cache_dir = os.path.join(_get_feature_cache_root(data_folder), site_id) + return os.path.join(cache_dir, f"{patient_id}_ses-{session_id}.sav") + + +def get_feature_export_dir(data_folder): + return os.path.join(_get_feature_cache_root(data_folder), 'exports') + + +def _get_feature_cache_csv_file(cache_file): + return f"{os.path.splitext(cache_file)[0]}.csv" + + +def _load_cached_feature_vector(cache_file): + if not os.path.exists(cache_file): + return None + + try: + payload = joblib.load(cache_file) + except Exception: + return None + + if isinstance(payload, dict): + payload = payload.get('features') + + if payload is None: + return None + + vector = _coerce_feature_vector(payload) + if vector.size != len(FEATURE_NAMES): + return None + return vector + + +def _save_cached_feature_vector(cache_file, feature_vector): + os.makedirs(os.path.dirname(cache_file), exist_ok=True) + temp_cache_file = f"{cache_file}.tmp" + payload = _coerce_feature_vector(feature_vector) + + try: + joblib.dump(payload, temp_cache_file, protocol=0) + os.replace(temp_cache_file, cache_file) + finally: + if os.path.exists(temp_cache_file): + os.remove(temp_cache_file) + + csv_file = _get_feature_cache_csv_file(cache_file) + temp_csv_file = f"{csv_file}.tmp" + try: + pd.DataFrame([payload], columns=get_feature_names()).to_csv(temp_csv_file, index=False) + os.replace(temp_csv_file, csv_file) + finally: + if os.path.exists(temp_csv_file): + os.remove(temp_csv_file) + + +def extract_demographic_features(data): + age_value = data.get(HEADERS['age']) + try: + age = float(age_value) + except (TypeError, ValueError): + age = np.nan + if not np.isfinite(age): + age = np.nan + age = np.array([age], dtype=np.float32) + + sex = load_sex(data) + if sex == 'Male': + sex_value = 1.0 + elif sex == 'Female': + sex_value = 0.0 + else: + sex_value = np.nan + sex_vec = np.array([sex_value], dtype=np.float32) + + return np.concatenate([age, sex_vec]).astype(np.float32) + + +def get_feature_names(): + return FEATURE_NAMES + + +def get_feature_group_indices(include_demographics=False): + groups = {} + start = len(FEATURE_NAME_GROUPS['demographics']) + + if include_demographics: + demo_indices = np.arange(start, dtype=np.int32) + else: + demo_indices = np.array([], dtype=np.int32) + + for group_name in ('resp', 'eeg', 'ecg'): + group_length = len(FEATURE_NAME_GROUPS[group_name]) + group_indices = np.arange(start, start + group_length, dtype=np.int32) + if include_demographics: + groups[group_name] = np.concatenate([demo_indices, group_indices]) + else: + groups[group_name] = group_indices + start += group_length + + groups['all'] = np.arange(len(FEATURE_NAMES), dtype=np.int32) + return groups + + +def _iter_signal_segments(physiological_data, physiological_fs): + if not physiological_data: + return [] + + durations = [] + for label, signal in physiological_data.items(): + fs = physiological_fs.get(label) + if fs is None or fs <= 0: + continue + durations.append(len(signal) / float(fs)) + + if not durations: + return [] + + max_duration_seconds = max(durations) + last_full_segment_start = max_duration_seconds - SEGMENT_DURATION_SECONDS + if last_full_segment_start < 0: + return [] + + segment_starts = np.arange(0.0, last_full_segment_start + 1e-9, SEGMENT_STRIDE_SECONDS, dtype=float) + segments = [] + + for start_seconds in segment_starts: + end_seconds = start_seconds + SEGMENT_DURATION_SECONDS + segment_data = {} + segment_fs = {} + + for label, signal in physiological_data.items(): + fs = physiological_fs.get(label) + if fs is None or fs <= 0: + continue + + start_index = int(round(start_seconds * fs)) + end_index = int(round(end_seconds * fs)) + if start_index >= len(signal) or end_index > len(signal): + continue + + sliced_signal = np.asarray(signal[start_index:end_index], dtype=float) + if sliced_signal.size == 0: + continue + + segment_data[label] = sliced_signal + segment_fs[label] = fs + + if segment_data: + segments.append((segment_data, segment_fs)) + + return segments + + +def _aggregate_segment_feature_vectors(feature_vectors, segment_feature_names): + aggregated_length = len(segment_feature_names) * len(SEGMENT_AGGREGATION_NAMES) + if not feature_vectors: + return np.full(aggregated_length, np.nan, dtype=np.float32) + + matrix = np.asarray(feature_vectors, dtype=np.float32) + aggregated_values = [] + + for column_index in range(matrix.shape[1]): + column_values = matrix[:, column_index] + finite_values = column_values[np.isfinite(column_values)] + + if finite_values.size == 0: + aggregated_values.extend([np.nan] * len(SEGMENT_AGGREGATION_NAMES)) + continue + + aggregated_values.extend( + SEGMENT_AGGREGATION_FUNCTIONS[aggregation_name](finite_values) + for aggregation_name in SEGMENT_AGGREGATION_NAMES + ) + + return np.asarray(aggregated_values, dtype=np.float32) + + +def _extract_segmented_features(extractor, segment_feature_names, physiological_data, physiological_fs, csv_path): + expected_length = len(segment_feature_names) + segments = _iter_signal_segments(physiological_data, physiological_fs) + if not segments: + aggregated_length = expected_length * len(SEGMENT_AGGREGATION_NAMES) + return np.full(aggregated_length, np.nan, dtype=np.float32) + + segment_feature_vectors = [] + for segment_data, segment_fs in segments: + try: + vector = _extract_optional_features( + extractor, + expected_length, + segment_data, + segment_fs, + csv_path=csv_path, + ) + except Exception: + continue + + if np.all(np.isnan(vector)): + continue + + segment_feature_vectors.append(vector) + + return _aggregate_segment_feature_vectors(segment_feature_vectors, segment_feature_names) + + +def extract_extended_physiological_features(physiological_data, physiological_fs, csv_path=DEFAULT_CSV_PATH): + resp_features = _extract_segmented_features( + processResp, + RESP_FEATURE_NAMES, + physiological_data, + physiological_fs, + csv_path, + ) + eeg_features = _extract_segmented_features( + processEEG, + EEG_FEATURE_NAMES, + physiological_data, + physiological_fs, + csv_path, + ) + ecg_features = _extract_segmented_features( + processECG, + ECG_FEATURE_NAMES, + physiological_data, + physiological_fs, + csv_path, + ) + + return np.hstack([resp_features, eeg_features, ecg_features]).astype(np.float32) + + +def _compute_record_feature_vector(patient_data, data_folder, site_id, patient_id, session_id, csv_path, require_physiological_data): + demographic_features = extract_demographic_features(patient_data) + physiological_data_file = _get_physiological_data_file( + data_folder, + site_id, + patient_id, + session_id, + ) + print(f"Extracting features for patient {patient_id}, session {session_id} from file: {physiological_data_file}") + if os.path.exists(physiological_data_file): + physiological_data, physiological_fs = _load_required_signal_data(physiological_data_file, csv_path) + physiological_features = extract_extended_physiological_features( + physiological_data, + physiological_fs, + csv_path=csv_path, + ) + elif require_physiological_data: + raise FileNotFoundError(f"Missing physiological data for {patient_id}.") + else: + physiological_features = np.full(TOTAL_PHYSIOLOGICAL_FEATURE_LENGTH, np.nan, dtype=np.float32) + + return np.hstack([demographic_features, physiological_features]).astype(np.float32) + + +def get_or_create_record_feature_vector(record, data_folder, patient_data, csv_path=DEFAULT_CSV_PATH, require_physiological_data=True): + patient_id = record[HEADERS['bids_folder']] + site_id = record[HEADERS['site_id']] + session_id = record[HEADERS['session_id']] + cache_file = _get_feature_cache_file(data_folder, site_id, patient_id, session_id) + cached_features = _load_cached_feature_vector(cache_file) + if cached_features is not None: + return cached_features + + feature_vector = _compute_record_feature_vector( + patient_data, + data_folder, + site_id, + patient_id, + session_id, + csv_path, + require_physiological_data, + ) + _save_cached_feature_vector(cache_file, feature_vector) + return feature_vector \ No newline at end of file diff --git a/src/pipeline/preprocessing.py b/src/pipeline/preprocessing.py new file mode 100644 index 0000000..1d37657 --- /dev/null +++ b/src/pipeline/preprocessing.py @@ -0,0 +1,245 @@ +import numpy as np +from sklearn.impute import KNNImputer +from sklearn.preprocessing import StandardScaler +from sklearn.decomposition import PCA + +from .config import FEATURE_CORRELATION_THRESHOLD + + +DEFAULT_KNN_NEIGHBORS = 5 +PCA_VARIANCE_THRESHOLD = 0.7 + + +def build_preprocessor(num_samples, categorical_indices=None, apply_pca=False): + neighbors = min(DEFAULT_KNN_NEIGHBORS, max(1, num_samples - 1)) if num_samples > 1 else 1 + return CorrelationAwarePreprocessor( + n_neighbors=neighbors, + categorical_indices=categorical_indices, + correlation_threshold=FEATURE_CORRELATION_THRESHOLD, + apply_pca=apply_pca, + ) + + +def get_processed_feature_names(feature_names, preprocessor=None): + if preprocessor is None: + return list(feature_names) + + if hasattr(preprocessor, 'get_feature_names_out'): + return list(preprocessor.get_feature_names_out(feature_names)) + + return list(feature_names) + + +def remap_feature_indices(preprocessor, feature_indices): + if preprocessor is None or not hasattr(preprocessor, 'transform_feature_indices'): + return { + name: np.asarray(indices, dtype=np.int32) + for name, indices in feature_indices.items() + } + + return preprocessor.transform_feature_indices(feature_indices) + + +class CorrelationThresholdSelector: + def __init__(self, threshold): + self.threshold = float(threshold) + self.selected_indices_ = None + + def fit(self, X): + X = np.asarray(X, dtype=np.float32) + n_samples, n_features = X.shape + + if n_features == 0: + self.selected_indices_ = np.array([], dtype=np.int32) + return self + + if n_samples < 2 or n_features == 1: + self.selected_indices_ = np.arange(n_features, dtype=np.int32) + return self + + with np.errstate(divide='ignore', invalid='ignore'): + corr = np.corrcoef(X, rowvar=False) + corr = np.asarray(corr, dtype=np.float32) + corr = np.nan_to_num(np.abs(corr), nan=0.0, posinf=0.0, neginf=0.0) + np.fill_diagonal(corr, 0.0) + + keep_mask = np.ones(n_features, dtype=bool) + for index in range(n_features): + if not keep_mask[index]: + continue + + correlated_indices = np.where(corr[index, index + 1:] > self.threshold)[0] + if correlated_indices.size: + keep_mask[correlated_indices + index + 1] = False + + if not np.any(keep_mask): + keep_mask[0] = True + + self.selected_indices_ = np.flatnonzero(keep_mask).astype(np.int32) + return self + + def transform(self, X): + X = np.asarray(X, dtype=np.float32) + if self.selected_indices_ is None: + raise ValueError('Correlation selector has not been fitted.') + return X[:, self.selected_indices_] + + def get_feature_names_out(self, input_features=None): + if self.selected_indices_ is None: + raise ValueError('Correlation selector has not been fitted.') + + if input_features is None: + input_features = [f'feature_{index}' for index in self.selected_indices_] + + return np.asarray([input_features[index] for index in self.selected_indices_], dtype=object) + + +class PCAReducer: + """Applies PCA for dimensionality reduction while preserving a specified variance threshold.""" + + def __init__(self, variance_threshold=PCA_VARIANCE_THRESHOLD): + self.variance_threshold = float(variance_threshold) + self.pca = None + self.n_components_used = None + + def fit(self, X): + """Fit PCA to the data, determining optimal number of components.""" + X = np.asarray(X, dtype=np.float32) + + if X.shape[0] < 2 or X.shape[1] < 2: + self.pca = None + self.n_components_used = X.shape[1] + return self + + # Fit PCA with all possible components initially + n_components = min(X.shape[0] - 1, X.shape[1]) + temp_pca = PCA(n_components=n_components, random_state=42) + temp_pca.fit(X) + + # Calculate cumulative variance explained + cumsum_var = np.cumsum(temp_pca.explained_variance_ratio_) + + # Find number of components needed for target variance + n_comp = np.argmax(cumsum_var >= self.variance_threshold) + 1 + n_comp = max(1, min(n_comp, n_components)) + + # Fit final PCA with optimal components + self.pca = PCA(n_components=n_comp, random_state=42) + self.pca.fit(X) + self.n_components_used = n_comp + + return self + + def transform(self, X): + """Transform data using fitted PCA.""" + X = np.asarray(X, dtype=np.float32) + + if self.pca is None: + return X + + return np.asarray(self.pca.transform(X), dtype=np.float32) + + def fit_transform(self, X): + """Fit PCA and transform data.""" + self.fit(X) + return self.transform(X) + + def get_feature_names_out(self, input_features=None): + """Generate names for PCA components.""" + if self.pca is None: + return input_features if input_features is not None else np.array(['feature_0']) + + n_features = self.pca.n_components_ + return np.asarray([f'PCA_{i}' for i in range(n_features)], dtype=object) + + +class CorrelationAwarePreprocessor: + def __init__(self, n_neighbors, categorical_indices, correlation_threshold, apply_pca=True): + self.n_neighbors = n_neighbors + if categorical_indices is None: + self.categorical_indices = np.array([], dtype=np.int32) + else: + self.categorical_indices = np.asarray(categorical_indices, dtype=np.int32) + self.imputer = KNNImputer(n_neighbors=n_neighbors, keep_empty_features=True) + self.scaler = StandardScaler() + self.selector = CorrelationThresholdSelector(correlation_threshold) + self.pca = PCAReducer() if apply_pca else None + self._numerical_indices = np.array([], dtype=np.int32) + + def _get_numerical_indices(self, n_features): + all_idx = np.arange(n_features, dtype=np.int32) + return np.setdiff1d(all_idx, self.categorical_indices) + + def _scale_numerical_columns(self, X_imputed, fit=False): + X_out = X_imputed.copy() + if self._numerical_indices.size == 0: + return X_out + + X_num = X_imputed[:, self._numerical_indices] + if fit: + X_num_scaled = np.asarray(self.scaler.fit_transform(X_num), dtype=np.float32) + else: + X_num_scaled = np.asarray(self.scaler.transform(X_num), dtype=np.float32) + + X_out[:, self._numerical_indices] = X_num_scaled + return X_out + + def fit_transform(self, X): + X = np.asarray(X, dtype=np.float32).copy() + X[~np.isfinite(X)] = np.nan + + X_imputed = np.asarray(self.imputer.fit_transform(X), dtype=np.float32) + self._numerical_indices = self._get_numerical_indices(X.shape[1]) + X_out = self._scale_numerical_columns(X_imputed, fit=True) + self.selector.fit(X_out) + X_selected = np.asarray(self.selector.transform(X_out), dtype=np.float32) + + # Apply PCA if enabled + if self.pca is not None: + X_final = self.pca.fit_transform(X_selected) + else: + X_final = X_selected + + return np.asarray(X_final, dtype=np.float32) + + def transform(self, X): + X = np.asarray(X, dtype=np.float32).copy() + X[~np.isfinite(X)] = np.nan + X_imputed = np.asarray(self.imputer.transform(X), dtype=np.float32) + X_out = self._scale_numerical_columns(X_imputed, fit=False) + X_selected = np.asarray(self.selector.transform(X_out), dtype=np.float32) + + # Apply PCA if enabled + if self.pca is not None: + X_final = self.pca.transform(X_selected) + else: + X_final = X_selected + + return np.asarray(X_final, dtype=np.float32) + + def transform_feature_indices(self, feature_indices): + if self.selector.selected_indices_ is None: + raise ValueError('Preprocessor has not been fitted.') + + index_lookup = { + int(raw_index): int(processed_index) + for processed_index, raw_index in enumerate(self.selector.selected_indices_) + } + remapped = {} + for name, indices in feature_indices.items(): + kept_indices = [ + index_lookup[int(raw_index)] + for raw_index in np.asarray(indices, dtype=np.int32) + if int(raw_index) in index_lookup + ] + remapped[name] = np.asarray(kept_indices, dtype=np.int32) + return remapped + + def get_feature_names_out(self, input_features=None): + selector_names = self.selector.get_feature_names_out(input_features) + + # If PCA is applied, return PCA component names + if self.pca is not None: + return self.pca.get_feature_names_out(selector_names) + + return selector_names \ No newline at end of file diff --git a/src/pipeline/training.py b/src/pipeline/training.py new file mode 100644 index 0000000..9fb1630 --- /dev/null +++ b/src/pipeline/training.py @@ -0,0 +1,456 @@ +import os +from concurrent.futures import ThreadPoolExecutor + +import numpy as np +import pandas as pd +from tqdm import tqdm +from xgboost import XGBClassifier + +from helper_code import DEMOGRAPHICS_FILE, HEADERS, find_patients, load_label + +from .config import ( + CV_RANDOM_STATE, + CV_SEARCH_ITERATIONS, + DEFAULT_CV_HYPERPARAMETERS, + MAX_TRAIN_WORKERS, + OPTIMIZE_HYPERPARAMETER_SEARCH, + RANDOM_CV_N_SPLITS, + USE_SITE_GROUPED_CV, +) +from .cross_validation import CrossValidationConfig, EnsembleCrossValidator, normalize_site_group +from .features import get_feature_group_indices, get_feature_names, get_or_create_record_feature_vector +from .preprocessing import build_preprocessor, get_processed_feature_names, remap_feature_indices, PCA_VARIANCE_THRESHOLD + + +DEFAULT_ENSEMBLE_THRESHOLD = 0.5 +ENSEMBLE_MODALITIES = ('resp', 'eeg', 'ecg') + +# Hyperparameter search space +PARAM_DIST = { + 'max_depth': [3, 4, 5], + 'min_child_weight': [1, 2, 3], + 'subsample': [0.7, 0.8, 0.9], + 'colsample_bytree': [0.6, 0.7, 0.8], + 'reg_lambda': [0.5, 1.0, 2.0], + 'reg_alpha': [0.0, 0.05, 0.1], +} + + +def build_preprocessor_for_cv(num_samples, categorical_indices=None): + """Build preprocessor without PCA for cross-validation.""" + return build_preprocessor(num_samples, categorical_indices, apply_pca=False) + + +def build_training_metadata_cache(patient_data_file): + metadata = pd.read_csv(patient_data_file) + demographics_cache = {} + diagnosis_cache = {} + + for row in metadata.to_dict('records'): + patient_id = row[HEADERS['bids_folder']] + session_id = row[HEADERS['session_id']] + demographics_cache[(patient_id, session_id)] = row + diagnosis_cache[patient_id] = load_label(row) + + return demographics_cache, diagnosis_cache + + +def process_training_record(record, data_folder, demographics_cache, diagnosis_cache, csv_path): + patient_id = record[HEADERS['bids_folder']] + session_id = record[HEADERS['session_id']] + site_id = record[HEADERS['site_id']] + + try: + patient_data = demographics_cache.get((patient_id, session_id), {}) + feature_vector = get_or_create_record_feature_vector( + record, + data_folder, + patient_data, + csv_path=csv_path, + require_physiological_data=True, + ) + + label = diagnosis_cache.get(patient_id) + metadata = { + 'patient_id': patient_id, + 'site_id': site_id, + 'session_id': session_id, + } + + if label == 0 or label == 1: + return metadata, feature_vector, label, None + + return metadata, None, None, f"Invalid label for {patient_id}. Skipping..." + + except FileNotFoundError as exc: + return { + 'patient_id': patient_id, + 'site_id': site_id, + 'session_id': session_id, + }, None, None, f"{exc} Skipping..." + except Exception as exc: + return { + 'patient_id': patient_id, + 'site_id': site_id, + 'session_id': session_id, + }, None, None, f"Error processing {patient_id}: {exc}" + +def prepare_feature_matrix(feature_matrix, preprocessor=None): + raw_feature_matrix = np.asarray(feature_matrix, dtype=np.float32) + if raw_feature_matrix.ndim == 1: + raw_feature_matrix = raw_feature_matrix.reshape(1, -1) + raw_feature_matrix = raw_feature_matrix.copy() + raw_feature_matrix[~np.isfinite(raw_feature_matrix)] = np.nan + + if preprocessor is not None: + processed_feature_matrix = np.asarray(preprocessor.transform(raw_feature_matrix), dtype=np.float32) + else: + processed_feature_matrix = raw_feature_matrix + + return raw_feature_matrix, processed_feature_matrix + +def export_feature_matrix_csv(output_path, metadata_rows, feature_matrix, feature_names, labels=None): + dataframe = pd.DataFrame(metadata_rows) + if labels is not None: + dataframe['label'] = labels + feature_frame = pd.DataFrame(feature_matrix, columns=feature_names) + dataframe = pd.concat([dataframe.reset_index(drop=True), feature_frame.reset_index(drop=True)], axis=1) + os.makedirs(os.path.dirname(output_path), exist_ok=True) + dataframe.to_csv(output_path, index=False) + +def get_feature_export_paths(export_root, prefix): + return { + 'raw': os.path.join(export_root, f'{prefix}_features_raw.csv'), + 'preprocessed': os.path.join(export_root, f'{prefix}_features_preprocessed.csv'), + } + + +def _get_feature_group_name(feature_index, modality_presence_indices): + for group_name in ('resp', 'eeg', 'ecg'): + group_index_set = set(np.asarray(modality_presence_indices[group_name], dtype=np.int32).tolist()) + if feature_index in group_index_set: + return group_name + + return 'demographics' + + +def export_selected_features_csv(output_path, feature_names, selected_raw_feature_indices, modality_presence_indices): + selected_rows = [] + for processed_index, raw_index in enumerate(np.asarray(selected_raw_feature_indices, dtype=np.int32)): + selected_rows.append({ + 'processed_index': int(processed_index), + 'raw_index': int(raw_index), + 'feature_name': feature_names[int(raw_index)], + 'group': _get_feature_group_name(int(raw_index), modality_presence_indices), + }) + + os.makedirs(os.path.dirname(output_path), exist_ok=True) + pd.DataFrame(selected_rows).to_csv(output_path, index=False) + +def export_feature_views(export_root, prefix, metadata_rows, feature_matrix, feature_names, preprocessor=None, labels=None): + raw_feature_matrix, processed_feature_matrix = prepare_feature_matrix( + feature_matrix, + preprocessor=preprocessor, + ) + export_paths = get_feature_export_paths(export_root, prefix) + export_feature_matrix_csv( + export_paths['raw'], + metadata_rows, + raw_feature_matrix, + feature_names, + labels=labels, + ) + export_feature_matrix_csv( + export_paths['preprocessed'], + metadata_rows, + processed_feature_matrix, + get_processed_feature_names(feature_names, preprocessor=preprocessor), + labels=labels, + ) + return export_paths + + +def _build_xgb_model(labels, extra_params=None): + + neg = int(np.sum(labels == 0)) + pos = int(np.sum(labels == 1)) + scale_pos_weight = (neg / pos) if pos > 0 else 1.0 + + base_params = dict( + scale_pos_weight=scale_pos_weight, + n_estimators=500, + learning_rate=0.05, + max_depth=4, + subsample=0.8, + colsample_bytree=0.7, + min_child_weight=2, + reg_alpha=0.0, + reg_lambda=1.0, + random_state=42, + eval_metric='auc', + tree_method='hist', + ) + if extra_params: + base_params.update(extra_params) + + return XGBClassifier(**base_params) + +def _fit_model(feature_matrix, labels, consensus_params=None): + model = _build_xgb_model(labels, extra_params=consensus_params) + model.fit(feature_matrix, labels) + return model + + +def _build_search_model(labels): + return XGBClassifier( + scale_pos_weight=(int(np.sum(labels == 0)) / max(int(np.sum(labels == 1)), 1)), + n_estimators=500, + learning_rate=0.05, + random_state=CV_RANDOM_STATE, + eval_metric='auc', + tree_method='hist', + ) + +def _fit_ensemble(feature_matrix, labels, feature_indices, consensus_params=None): + models = {} + if feature_indices['all'].size == 0: + raise ValueError('Correlation selector removed all features for the global model.') + + models['all'] = _fit_model( + feature_matrix[:, feature_indices['all']], labels, consensus_params + ) + for modality in ENSEMBLE_MODALITIES: + if feature_indices[modality].size == 0: + continue + models[modality] = _fit_model( + feature_matrix[:, feature_indices[modality]], labels, consensus_params + ) + return models + + +def _has_modality_signal(feature_vector, modality_presence_indices): + modality_values = feature_vector[modality_presence_indices] + return bool(np.any(np.isfinite(modality_values))) + + +def predict_ensemble_probabilities(model_bundle, feature_matrix): + raw_feature_matrix, processed_feature_matrix = prepare_feature_matrix( + feature_matrix, + preprocessor=model_bundle.get('preprocessor'), + ) + + # Debug: Print feature dimensions + preprocessor = model_bundle.get('preprocessor') + if preprocessor and hasattr(preprocessor, 'pca') and preprocessor.pca is not None: + print(f" [DEBUG] Using preprocessor with PCA: {raw_feature_matrix.shape[1]} → {processed_feature_matrix.shape[1]} features") + + models = model_bundle['models'] + feature_indices = { + name: np.asarray(indices, dtype=np.int32) + for name, indices in model_bundle['feature_indices'].items() + } + modality_presence_indices = { + name: np.asarray(indices, dtype=np.int32) + for name, indices in model_bundle['modality_presence_indices'].items() + } + + probabilities = np.zeros(raw_feature_matrix.shape[0], dtype=np.float32) + for row_index, raw_feature_vector in enumerate(raw_feature_matrix): + processed_feature_vector = processed_feature_matrix[row_index] + modality_probabilities = [] + for modality in ENSEMBLE_MODALITIES: + if modality not in models or feature_indices[modality].size == 0: + continue + if _has_modality_signal(raw_feature_vector, modality_presence_indices[modality]): + modality_vector = processed_feature_vector[feature_indices[modality]].reshape(1, -1) + modality_probability = models[modality].predict_proba(modality_vector)[0][1] + modality_probabilities.append(float(modality_probability)) + + if modality_probabilities: + probabilities[row_index] = float(np.mean(modality_probabilities)) + else: + all_features = processed_feature_vector[feature_indices['all']].reshape(1, -1) + probabilities[row_index] = float(models['all'].predict_proba(all_features)[0][1]) + + return probabilities + + +def predict_ensemble_labels(model_bundle, feature_matrix): + threshold = float(model_bundle.get('threshold', DEFAULT_ENSEMBLE_THRESHOLD)) + probabilities = predict_ensemble_probabilities(model_bundle, feature_matrix) + labels = (probabilities >= threshold).astype(np.int32) + return labels, probabilities + + +def train_multimodal_ensemble(data_folder, verbose, csv_path, export_folder=None): + patient_data_file = os.path.join(data_folder, DEMOGRAPHICS_FILE) + patient_metadata_list = find_patients(patient_data_file) + demographics_cache, diagnosis_cache = build_training_metadata_cache(patient_data_file) + num_records = len(patient_metadata_list) + + if num_records == 0: + raise FileNotFoundError('No data were provided.') + + features = [] + labels = [] + metadata_rows = [] + + with ThreadPoolExecutor(max_workers=MAX_TRAIN_WORKERS) as executor: + results = executor.map( + lambda record: process_training_record( + record, + data_folder, + demographics_cache, + diagnosis_cache, + csv_path, + ), + patient_metadata_list, + ) + + pbar = tqdm(results, total=num_records, desc='Extracting Features', unit='record', disable=not verbose) + for metadata, feature_vector, label, message in pbar: + if verbose: + pbar.set_postfix({'patient': metadata['patient_id']}) + + if message is not None: + tqdm.write(f" ! {message}") + continue + + features.append(feature_vector) + labels.append(label) + metadata_rows.append(metadata) + + pbar.close() + + features = np.asarray(features, dtype=np.float32) + labels = np.asarray(labels, dtype=np.int32) + + if features.size == 0 or features.ndim != 2 or features.shape[0] == 0: + raise ValueError('No valid training samples were extracted. Review feature extraction logs for the skipped records.') + + feature_names = list(get_feature_names()) + feature_indices = get_feature_group_indices(include_demographics=True) + modality_presence_indices = get_feature_group_indices(include_demographics=False) + + categorical_indices = [ + i for i, name in enumerate(feature_names) + if name.lower() in ('sex', 'gender') + ] + site_groups = np.asarray([ + normalize_site_group(metadata_row['site_id']) + for metadata_row in metadata_rows + ]) + cv_config = CrossValidationConfig( + use_site_grouped_cv=USE_SITE_GROUPED_CV, + optimize_hyperparameter_search=OPTIMIZE_HYPERPARAMETER_SEARCH, + outer_random_splits=RANDOM_CV_N_SPLITS, + random_state=CV_RANDOM_STATE, + search_iterations=CV_SEARCH_ITERATIONS, + fixed_hyperparameters=DEFAULT_CV_HYPERPARAMETERS, + ) + cv_runner = EnsembleCrossValidator( + config=cv_config, + param_dist=PARAM_DIST, + default_threshold=DEFAULT_ENSEMBLE_THRESHOLD, + build_preprocessor=build_preprocessor_for_cv, + build_search_model=_build_search_model, + fit_ensemble=_fit_ensemble, + predict_probabilities=predict_ensemble_probabilities, + ) + print(f" Categorical feature indices: {categorical_indices} " + f"({[feature_names[i] for i in categorical_indices]})") + print(f" Hospital CV groups: {sorted(np.unique(site_groups).tolist())}") + print(f" CV strategy: {'grouped by hospital' if cv_config.use_site_grouped_cv else 'random stratified folds'}") + print(f" Hyperparameter search: {'enabled' if cv_config.optimize_hyperparameter_search else 'disabled'}") + print(' Feature selection is fitted inside each CV fold and then re-fitted on all training data for the final model.') + + # --- Step 1: Nested CV for threshold calibration and consensus hyperparameters --- + print("Running nested CV for threshold calibration and hyperparameter consensus...") + cv_result = cv_runner.run( + features, + labels, + feature_indices, + modality_presence_indices, + categorical_indices=categorical_indices if categorical_indices else None, + site_groups=site_groups, + ) + threshold = cv_result.threshold + consensus = cv_result.consensus_params + cv_metrics = cv_result.metrics + + # --- Step 2: Fit final models on ALL data using consensus hyperparameters --- + print("Fitting final ensemble on all training data with consensus hyperparameters...") + preprocessor = build_preprocessor(len(labels), categorical_indices if categorical_indices else None, apply_pca=True) + processed_features = np.asarray(preprocessor.fit_transform(features), dtype=np.float32) + processed_feature_names = get_processed_feature_names(feature_names, preprocessor=preprocessor) + selected_raw_feature_indices = np.asarray(preprocessor.selector.selected_indices_, dtype=np.int32) + print( + f"Correlation selector: kept {len(preprocessor.selector.selected_indices_)}/{len(feature_names)} features" + ) + if preprocessor.pca is not None: + print( + f"PCA: reduced {len(preprocessor.selector.selected_indices_)} features to {preprocessor.pca.n_components_used} components " + f"(explaining {PCA_VARIANCE_THRESHOLD*100:.1f}% of variance)" + ) + # When PCA is applied, all components are used for all modalities since PCA creates new synthetic features + n_pca_components = preprocessor.pca.n_components_used + selected_feature_indices = { + 'all': np.arange(n_pca_components, dtype=np.int32), + 'resp': np.arange(n_pca_components, dtype=np.int32), + 'eeg': np.arange(n_pca_components, dtype=np.int32), + 'ecg': np.arange(n_pca_components, dtype=np.int32), + } + else: + selected_feature_indices = remap_feature_indices(preprocessor, feature_indices) + + models = _fit_ensemble(processed_features, labels, selected_feature_indices, consensus_params=consensus) + + export_root = export_folder or os.path.join(os.getcwd(), 'feature_exports') + feature_exports = export_feature_views( + export_root, + 'training', + metadata_rows, + features, + feature_names, + preprocessor=preprocessor, + labels=labels, + ) + selected_features_csv = os.path.join(export_root, 'training_features_selected.csv') + export_selected_features_csv( + selected_features_csv, + feature_names, + selected_raw_feature_indices, + modality_presence_indices, + ) + feature_exports['selected'] = selected_features_csv + + print(f" [INFO] Final model information:") + print(f" - PCA enabled: {preprocessor.pca is not None}") + if preprocessor.pca is not None: + print(f" - PCA components: {preprocessor.pca.n_components_used}") + print(f" - Feature indices keys: {list(selected_feature_indices.keys())}") + print(f" - All modality features: {selected_feature_indices['all']}") + + return { + 'type': 'multimodal_xgb_ensemble', + 'threshold': threshold, + 'feature_names': feature_names, + 'processed_feature_names': processed_feature_names, + 'selected_raw_feature_indices': selected_raw_feature_indices.tolist(), + 'feature_indices': { + name: indices.tolist() + for name, indices in selected_feature_indices.items() + if name in {'all', 'resp', 'eeg', 'ecg'} + }, + 'modality_presence_indices': { + modality: modality_presence_indices[modality].tolist() + for modality in ENSEMBLE_MODALITIES + }, + 'models': models, + 'preprocessor': preprocessor, + 'pca_enabled': preprocessor.pca is not None, + 'pca_n_components': preprocessor.pca.n_components_used if preprocessor.pca is not None else None, + 'pca_variance_threshold': 0.95, + 'feature_exports': feature_exports, + 'cv_metrics': cv_metrics, + } diff --git a/src/pipeline/training_JM.py b/src/pipeline/training_JM.py new file mode 100644 index 0000000..b98f0fe --- /dev/null +++ b/src/pipeline/training_JM.py @@ -0,0 +1,445 @@ +import os +from concurrent.futures import ThreadPoolExecutor + +import numpy as np +import pandas as pd +from sklearn.metrics import f1_score +from sklearn.model_selection import train_test_split +from sklearn.pipeline import Pipeline +from tqdm import tqdm +from xgboost import XGBClassifier, data + +from sklearn.compose import ColumnTransformer +from sklearn.impute import KNNImputer, SimpleImputer +from sklearn.preprocessing import StandardScaler, OneHotEncoder + +from helper_code import DEMOGRAPHICS_FILE, HEADERS, find_patients, load_label + +from .config import MAX_TRAIN_WORKERS +from .features import get_feature_group_indices, get_feature_names, get_or_create_record_feature_vector + + +DEFAULT_ENSEMBLE_THRESHOLD = 0.5 +ENSEMBLE_MODALITIES = ('resp', 'eeg', 'ecg') +DEFAULT_KNN_NEIGHBORS = 5 + + +def build_training_metadata_cache(patient_data_file): + metadata = pd.read_csv(patient_data_file) + demographics_cache = {} + diagnosis_cache = {} + + for row in metadata.to_dict('records'): + patient_id = row[HEADERS['bids_folder']] + session_id = row[HEADERS['session_id']] + demographics_cache[(patient_id, session_id)] = row + diagnosis_cache[patient_id] = load_label(row) + + return demographics_cache, diagnosis_cache + + +def process_training_record(record, data_folder, demographics_cache, diagnosis_cache, csv_path): + patient_id = record[HEADERS['bids_folder']] + session_id = record[HEADERS['session_id']] + site_id = record[HEADERS['site_id']] + + try: + patient_data = demographics_cache.get((patient_id, session_id), {}) + feature_vector = get_or_create_record_feature_vector( + record, + data_folder, + patient_data, + csv_path=csv_path, + require_physiological_data=True, + ) + + label = diagnosis_cache.get(patient_id) + metadata = { + 'patient_id': patient_id, + 'site_id': site_id, + 'session_id': session_id, + } + + if label == 0 or label == 1: + return metadata, feature_vector, label, None + + return metadata, None, None, f"Invalid label for {patient_id}. Skipping..." + + except FileNotFoundError as exc: + return { + 'patient_id': patient_id, + 'site_id': site_id, + 'session_id': session_id, + }, None, None, f"{exc} Skipping..." + except Exception as exc: + return { + 'patient_id': patient_id, + 'site_id': site_id, + 'session_id': session_id, + }, None, None, f"Error processing {patient_id}: {exc}" + + +def _export_feature_matrix_csv(output_path, metadata_rows, labels, feature_matrix, feature_names): + dataframe = pd.DataFrame(metadata_rows) + dataframe['label'] = labels + feature_frame = pd.DataFrame(feature_matrix, columns=feature_names) + dataframe = pd.concat([dataframe.reset_index(drop=True), feature_frame.reset_index(drop=True)], axis=1) + os.makedirs(os.path.dirname(output_path), exist_ok=True) + dataframe.to_csv(output_path, index=False) + + + + + + +def best_threshold(probabilities, labels): + thresholds = np.linspace(0, 1, 101) + best_score = -1.0 + best_value = DEFAULT_ENSEMBLE_THRESHOLD + + for threshold in thresholds: + predictions = (probabilities >= threshold).astype(np.int32) + score = f1_score(labels, predictions, zero_division=0) + if score > best_score: + best_score = score + best_value = float(threshold) + + return best_value + + +def _build_xgb_model(labels): + neg = int(np.sum(labels == 0)) + pos = int(np.sum(labels == 1)) + scale_pos_weight = (neg / pos) if pos > 0 else 1.0 + + return XGBClassifier( + scale_pos_weight=scale_pos_weight, + n_estimators=300, + max_depth=4, + learning_rate=0.05, + subsample=0.8, + random_state=42, + eval_metric='auc', + tree_method='hist', + enable_categorical=False, + early_stopping_rounds=20) + +def preprocess_multimodal_data( df_model): + + #Llamamos al dataset de entrada que contiene las variables calculadas y confirma que no hay IDs duplicados + df_model2 = df_model.drop_duplicates(subset='ID') + # Extrae las columnas que se usarán + df_model2=df_model2[['ID','label', 'Age','Sex','Nasal_Peakedness_Max', 'Nasal_Peakedness_Min', 'Nasal_Peakedness_Median','Nasal_Peakedness_Std', + 'Chest_Peakedness_Max', 'Chest_Peakedness_Min','Abdomen_Peakedness_Max','Abdomen_Peakedness_Min','Abdomen_Peakedness_Std', + 'Flow_Peakedness_Max','Flow_Peakedness_Min','Flow_Peakedness_Median','Flow_Peakedness_Std','SpO2_Max','SpO2_Min','SpO2_Std', + 'CET90','ODI_Mean','ODI_deepness','C3-M2_Hjorth_Complexity','C4-M1_Hjorth_Complexity','F3-M2_Hjorth_Complexity', + 'F4-M1_Hjorth_Complexity','C3-M2_Hjorth_Mobility','F3-M2_Hjorth_Mobility','F4-M1_Hjorth_Mobility','C3-M2_Ratio_Slow_Fast', + 'C4-M1_Ratio_Slow_Fast','F3-M2_Ratio_Slow_Fast','F4-M1_Ratio_Slow_Fast','C3-M2_Rel_Beta','F3-M2_Rel_Beta','F4-M1_Rel_Beta', + 'C4-M1_Rel_Sigma','F3-M2_Rel_Sigma','C3-M2_Relative_Delta_Power','C4-M1_Relative_Delta_Power','F3-M2_Relative_Delta_Power', + 'F4-M1_Relative_Delta_Power','C3-M2_Theta_Alpha_Ratio','C4-M1_Theta_Alpha_Ratio','F3-M2_Theta_Alpha_Ratio','F4-M1_Theta_Alpha_Ratio', + 'C3-M2_Theta_Beta_Ratio','C4-M1_Theta_Beta_Ratio','F3-M2_Theta_Beta_Ratio','F4-M1_Theta_Beta_Ratio','C3-M2_kurtosis_Alpha', + 'C3-M2_kurtosis_Beta','C4-M1_kurtosis_Beta','F3-M2_kurtosis_Beta','F4-M1_kurtosis_Beta','C3-M2_kurtosis_Delta','C4-M1_kurtosis_Delta', + 'F3-M2_kurtosis_Delta','F4-M1_kurtosis_Delta','C3-M2_kurtosis_Sigma','C4-M1_kurtosis_Sigma','F3-M2_kurtosis_Sigma','F4-M1_kurtosis_Sigma', + 'C3-M2_kurtosis_Theta','C4-M1_kurtosis_Theta','F3-M2_kurtosis_Theta','F4-M1_kurtosis_Theta','C3-M2_variability_Delta', + 'C4-M1_variability_Delta','F3-M2_variability_Delta','F4-M1_variability_Delta','PIP_med','PIP_std','PNNSS_med', + 'PNNSS_std','AVNN_med','AVNN_std','SDNN_med','SDNN_std','RMSSD_std','HF_med','HF_std','ECTOPIC_med','ECTOPIC_std']] + + + # Ajusta el tipo de dato, todas deben ser numericas excepto Sex, Label y ID. Sex y label tendría que ser logical? o categorical? + df_model2['Sex'] = df_model2['Sex'].astype('category') + df_model2['label'] = df_model2['label'].astype(int) + + #Unificar los valores faltantes, que sean consistentes + df_model2 = df_model2.replace([-999, "NA", "null", "None", "nan", ""], np.nan) + # Eliminar filas con demasiados NaNs (mínimo 50% de datos válidos) + umbral = int(len(df_model2.columns) * 0.5) + df_model2 = df_model2.dropna(thresh=umbral) + + #Procesa los datos, imputar nans y estandarizar por la media y desviacion estandar (zscore) + y = df_model2['label'] + X = df_model2.drop(columns=[ 'label','ID']) # Excluimos ID, label del entrenamiento + + return {"X": X, + "y": y} + +def prepare_multimodal_data(dddf, testsize=0.1): + + proc_data=preprocess_multimodal_data(dddf) + X = proc_data["X"] + y = proc_data["y"] + X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=testsize, random_state=1999, stratify=y) + + cols_categoricas = ['Sex'] + cols_numericas = [c for c in X_train.columns if c not in cols_categoricas] + + preprocessor = ColumnTransformer([ + ("num", Pipeline([ + ("imputer", KNNImputer(n_neighbors=5)), + ("scaler", StandardScaler()) + ]), cols_numericas), + + ("cat", Pipeline([ + ("imputer", SimpleImputer(strategy="most_frequent")), + ("encoder", OneHotEncoder(handle_unknown="ignore", sparse_output=False)) + ]), cols_categoricas) + ]) + preprocessor.set_output(transform="pandas") + + X_train = preprocessor.fit_transform(X_train) + X_test = preprocessor.transform(X_test) + + feature_names = preprocessor.get_feature_names_out() + resp_mask = np.array([ any(k in col for k in ["Age","Sex","Nasal_Peakedness", "Chest_Peakedness", "Abdomen_Peakedness", "Flow_Peakedness", "SpO2", "CET90", "ODI" ]) + for col in feature_names]) + eeg_mask = np.array([ any(k in col for k in ["Age","Sex","C3-M2", "C4-M1", "F3-M2", "F4-M1"]) + for col in feature_names]) + ecg_mask = np.array([ any(k in col for k in ["Age","Sex","PIP_", "PNNSS_", "AVNN_", "SDNN_", "RMSSD_", "HF_", "ECTOPIC_"]) + for col in feature_names]) + + # Subsets finales + return { + "X_train": X_train, + "X_test": X_test, + "y_train": y_train, + "y_test": y_test, + "resp": (X_train.loc[:, resp_mask], X_test.loc[:, resp_mask]), + "eeg": (X_train.loc[:, eeg_mask], X_test.loc[:, eeg_mask]), + "ecg": (X_train.loc[:, ecg_mask], X_test.loc[:, ecg_mask]), + "preprocessor": preprocessor + } + +def _fit_ensembleJM(data): + + models = {} + y_train = data["y_train"] + y_test = data["y_test"] + + probas_test = [] + + # =============================== + # 1. Entrenamiento por modalidad + # =============================== + for modality in ['resp', 'eeg', 'ecg']: + + if modality not in data or data[modality] is None: + continue + + X_train, X_test = data[modality] + + if X_train.shape[1] == 0: + continue + + model = _build_xgb_model(y_train) + + model.fit( + X_train, + y_train, + eval_set=[(X_test, y_test)], + verbose=False + ) + + models[modality] = model + + proba = model.predict_proba(X_test)[:, 1] + probas_test.append(proba) + + if not probas_test: + raise ValueError("No hay modalidades válidas para entrenar.") + + # =============================== + # 2. Ensemble + # =============================== + proba_final = np.mean(probas_test, axis=0) + + # =============================== + # 3. Threshold (mejorado) + # =============================== + best_thr = best_threshold(proba_final, y_test) + + return { + "models": models, + "threshold": best_thr + } + +from sklearn.model_selection import StratifiedKFold + +def _fit_ensemble_with_cv(data, n_splits=5): + + X_train_full = data["X_train"] + y_train_full = data["y_train"] + + skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42) + + oof_probas = [] + oof_labels = [] + + for fold, (train_idx, val_idx) in enumerate(skf.split(X_train_full, y_train_full)): + + print(f"Fold {fold+1}/{n_splits}") + + probas_fold = [] + + for modality in ['resp', 'eeg', 'ecg']: + + X_train_mod, _ = data[modality] + + X_tr = X_train_mod.iloc[train_idx] + X_val = X_train_mod.iloc[val_idx] + + y_tr = y_train_full.iloc[train_idx] + y_val = y_train_full.iloc[val_idx] + + if X_tr.shape[1] == 0: + continue + + model = _build_xgb_model(y_tr) + + model.fit( + X_tr, + y_tr, + eval_set=[(X_val, y_val)], + verbose=False + ) + + proba = model.predict_proba(X_val)[:, 1] + probas_fold.append(proba) + + if not probas_fold: + continue + + # Ensemble por fold + proba_ensemble = np.mean(probas_fold, axis=0) + + oof_probas.extend(proba_ensemble) + oof_labels.extend(y_val) + + oof_probas = np.array(oof_probas) + oof_labels = np.array(oof_labels) + + # Threshold robusto (sin leakage) + best_thr = best_threshold(oof_probas, oof_labels) + + # =============================== + # Entrenar modelos finales (full data) + # =============================== + final_models = {} + + for modality in ['resp', 'eeg', 'ecg']: + + X_train_mod, X_test_mod = data[modality] + + if X_train_mod.shape[1] == 0: + continue + + model = _build_xgb_model(y_train_full) + + model.fit( + X_train_mod, + y_train_full, + eval_set=[(X_test_mod, data["y_test"])], + verbose=False + ) + + final_models[modality] = model + + return { + "models": final_models, + "threshold": best_thr + } + +def train_multimodal_ensemble(data_folder, verbose, csv_path, export_folder=None): + + # =============================== + # 1. CARGA Y EXTRACCIÓN + # =============================== + patient_data_file = os.path.join(data_folder, DEMOGRAPHICS_FILE) + patient_metadata_list = find_patients(patient_data_file) + + demographics_cache, diagnosis_cache = build_training_metadata_cache(patient_data_file) + num_records = len(patient_metadata_list) + + if num_records == 0: + raise FileNotFoundError('No data were provided.') + + features = [] + labels = [] + metadata_rows = [] + + with ThreadPoolExecutor(max_workers=MAX_TRAIN_WORKERS) as executor: + results = executor.map( + lambda record: process_training_record( + record, + data_folder, + demographics_cache, + diagnosis_cache, + csv_path, + ), + patient_metadata_list, + ) + + pbar = tqdm(results, total=num_records, desc='Extracting Features', + unit='record', disable=not verbose) + + for metadata, feature_vector, label, message in pbar: + + if verbose: + pbar.set_postfix({'patient': metadata['patient_id']}) + + if message is not None: + tqdm.write(f" ! {message}") + continue + + features.append(feature_vector) + labels.append(label) + metadata_rows.append(metadata) + + pbar.close() + + # =============================== + # 2. VALIDACIÓN + # =============================== + features = np.asarray(features, dtype=np.float32) + labels = np.asarray(labels, dtype=np.int32) + + if features.size == 0 or features.ndim != 2 or features.shape[0] == 0: + raise ValueError('No valid training samples were extracted.') + + feature_names = list(get_feature_names()) + + # =============================== + # 3. EXPORT RAW (opcional) + # =============================== + export_root = export_folder or os.path.join(os.getcwd(), 'feature_exports') + raw_feature_export_path = os.path.join(export_root, 'training_features_raw.csv') + + _export_feature_matrix_csv( + raw_feature_export_path, + metadata_rows, + labels, + features, + feature_names + ) + + # =============================== + # 2. CONVERTIR A DATAFRAME + df = pd.DataFrame(features, columns=feature_names) + df["label"] = labels + + data = prepare_multimodal_data(df, testsize=0.1) + preprocessor=data["preprocessor"] + + models, threshold = _fit_ensembleJM(data) + #result = _fit_ensemble_with_cv(data, n_splits=5) + #models = result["models"] + #threshold = result["threshold"] + + return { + 'type': 'multimodal_xgb_ensemble', + 'threshold': threshold, + 'feature_names': feature_names, + 'models': models, + 'preprocessor': preprocessor, + 'feature_exports': { + 'raw': raw_feature_export_path, + }, + } \ No newline at end of file diff --git a/src/resp_processing.py b/src/resp_processing.py new file mode 100644 index 0000000..11b474f --- /dev/null +++ b/src/resp_processing.py @@ -0,0 +1,141 @@ +from .lib import resp_features +import numpy as np +from src.common.channel_utils import get_cached_channel_table, normalize_channel_label, split_channel_aliases +from src.common.signal_utils import resample_signal + +RESP_CHANNEL_GROUPS = ("Abdomen", "Chest", "Nasal", "Flow") +RESP_SEGMENT_FEATURE_NAMES = [ + f"{group}_Peakedness" + for group in RESP_CHANNEL_GROUPS +] + [ + "SpO2", + "CET90", + "ODI", + "ODI_deepness", +] +RESP_SEGMENT_FEATURE_LENGTH = len(RESP_SEGMENT_FEATURE_NAMES) +RESP_FEATURE_NAMES = RESP_SEGMENT_FEATURE_NAMES +RESP_FEATURE_LENGTH = RESP_SEGMENT_FEATURE_LENGTH +RESP_ALIAS_GROUPS_CACHE = {} + + +def _build_resp_alias_groups(channels): + resp_rows = channels[channels['Category'].eq('resp')].reset_index(drop=True) + if len(resp_rows) < 7: + return {} + return { + 'Abdomen': split_channel_aliases(resp_rows.iloc[0]['Channel_Names']), + 'Chest': split_channel_aliases(resp_rows.iloc[1]['Channel_Names']), + 'Nasal': split_channel_aliases(resp_rows.iloc[2]['Channel_Names']), + 'Flow': split_channel_aliases(resp_rows.iloc[3]['Channel_Names']), + 'SpO2': split_channel_aliases(resp_rows.iloc[6]['Channel_Names']), + } + + +def _get_resp_alias_groups(csv_path): + channels, normalized_csv_path = get_cached_channel_table(csv_path) + alias_groups = RESP_ALIAS_GROUPS_CACHE.get(normalized_csv_path) + if alias_groups is None: + alias_groups = _build_resp_alias_groups(channels) + RESP_ALIAS_GROUPS_CACHE[normalized_csv_path] = alias_groups + return alias_groups + + +def _find_resp_group(label, alias_groups): + normalized = normalize_channel_label(label) + for group_name, aliases in alias_groups.items(): + if normalized in aliases: + return group_name + return None + + +def _compute_resp_quality(used, hat_br): + used_array = np.asarray(used, dtype=float) + if used_array.size: + quality = float(np.nanmean(used_array)) + if np.isfinite(quality): + return quality + hat_br = np.asarray(hat_br, dtype=float) + if hat_br.size == 0: + return 0.0 + return float(np.mean(np.isfinite(hat_br))) + + +def _compute_peakedness_metric(hat_br): + finite_values = np.asarray(hat_br, dtype=float) + finite_values = finite_values[np.isfinite(finite_values)] + if finite_values.size == 0: + return np.nan + return float(np.mean(finite_values)) + + +def _compute_spo2_segment_metrics(data, fs): + if data.size == 0: + return {} + working = np.asarray(data, dtype=float).copy() + if np.nanmax(working) < 2: + working = np.round((working / 1.055) * 100) + + desaturation_mask = working.copy() + threshold = 0.7 + for index, value in enumerate(working): + if value < threshold: + start = int(max(0, index - fs * 2)) + end = int(min(working.size, index + fs * 2)) + desaturation_mask[start:end] = np.nan + + cet90 = float(np.count_nonzero(desaturation_mask < 90) / max(working.size, 1)) + valid = desaturation_mask[np.isfinite(desaturation_mask)] + if valid.size == 0: + return {'CET90': cet90} + + odi_mean, odi_deepness = resp_features.odi_application(desaturation_mask, fs) + return { + 'SpO2': float(np.mean(valid)), + 'CET90': cet90, + 'ODI': float(odi_mean), + 'ODI_deepness': float(odi_deepness), + } + + +def processResp(physiological_data, physiological_fs, csv_path): + alias_groups = _get_resp_alias_groups(csv_path) + results = {feature_name: np.nan for feature_name in RESP_SEGMENT_FEATURE_NAMES} + best_quality = {group_name: -np.inf for group_name in RESP_CHANNEL_GROUPS} + + for label, signal in physiological_data.items(): + if label not in physiological_fs: + continue + + group_name = _find_resp_group(label, alias_groups) + if group_name is None: + continue + + resampled, fs = resample_signal(signal, physiological_fs[label], 25) + resampled = np.nan_to_num(resampled, nan=0.0, posinf=0.0, neginf=0.0) + + if group_name == 'SpO2': + results.update(_compute_spo2_segment_metrics(resampled, fs)) + continue + + try: + hat_br, _, _, used = resp_features.peakedness_application( + resampled, + stage=label, + subject_id=label, + ) + except Exception: + continue + + peakedness_metric = _compute_peakedness_metric(hat_br) + if not np.isfinite(peakedness_metric): + continue + + quality = _compute_resp_quality(used, hat_br) + if quality <= best_quality[group_name]: + continue + + best_quality[group_name] = quality + results[f'{group_name}_Peakedness'] = peakedness_metric + + return np.array([results[name] for name in RESP_SEGMENT_FEATURE_NAMES], dtype=np.float32) diff --git a/team_code.py b/team_code.py index 910c4ed..39abf01 100644 --- a/team_code.py +++ b/team_code.py @@ -1,5 +1,6 @@ #!/usr/bin/env python - +# -d data/test_set -m model -o outputs -v +# -d data/training_set -m model -v # Edit this script to add your team's code. Some functions are *required*, but you can edit most parts of the required functions, # change or remove non-required functions, and add your own functions. @@ -10,23 +11,88 @@ ################################################################################ import joblib -import numpy as np import os -from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor +import atexit +import builtins +import re import sys + +import numpy as np from tqdm import tqdm from helper_code import * - +from src.pipeline.config import DEFAULT_CSV_PATH +from src.pipeline.features import get_feature_export_dir, get_or_create_record_feature_vector +from src.pipeline.training import ( + export_feature_views, + predict_ensemble_labels, + train_multimodal_ensemble, +) ################################################################################ # Path & Constant Configuration (Added for Robustness) ################################################################################ -# Get the absolute directory where this script is located -SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +# Progress bar state for run_model (initialized lazily) +RUN_MODEL_PBAR = None +RUN_MODEL_PBAR_TOTAL = None +RUN_MODEL_EXPORT_STATE = None +ORIGINAL_PRINT = builtins.print +PRINT_FILTER_ACTIVE = False +RUN_PROGRESS_LINE_RE = re.compile(r'^-\s+\d+/\d+:\s') +def _close_run_model_pbar(): + global RUN_MODEL_PBAR + if RUN_MODEL_PBAR is not None: + RUN_MODEL_PBAR.close() + RUN_MODEL_PBAR = None + + +def _flush_run_feature_exports(): + global RUN_MODEL_EXPORT_STATE + + if RUN_MODEL_EXPORT_STATE is None or not RUN_MODEL_EXPORT_STATE['metadata_rows']: + return + + feature_exports = export_feature_views( + RUN_MODEL_EXPORT_STATE['export_root'], + 'test', + RUN_MODEL_EXPORT_STATE['metadata_rows'], + np.asarray(RUN_MODEL_EXPORT_STATE['raw_features'], dtype=np.float32), + RUN_MODEL_EXPORT_STATE['feature_names'], + preprocessor=RUN_MODEL_EXPORT_STATE['preprocessor'], + ) + + if RUN_MODEL_EXPORT_STATE['verbose']: + print(f"Raw features CSV: {feature_exports.get('raw')}") + print(f"Preprocessed features CSV: {feature_exports.get('preprocessed')}") -# Build the absolute path to the CSV file relative to the script location -DEFAULT_CSV_PATH = os.path.join(SCRIPT_DIR, 'channel_table.csv') + RUN_MODEL_EXPORT_STATE = None + + +def _install_run_print_filter(): + global PRINT_FILTER_ACTIVE + if PRINT_FILTER_ACTIVE: + return + + def _filtered_print(*args, **kwargs): + message = kwargs.get('sep', ' ').join(str(a) for a in args) if args else '' + if RUN_PROGRESS_LINE_RE.match(message): + return + return ORIGINAL_PRINT(*args, **kwargs) + + builtins.print = _filtered_print + PRINT_FILTER_ACTIVE = True + + +def _restore_print(): + global PRINT_FILTER_ACTIVE + if PRINT_FILTER_ACTIVE: + builtins.print = ORIGINAL_PRINT + PRINT_FILTER_ACTIVE = False + + +atexit.register(_close_run_model_pbar) +atexit.register(_flush_run_feature_exports) +atexit.register(_restore_print) ################################################################################ @@ -44,105 +110,29 @@ def train_model(data_folder, model_folder, verbose, csv_path=DEFAULT_CSV_PATH): if verbose: print('Finding the Challenge data...') - patient_data_file = os.path.join(data_folder, DEMOGRAPHICS_FILE) - patient_metadata_list = find_patients(patient_data_file) - num_records = len(patient_metadata_list) - - if num_records == 0: - raise FileNotFoundError('No data were provided.') - - # Extract the features and labels from the data. - if verbose: - print('Extracting features and labels from the data...') - - # Iterate over the records to extract the features and labels. - features = list() - labels = list() - - pbar = tqdm(range(num_records), desc="Extracting Features", unit="record", disable=not verbose) - for i in pbar: - try: - # Extract identifiers for this specific record - record = patient_metadata_list[i] - patient_id = record[HEADERS['bids_folder']] - site_id = record[HEADERS['site_id']] - session_id = record[HEADERS['session_id']] - - if verbose: - pbar.set_postfix({"patient": patient_id}) - - # Load the patient data. - patient_data_file = os.path.join(data_folder, DEMOGRAPHICS_FILE) - patient_data = load_demographics(patient_data_file, patient_id, session_id) - demographic_features = extract_demographic_features(patient_data) - - # Load signal data. - - # Load the physiological signal. - physiological_data_file = os.path.join(data_folder, PHYSIOLOGICAL_DATA_SUBFOLDER, site_id, f"{patient_id}_ses-{session_id}.edf") - # --- Check if the file actually exists before proceeding --- - if not os.path.exists(physiological_data_file): - if verbose: - print(f" ! Missing physiological data for {patient_id}. Skipping...") - continue # skip record - physiological_data, physiological_fs = load_signal_data(physiological_data_file) - physiological_features = extract_physiological_features(physiological_data, physiological_fs, csv_path=csv_path) # This function can rename, re-reference, resample, etc. the signal data. - - # Load the algorithmic annotations. - algorithmic_annotations_file = os.path.join(data_folder, ALGORITHMIC_ANNOTATIONS_SUBFOLDER, site_id, f"{patient_id}_ses-{session_id}_caisr_annotations.edf") - algorithmic_annotations, algorithmic_fs = load_signal_data(algorithmic_annotations_file) - algorithmic_features = extract_algorithmic_annotations_features(algorithmic_annotations) - - # Load the human annotations; these data will not be available in the hidden validation and test sets. - human_annotations_file = os.path.join(data_folder, HUMAN_ANNOTATIONS_SUBFOLDER, site_id, f"{patient_id}_ses-{session_id}_expert_annotations.edf") - human_annotations, human_fs = load_signal_data(human_annotations_file) - human_features = extract_human_annotations_features(human_annotations) - - # Load the diagnoses; these data will not be available in the hidden validation and test sets. - diagnosis_file = os.path.join(data_folder, DEMOGRAPHICS_FILE) - label = load_diagnoses(diagnosis_file, patient_id) - - # Store the features and labels, but - # the human annotations are not available on the hidden validation and test sets, but you - # may want to consider how to use them for training. - if label == 0 or label == 1: - features.append(np.hstack([demographic_features, physiological_features, algorithmic_features])) - labels.append(label) - - if 'physiological_data' in locals(): del physiological_data - if 'algorithmic_annotations' in locals(): del algorithmic_annotations - - except Exception as e: - # If an error occurs (e.g., a record is corrupted), log it and move to the next - tqdm.write(f" !!! Error processing record {i+1} ({patient_id}): {e}") - continue - - pbar.close() - - features = np.asarray(features, dtype=np.float32) - labels = np.asarray(labels, dtype=bool) - - # Train the models on the features. if verbose: print('Training the model on the data...') - # This very simple model trains a random forest model with very simple features. - - # Define the parameters for the random forest classifier and regressor. - n_estimators = 12 # Number of trees in the forest. - max_leaf_nodes = 34 # Maximum number of leaf nodes in each tree. - random_state = 56 # Random state; set for reproducibility. - - # Fit the model. - model = RandomForestClassifier( - n_estimators=n_estimators, max_leaf_nodes=max_leaf_nodes, random_state=random_state).fit(features, labels) - # Create a folder for the model if it does not already exist. os.makedirs(model_folder, exist_ok=True) + model = train_multimodal_ensemble( + data_folder, + verbose, + csv_path, + export_folder=get_feature_export_dir(data_folder), + ) + # Save the model. save_model(model_folder, model) + feature_exports = model.get('feature_exports', {}) + if verbose and feature_exports: + print(f"Raw features CSV: {feature_exports.get('raw')}") + print(f"Preprocessed features CSV: {feature_exports.get('preprocessed')}") + if feature_exports.get('selected'): + print(f"Selected features CSV: {feature_exports.get('selected')}") + if verbose: print('Done.') print() @@ -150,6 +140,9 @@ def train_model(data_folder, model_folder, verbose, csv_path=DEFAULT_CSV_PATH): # Load your trained models. This function is *required*. You should edit this function to add your code, but do *not* change the # arguments of this function. If you do not train one of the models, then you can return None for the model. def load_model(model_folder, verbose): + if verbose: + _install_run_print_filter() + model_filename = os.path.join(model_folder, 'model.sav') model = joblib.load(model_filename) return model @@ -157,6 +150,8 @@ def load_model(model_folder, verbose): # Run your trained model. This function is *required*. You should edit this function to add your code, but do *not* change the # arguments of this function. def run_model(model, record, data_folder, verbose): + global RUN_MODEL_PBAR, RUN_MODEL_PBAR_TOTAL, RUN_MODEL_EXPORT_STATE + # Load the model. model = model['model'] @@ -165,382 +160,76 @@ def run_model(model, record, data_folder, verbose): site_id = record[HEADERS['site_id']] session_id = record[HEADERS['session_id']] + # Initialize tqdm progress bar lazily so it advances across run_model calls. + if verbose and RUN_MODEL_PBAR is None: + patient_data_file = os.path.join(data_folder, DEMOGRAPHICS_FILE) + try: + RUN_MODEL_PBAR_TOTAL = len(find_patients(patient_data_file)) + except Exception: + RUN_MODEL_PBAR_TOTAL = None + + RUN_MODEL_PBAR = tqdm( + total=RUN_MODEL_PBAR_TOTAL, + desc="Running Model", + unit="record", + leave=True, + file=sys.stdout, + delay=0.5, + disable=not verbose + ) + + if RUN_MODEL_EXPORT_STATE is None: + export_root = get_feature_export_dir(data_folder) + RUN_MODEL_EXPORT_STATE = { + 'export_root': export_root, + 'metadata_rows': [], + 'raw_features': [], + 'feature_names': list(model.get('feature_names', [])), + 'preprocessor': model.get('preprocessor'), + 'verbose': verbose, + } + + if verbose and RUN_MODEL_PBAR is not None: + RUN_MODEL_PBAR.set_postfix({"patient": patient_id}) + # Load the patient data. patient_data_file = os.path.join(data_folder, DEMOGRAPHICS_FILE) patient_data = load_demographics(patient_data_file, patient_id, session_id) - demographic_features = extract_demographic_features(patient_data) - - # Load signal data. - phys_file = os.path.join(data_folder, PHYSIOLOGICAL_DATA_SUBFOLDER, site_id, f"{patient_id}_ses-{session_id}.edf") - if os.path.exists(phys_file): - phys_data, phys_fs = load_signal_data(phys_file) - # Ensure csv_path is accessible or defined - physiological_features = extract_physiological_features(phys_data, phys_fs) - else: - # Fallback to zeros if file is missing (length 49) - physiological_features = np.zeros(49) - - # Load Algorithmic Annotations - algo_file = os.path.join(data_folder, ALGORITHMIC_ANNOTATIONS_SUBFOLDER, site_id, f"{patient_id}_ses-{session_id}_caisr_annotations.edf") - if os.path.exists(algo_file): - algo_data, _ = load_signal_data(algo_file) - algorithmic_features = extract_algorithmic_annotations_features(algo_data) - else: - # Fallback to zeros (length 12) - algorithmic_features = np.zeros(12) - - features = np.hstack([demographic_features, physiological_features, algorithmic_features]).reshape(1, -1) + features = get_or_create_record_feature_vector( + record, + data_folder, + patient_data, + csv_path=DEFAULT_CSV_PATH, + require_physiological_data=False, + ) + features = np.asarray(features, dtype=np.float32).reshape(1, -1) + + RUN_MODEL_EXPORT_STATE['metadata_rows'].append({ + 'patient_id': patient_id, + 'site_id': site_id, + 'session_id': session_id, + }) + RUN_MODEL_EXPORT_STATE['raw_features'].append(features[0]) # Get the model outputs. - binary_output = model.predict(features)[0] - probability_output = model.predict_proba(features)[0][1] - - return binary_output, probability_output + labels, probabilities = predict_ensemble_labels(model, features) + binary_output = bool(int(labels[0])) + probability_output = float(probabilities[0]) -################################################################################ -# -# Optional functions. You can change or remove these functions and/or add new functions. -# -################################################################################ + if verbose and RUN_MODEL_PBAR is not None: + RUN_MODEL_PBAR.update(1) + if RUN_MODEL_PBAR_TOTAL is not None and RUN_MODEL_PBAR.n >= RUN_MODEL_PBAR_TOTAL: + RUN_MODEL_PBAR.close() + RUN_MODEL_PBAR = None -def extract_demographic_features(data): - """ - Extracts and encodes demographic features from a metadata dictionary. - - Inputs: - data (dict): A dictionary containing patient metadata (e.g., from a CSV row). - - Returns: - np.array: A feature vector of length 11: - - [0]: Age (Continuous) - - [1:4]: Sex (One-hot: Female, Male, Other/Unknown) - - [4:9]: Race (One-hot: Asian, Black, Other, Unavailable, White) - - [9]: BMI (Continuous) - """ - # 1. Age Feature (1 dimension) - # Convert 'Age' to a float; default to 0 if missing - age = np.array([load_age(data)]) - - # 2. Sex One-Hot Encoding (3 dimensions: Female, Male, Other/Unknown) - # Uses lowercase prefix matching to handle variants like 'F', 'Female', 'M', or 'Male' - sex = load_sex(data) - sex_vec = np.zeros(3) - if sex == 'Female': - sex_vec[0] = 1 # Index 0: Female - elif sex == 'Male': - sex_vec[1] = 1 # Index 1: Male - else: - sex_vec[2] = 1 # Index 2: Other/Unknown - - # 3. Race One-Hot Encoding (6 dimensions) - # Standardizes the raw text into one of six categories using the helper function - race_category = get_standardized_race(data).lower() - race_vec = np.zeros(5) - # Pre-defined mapping for index consistency - race_mapping = {'asian': 0, 'black': 1, 'others': 2, 'unavailable': 3, 'white': 4} - race_vec[race_mapping.get(race_category, 2)] = 1 - - # 4. Body Mass Index (BMI) Feature (1 dimension) - # Extracts the pre-calculated mean BMI; handles strings, NaNs, and missing keys - bmi = np.array([load_bmi(data)]) - - # 5. Concatenate all components into a single vector (1 + 3 + 5 + 1 = 10) - - return np.concatenate([age, sex_vec, race_vec, bmi]) - - -def extract_physiological_features(physiological_data, physiological_fs, csv_path=DEFAULT_CSV_PATH): - """ - Standardizes channels and extracts statistical/spectral features. - """ - original_labels = list(physiological_data.keys()) - - # Step 1: Load rules and standardize names - # Note: Use script-relative path or absolute path for robustness - rename_rules = load_rename_rules(os.path.abspath(csv_path)) - rename_map, cols_to_drop = standardize_channel_names_rename_only(original_labels, rename_rules) - - # Step 2: Apply renaming to BOTH signals and their corresponding FS - processed_channels = {} - processed_fs = {} - for old_label, data in physiological_data.items(): - if old_label in cols_to_drop: - continue - new_label = rename_map.get(old_label, old_label.lower()) - processed_channels[new_label] = data - # Mapping the sampling rate to the new label - if old_label in physiological_fs: - processed_fs[new_label] = physiological_fs[old_label] - else: - # Report error and stop if no FS is found for a kept channel - raise KeyError(f"Sampling frequency (fs) not found for channel '{old_label}' ") - - if 'physiological_data' in locals(): del physiological_data - - # Step 3: Construct Bipolar Derivations - bipolar_configs = [ - ('f3-m2', 'f3', ['m2']), ('f4-m1', 'f4', ['m1']), - ('c3-m2', 'c3', ['m2']), ('c4-m1', 'c4', ['m1']), - ('o1-m2', 'o1', ['m2']), ('o2-m1', 'o2', ['m1']), - ('e1-m2', 'e1', ['m2']), ('e2-m1', 'e2', ['m1']), - ('chin1-chin2', 'chin 1', ['chin 2']), - ('lat', 'lleg+', ['lleg-']), ('rat', 'rleg+', ['rleg-']) - ] - - for target, pos, neg_list in bipolar_configs: - # 1. Skip if target already exists or pos channel missing - if target in processed_channels or pos not in processed_channels: - continue - - # 2. Check all neg channels exist - if not all(n in processed_channels for n in neg_list): - continue - - # 3. Check sampling rate consistency - all_involved = [pos] + neg_list - fs_values = [processed_fs[ch] for ch in all_involved] - - if len(set(fs_values)) > 1: - raise ValueError(f"Sampling rate mismatch for {target}: {dict(zip(all_involved, fs_values))}") - - # 4. Derive bipolar signal - ref_sig = processed_channels[neg_list[0]] if len(neg_list) == 1 else tuple(processed_channels[n] for n in neg_list) - - derived = derive_bipolar_signal(processed_channels[pos], ref_sig) - - if derived is not None: - processed_channels[target] = derived - processed_fs[target] = processed_fs[pos] - - leads_to_check = { - 'eeg': ['f3-m2', 'f4-m1', 'c3-m2', 'c4-m1'], - 'eog': ['e1-m2', 'e2-m1'], - 'chin': ['chin1-chin2', 'chin'], - 'leg': ['lat', 'rat'], - 'ecg': ['ecg', 'ekg'], - 'resp': ['airflow', 'ptaf', 'abd', 'chest'], - 'spo2': ['spo2', 'sao2'] # Added sao2 as fallback for spo2 - } - - final_features = [] - for lead_type, candidates in leads_to_check.items(): - sig = None - fs = None - - # Identify the first available candidate - for candidate in candidates: - if candidate in processed_channels and processed_channels[candidate] is not None: - sig = processed_channels[candidate] - fs = processed_fs.get(candidate) - break - - # if sig is not None and len(sig) > 0 and fs is not None: - # # --- 1. Time Domain Features --- - # std_val = np.std(sig) - # mav_val = np.mean(np.abs(sig)) - # energy_val = np.sum(sig**2) / len(sig) - - # # --- 2. Frequency Domain Features (Spectral) --- - # n = len(sig) - # # Correct spacing for frequency axis based on channel-specific fs - # freqs = np.fft.rfftfreq(n, d=1/fs) - - # # Compute Power Spectral Density (PSD) - # # Multiplied by 2 for rfft (except DC/Nyquist) and divided by fs for density - # fft_res = np.abs(np.fft.rfft(sig)) - # psd = (fft_res**2) / (n * fs) - - # # Define band masks - # delta_mask = (freqs >= 0.5) & (freqs <= 4) - # theta_mask = (freqs > 4) & (freqs <= 8) - # alpha_mask = (freqs > 8) & (freqs <= 12) - - # # Calculate power in bands using trapezoidal integration for physical accuracy - # delta_p = np.trapezoid(psd[delta_mask], freqs[delta_mask]) if np.any(delta_mask) else 0.0 - # theta_p = np.trapezoid(psd[theta_mask], freqs[theta_mask]) if np.any(theta_mask) else 0.0 - # alpha_p = np.trapezoid(psd[alpha_mask], freqs[alpha_mask]) if np.any(alpha_mask) else 0.0 - - # # Ratio biomarker: Delta/Theta (Indicator of cognitive slowing) - # dt_ratio = delta_p / theta_p if theta_p > 0 else 0.0 - - # final_features.extend([std_val, mav_val, energy_val, delta_p, theta_p, alpha_p, dt_ratio]) - - if sig is not None and len(sig) > 1: - # --- Time Domain Features (Very Fast) --- - std_val = np.std(sig) - mav_val = np.mean(np.abs(sig)) - - # Zero Crossing Rate (Proxy for frequency/slowing) - zcr = np.mean(np.diff(np.sign(sig)) != 0) - - # Root Mean Square - rms = np.sqrt(np.mean(sig**2)) - - # Signal Activity (Variance) - activity = np.var(sig) - - # Mobility (Hjorth Parameter) - Proxy for mean frequency - # sqrt(var(diff(sig)) / var(sig)) - diff_sig = np.diff(sig) - mobility = np.sqrt(np.var(diff_sig) / activity) if activity > 0 else 0.0 - - # Complexity (Hjorth Parameter) - Proxy for bandwidth - diff2_sig = np.diff(diff_sig) - var_d2 = np.var(diff2_sig) - var_d1 = np.var(diff_sig) - complexity = (np.sqrt(var_d2 / var_d1) / mobility) if (var_d1 > 0 and mobility > 0) else 0.0 - - final_features.extend([std_val, mav_val, zcr, rms, activity, mobility, complexity]) - - else: - # Padding: 7 features per lead type - final_features.extend([0.0] * 7) - - if 'processed_channels' in locals(): del processed_channels - - return np.array(final_features) - -def extract_algorithmic_annotations_features(algo_data): - """ - Extracts sleep architecture and event density features from CAISR outputs. - Output vector length: 12 - """ - if not algo_data: - return np.zeros(12) - - features = [] - - # --- 1. Respiratory & Arousal Event Densities --- - # Total duration in hours (assuming 1Hz for event traces) - # If the signal exists, we calculate events per hour (Index) - total_hours = len(algo_data.get('resp_caisr', [])) / 3600.0 - - def count_discrete_events(key): - if key not in algo_data or total_hours <= 0: - return 0.0 - - sig = algo_data[key].astype(float) - # Create a binary mask: 1 if there is an event, 0 if not - binary_sig = (sig > 0).astype(int) - - # Detect rising edges: 0 to 1 transition - # diff will be 1 at the start of an event, -1 at the end - diff = np.diff(binary_sig, prepend=0) - num_events = np.count_nonzero(diff == 1) - - return num_events / total_hours - - ahi_auto = count_discrete_events('resp_caisr') # Automated Apnea-Hypopnea Index - arousal_auto = count_discrete_events('arousal_caisr') # Automated Arousal Index - limb_auto = count_discrete_events('limb_caisr') # Automated Limb Movement Index - - features.extend([ahi_auto, arousal_auto, limb_auto]) - - # --- 2. Sleep Architecture (from stage_caisr) --- - # Standard labels: 5=W, 4=R, 3=N1, 2=N2, 1=N3 (or similar mapping) - stages = algo_data.get('stage_caisr', np.array([])) - # Filter out invalid/background values (like the 9.0 in your sample) - valid_stages = stages[stages < 9.0] - - if len(valid_stages) > 0: - total_epochs = len(valid_stages) - # Percentage of each stage - w_pct = np.mean(valid_stages == 5) - r_pct = np.mean(valid_stages == 4) - n1_pct = np.mean(valid_stages == 3) - n2_pct = np.mean(valid_stages == 2) - n3_pct = np.mean(valid_stages == 1) - - # Sleep Efficiency: (N1+N2+N3+R) / Total - efficiency = np.mean((valid_stages >= 1) & (valid_stages <= 4)) - else: - w_pct = n1_pct = n2_pct = n3_pct = r_pct = efficiency = 0.0 - - features.extend([w_pct, n1_pct, n2_pct, n3_pct, r_pct, efficiency]) - - # --- 3. Model Confidence / Uncertainty --- - # Mean probability of Wake and REM (indicators of sleep stability) - # We use the raw probability traces - prob_w = np.mean(algo_data.get('caisr_prob_w', [0])) - prob_n3 = np.mean(algo_data.get('caisr_prob_n3', [0])) - prob_arous = np.mean(algo_data.get('caisr_prob_arous', [0])) - - # Standardize '9.0' or other filler values to 0 - clean_prob = lambda x: x if x < 1.0 else 0.0 - features.extend([clean_prob(prob_w), clean_prob(prob_n3), clean_prob(prob_arous)]) - - return np.array(features) - -def extract_human_annotations_features(human_data): - """ - Extracts features from expert-scored human annotations. - Output vector length: 12 (to match algorithmic feature length) - """ - # If data is missing (common in hidden test sets), return a zero vector - if not human_data or 'resp_expert' not in human_data: - return np.zeros(12) - - features = [] - - # --- 1. Human Event Indices (Events per Hour) --- - # Total duration in hours based on 1Hz signal - total_seconds = len(human_data.get('resp_expert', [])) - total_hours = total_seconds / 3600.0 - - def count_discrete_events(key): - if key not in human_data or total_hours <= 0: - return 0.0 - sig = (human_data[key] > 0).astype(int) - # Identify the start of each continuous event block - diff = np.diff(sig, prepend=0) - return np.count_nonzero(diff == 1) / total_hours - - ahi_human = count_discrete_events('resp_expert') # Human AHI - arousal_human = count_discrete_events('arousal_expert') # Human Arousal Index - limb_human = count_discrete_events('limb_expert') # Human PLMI - - features.extend([ahi_human, arousal_human, limb_human]) - - # --- 2. Human Sleep Architecture --- - # Standard labels: 0=W, 1=N1, 2=N2, 3=N3, 4=R, 5=Unknown/Movement - stages = human_data.get('stage_expert', np.array([])) - - # Filter out label 5 (often used by experts for movement/unscored) - valid_mask = (stages < 9.0) - valid_stages = stages[valid_mask] - - if len(valid_stages) > 0: - w_pct = np.mean(valid_stages == 5) - r_pct = np.mean(valid_stages == 4) - n1_pct = np.mean(valid_stages == 3) - n2_pct = np.mean(valid_stages == 2) - n3_pct = np.mean(valid_stages == 1) - efficiency = np.mean(valid_stages > 0) - else: - w_pct = n1_pct = n2_pct = n3_pct = r_pct = efficiency = 0.0 - - features.extend([w_pct, n1_pct, n2_pct, n3_pct, r_pct, efficiency]) - - # --- 3. Fragmentation & Stability (Replacing Probabilities) --- - # These metrics quantify how "broken" the sleep is, which is a key marker. - if len(valid_stages) > 1: - # Number of stage transitions - transitions = np.count_nonzero(np.diff(valid_stages)) / total_hours - # Wake After Sleep Onset (WASO) proxy: non-zero stages followed by zero - waso_minutes = (np.count_nonzero(valid_stages == 0) * 30) / 60.0 - # REM Latency (epochs until first REM) - rem_indices = np.where(valid_stages == 4)[0] - rem_latency = rem_indices[0] if len(rem_indices) > 0 else 0.0 - else: - transitions = waso_minutes = rem_latency = 0.0 - - features.extend([transitions, waso_minutes, rem_latency]) - - return np.array(features) + if RUN_MODEL_PBAR_TOTAL is not None and RUN_MODEL_EXPORT_STATE is not None: + if len(RUN_MODEL_EXPORT_STATE['metadata_rows']) >= RUN_MODEL_PBAR_TOTAL: + _flush_run_feature_exports() + return binary_output, probability_output # Save your trained model. def save_model(model_folder, model): d = {'model': model} filename = os.path.join(model_folder, 'model.sav') - joblib.dump(d, filename, protocol=0) \ No newline at end of file + joblib.dump(d, filename, protocol=0) diff --git a/tests/test_eeg_slow_wave_features.py b/tests/test_eeg_slow_wave_features.py new file mode 100644 index 0000000..fda5c30 --- /dev/null +++ b/tests/test_eeg_slow_wave_features.py @@ -0,0 +1,169 @@ +from contextlib import redirect_stdout +import io +import unittest +from unittest.mock import patch + +import numpy as np + +from src import eeg_processing +from src.lib import eeg_features +from src.lib.swa.swa_FindSWRef import swa_FindSWRef +from src.lib.swa.swa_getInfoDefaults import swa_getInfoDefaults + + +class SlowWaveFeatureTests(unittest.TestCase): + def test_summarize_slow_waves_returns_fixed_numeric_features(self): + waves = [ + { + 'Ref_DownInd': 10, + 'Ref_UpInd': 110, + 'Ref_PeakAmp': -80, + 'Ref_P2PAmp': 130, + 'Ref_NegSlope': -4, + 'Ref_PosSlope': 5, + }, + { + 'Ref_DownInd': 200, + 'Ref_UpInd': 350, + 'Ref_PeakAmp': -120, + 'Ref_P2PAmp': 190, + 'Ref_NegSlope': -8, + 'Ref_PosSlope': 9, + }, + ] + + features = eeg_features.summarize_slow_waves( + waves, fs=100, signal_duration_seconds=120 + ) + + self.assertEqual(tuple(features), eeg_features.SLOW_WAVE_FEATURE_NAMES) + self.assertEqual(features['TotalSW'], 2.0) + self.assertEqual(features['SWdensity'], 1.0) + self.assertEqual(features['SWpeakAmp_mean'], -100.0) + self.assertEqual(features['SWpeakAmp_std'], 20.0) + self.assertEqual(features['SWp2p_mean'], 160.0) + self.assertEqual(features['SWnegSlope_mean'], -6.0) + self.assertEqual(features['SWposSlope_mean'], 7.0) + self.assertAlmostEqual(features['SWduration_mean'], 1.25) + self.assertAlmostEqual(features['SWduration_std'], 0.25) + + def test_summarize_no_waves_distinguishes_absence_from_missing_morphology(self): + features = eeg_features.summarize_slow_waves( + [], fs=200, signal_duration_seconds=300 + ) + + self.assertEqual(features['TotalSW'], 0.0) + self.assertEqual(features['SWdensity'], 0.0) + for name in eeg_features.SLOW_WAVE_FEATURE_NAMES[2:]: + self.assertTrue(np.isnan(features[name]), name) + + def test_swa_reference_detector_uses_channels_by_samples_orientation(self): + fs = 100 + seconds = 20 + time = np.arange(fs * seconds) / fs + reference = 100.0 * np.sin(2 * np.pi * time) + + info = swa_getInfoDefaults({}, 'SW', method='envelope') + info['Recording'] = {'sRate': fs} + info['Parameters']['Ref_InspectionPoint'] = 'ZC' + info['Parameters']['Ref_AmplitudeCriteria'] = 'absolute' + info['Parameters']['Ref_AmplitudeAbsolute'] = 50.0 + data = {'SWRef': reference[np.newaxis, :]} + + with redirect_stdout(io.StringIO()): + _, _, waves = swa_FindSWRef(data, info) + + self.assertGreater(len(waves), 0) + self.assertTrue(all(0 <= wave['Ref_PeakInd'] < reference.size for wave in waves)) + + def test_get_sw_features_detects_a_synthetic_slow_wave_signal(self): + fs = 100 + time = np.arange(fs * 20) / fs + signal = 100.0 * np.sin(2 * np.pi * time) + + features = eeg_features.get_SW_features(signal, fs) + + self.assertGreater(features['TotalSW'], 10) + self.assertGreater(features['SWdensity'], 30) + self.assertLess(features['SWpeakAmp_mean'], -80) + self.assertGreater(features['SWp2p_mean'], 160) + self.assertTrue(0.4 < features['SWduration_mean'] < 0.6) + + def test_get_sw_features_aggregates_swa_events(self): + fs = 100 + signal = np.zeros(fs * 60, dtype=float) + waves = [{ + 'Ref_DownInd': 100, + 'Ref_UpInd': 180, + 'Ref_PeakInd': 140, + 'Ref_PeakAmp': -75, + 'Ref_P2PAmp': 125, + 'Ref_NegSlope': -3, + 'Ref_PosSlope': 4, + }] + + with ( + patch.object( + eeg_features.swa_CalculateReference, + 'swa_CalculateReference', + return_value=(signal[np.newaxis, :], { + 'Recording': {'sRate': fs}, + 'Parameters': {}, + 'Electrodes': ['EEG'], + }), + ), + patch.object( + eeg_features.swa_FindSWRef, + 'swa_FindSWRef', + side_effect=lambda data, info: (data, info, waves), + ), + patch.object( + eeg_features.swa_FindSWChannels, + 'swa_FindSWChannels', + side_effect=lambda data, info, detected, flag_progress: ( + data, info, detected + ), + ), + ): + features = eeg_features.get_SW_features(signal, fs) + + self.assertEqual(tuple(features), eeg_features.SLOW_WAVE_FEATURE_NAMES) + self.assertEqual(features['TotalSW'], 1.0) + self.assertEqual(features['SWdensity'], 1.0) + self.assertEqual(features['SWpeakAmp_mean'], -75.0) + + def test_sw_failure_does_not_discard_other_channel_metrics(self): + fs = 200 + time = np.arange(fs * 30) / fs + signal = ( + 20.0 * np.sin(2 * np.pi * 10 * time) + + 5.0 * np.sin(2 * np.pi * time) + ) + + with patch.object( + eeg_features, + 'get_SW_features', + side_effect=RuntimeError('detector failed'), + ): + metrics = eeg_processing._extract_channel_metrics(signal, fs) + + self.assertIsNotNone(metrics) + self.assertTrue(np.isfinite(metrics['Relative_Delta_Power'])) + for name in eeg_features.SLOW_WAVE_FEATURE_NAMES: + self.assertTrue(np.isnan(metrics[name]), name) + + def test_all_slow_wave_features_are_exposed_for_each_eeg_channel(self): + for channel_name in eeg_processing.EEG_CHANNEL_SPECS: + for feature_name in eeg_features.SLOW_WAVE_FEATURE_NAMES: + self.assertIn( + f'{channel_name}_{feature_name}', + eeg_processing.EEG_FEATURE_NAMES, + ) + self.assertEqual( + len(eeg_processing.EEG_FEATURE_NAMES), + eeg_processing.EEG_FEATURE_LENGTH, + ) + + +if __name__ == '__main__': + unittest.main() diff --git a/train_output.txt b/train_output.txt new file mode 100644 index 0000000..e5cb2aa Binary files /dev/null and b/train_output.txt differ