From 3497ff31661f7005d183dc7a1809f80d875c9646 Mon Sep 17 00:00:00 2001 From: dcajal Date: Mon, 2 Mar 2026 17:47:15 +0100 Subject: [PATCH 01/93] Add .gitignore --- .gitignore | 213 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 213 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f298c64 --- /dev/null +++ b/.gitignore @@ -0,0 +1,213 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[codz] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py.cover +.hypothesis/ +.pytest_cache/ +cover/ +data/ +graphs/ + +graphs +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# UV +# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +#uv.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock +#poetry.toml + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python. +# https://pdm-project.org/en/latest/usage/project/#working-with-version-control +#pdm.lock +#pdm.toml +.pdm-python +.pdm-build/ + +# pixi +# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control. +#pixi.lock +# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one +# in the .venv directory. It is recommended not to include this directory in version control. +.pixi + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.envrc +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +# Abstra +# Abstra is an AI-powered process automation framework. +# Ignore directories containing user credentials, local state, and settings. +# Learn more at https://abstra.io/docs +.abstra/ + +# Visual Studio Code +# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore +# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore +# and can be added to the global gitignore or merged into this file. However, if you prefer, +# you could uncomment the following to ignore the entire vscode folder +# .vscode/ + +# Ruff stuff: +.ruff_cache/ + +# PyPI configuration file +.pypirc + +# Cursor +# Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to +# exclude from AI features like autocomplete and code analysis. Recommended for sensitive data +# refer to https://docs.cursor.com/context/ignore-files +.cursorignore +.cursorindexingignore + +# Marimo +marimo/_static/ +marimo/_lsp/ +__marimo__/ +results_summaryEEG_I0002.csv +results_summaryEEG_I0004.csv +results_summaryEEG_I0006.csv +results_summaryEEG_I0007.csv From 3fd3769cb0f06ea4efe66598f4d0c739204aad03 Mon Sep 17 00:00:00 2001 From: dcajal Date: Mon, 2 Mar 2026 17:47:20 +0100 Subject: [PATCH 02/93] Add AUTHORS.txt with contributor names and affiliations --- AUTHORS.txt | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 AUTHORS.txt diff --git a/AUTHORS.txt b/AUTHORS.txt new file mode 100644 index 0000000..143148b --- /dev/null +++ b/AUTHORS.txt @@ -0,0 +1,4 @@ +Sofia Romagnoli - Universidad de Zaragoza +Diego Cajal - CIBER-BBN +Josseline Madrid - Universidad de Zaragoza +Rodrigo Lozano - Universidad de Zaragoza From 2952aa1042398832157a584946425e74aacf1467 Mon Sep 17 00:00:00 2001 From: dcajal Date: Mon, 2 Mar 2026 17:47:43 +0100 Subject: [PATCH 03/93] Add models from previous challenges --- models/lolai_models.py | 2214 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 2214 insertions(+) create mode 100644 models/lolai_models.py diff --git a/models/lolai_models.py b/models/lolai_models.py new file mode 100644 index 0000000..c93eb98 --- /dev/null +++ b/models/lolai_models.py @@ -0,0 +1,2214 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +import numpy as np + +import torch.nn as nn +import torch +import torch.nn as nn +import torch.nn.functional as F +import numpy as np +import matplotlib.pyplot as plt +from captum.attr import ( + IntegratedGradients, + LayerGradCam, + LayerAttribution, + Occlusion, + GradientShap +) + +from typing import Dict, List, Tuple, Union, Optional + + +class CNN_LSTM_Classifier(nn.Module): + def __init__(self, input_channels=3, hidden_dim=64, num_classes=3, dropout=0.3): + super(CNN_LSTM_Classifier, self).__init__() + + self.cnn = nn.Sequential( + nn.Conv1d(input_channels, 6, kernel_size=5, padding=2), + nn.BatchNorm1d(6), + nn.ReLU(), + nn.MaxPool1d(kernel_size=2), + nn.Dropout(dropout), + + nn.Conv1d(6, 9, kernel_size=3, padding=1), + nn.BatchNorm1d(9), + nn.ReLU(), + nn.MaxPool1d(kernel_size=2), + nn.Dropout(dropout), + + + nn.Conv1d(9, 18, kernel_size=3, padding=1), + nn.BatchNorm1d(18), + nn.ReLU(), + nn.MaxPool1d(kernel_size=2), + nn.Dropout(dropout) + ) + + self.lstm = nn.LSTM( + input_size=18, + hidden_size=hidden_dim, + batch_first=True, + bidirectional=True # Use bidirectional LSTM for better context + ) + + self.classifier = nn.Sequential( + nn.Linear(2 * hidden_dim, 64), # Adjust for bidirectional LSTM + nn.ReLU(), + nn.Dropout(dropout), + nn.Linear(64, num_classes) + ) + + def forward(self, x): + # x: (batch, channels, time) + x = self.cnn(x) # (batch, features, time) + x = x.permute(0, 2, 1) # (batch, time, features) + _, (h_n, _) = self.lstm(x) # h_n: (num_layers * num_directions, batch, hidden_dim) + h_n = torch.cat((h_n[-2], h_n[-1]), dim=1) # Concatenate forward and backward states + out = self.classifier(h_n) # (batch, num_classes) + return out + +import torch +import torch.nn as nn +import torch.nn.functional as F +import matplotlib.pyplot as plt +import numpy as np + +import torch +import torch.nn as nn +import torch.nn.functional as F + +class CNN_LSTM_Classifier_XAI(nn.Module): + def __init__(self, input_channels=3, hidden_dim=32, num_classes=3, dropout=0.4): + super(CNN_LSTM_Classifier_XAI, self).__init__() + + self.cnn_activations = [] + self.lstm_activations = None + self.attention_weights = None + self.gradients = None + self.last_cnn_output = None + self.input = None + + # CNN + self.conv1 = nn.Conv1d(input_channels, 16, kernel_size=5, padding=2) + self.bn1 = nn.BatchNorm1d(16) + self.conv2 = nn.Conv1d(16, 32, kernel_size=3, padding=1) + self.bn2 = nn.BatchNorm1d(32) + self.conv3 = nn.Conv1d(32, 64, kernel_size=3, padding=1) + self.bn3 = nn.BatchNorm1d(64) + self.pool = nn.MaxPool1d(kernel_size=2) + self.dropout = nn.Dropout(dropout) + + # LSTM + self.lstm = nn.LSTM(input_size=64, hidden_size=hidden_dim, + batch_first=True, bidirectional=True) + + # Atención + self.attention = nn.Linear(2 * hidden_dim, 1) + + # Clasificador + self.classifier = nn.Sequential( + nn.Linear(2 * hidden_dim, 32), + nn.ReLU(), + nn.Dropout(dropout), + nn.Linear(32, num_classes) + ) + + def activations_hook(self, grad): + self.gradients = grad + + def forward(self, x, return_attention=False, track_gradients=False): + self.input = x + + x = self.conv1(x) + x = self.bn1(x) + x = F.relu(x) + self.cnn_activations.append(x.detach()) + x = self.pool(x) + x = self.dropout(x) + + x = self.conv2(x) + x = self.bn2(x) + x = F.relu(x) + self.cnn_activations.append(x.detach()) + x = self.pool(x) + x = self.dropout(x) + + x = self.conv3(x) + x = self.bn3(x) + x = F.relu(x) + cnn_output = x + + if track_gradients and cnn_output.requires_grad: + cnn_output.register_hook(self.activations_hook) + + self.last_cnn_output = cnn_output # necesario para Grad-CAM + self.cnn_activations.append(cnn_output.detach()) + + x = self.pool(cnn_output) + x = self.dropout(x) + + x = x.permute(0, 2, 1) # (batch, time, features) + lstm_out, _ = self.lstm(x) + self.lstm_activations = lstm_out.detach() + + attention_scores = self.attention(lstm_out).squeeze(-1) + attention_weights = F.softmax(attention_scores, dim=1) + self.attention_weights = attention_weights.detach() + + context_vector = torch.bmm(attention_weights.unsqueeze(1), lstm_out).squeeze(1) + out = self.classifier(context_vector) + + if return_attention: + return out, attention_weights + return out + + def reset_activation_storage(self): + self.cnn_activations = [] + self.lstm_activations = None + self.attention_weights = None + self.gradients = None + self.last_cnn_output = None + self.input = None + + def interpret(self, x, class_idx=None): + self.reset_activation_storage() + + was_training = self.training + lstm_was_training = self.lstm.training + + self.eval() + self.lstm.train() # necesario para CuDNN backward + + x.requires_grad_() + logits, attention = self.forward(x, return_attention=True, track_gradients=True) + pred = torch.softmax(logits, dim=1) + + if class_idx is None: + class_idx = pred.argmax(dim=1) + + for i in range(x.shape[0]): + pred[i, class_idx[i]].backward(retain_graph=True) + + self.train(was_training) + self.lstm.train(lstm_was_training) + + feature_importance = self.get_feature_importance() + temporal_channel_importance = self.get_temporal_channel_importance() + channel_imp=self.get_channel_importance() + + self.input = None # limpieza para evitar problemas de memoria + torch.cuda.empty_cache() + + return { + 'prediction': pred.detach(), + 'class_idx': class_idx, + 'attention_weights': self.attention_weights, + 'feature_importance': feature_importance, + 'cnn_activations': self.cnn_activations, + 'temporal_channel_importance': temporal_channel_importance, + 'channel_importance': channel_imp + } + + def get_feature_importance(self): + """ + Grad-CAM temporal sobre la salida del último bloque CNN. + Devuelve tensor (batch, time) + """ + if self.gradients is None or self.last_cnn_output is None: + return None + + pooled_gradients = torch.mean(self.gradients, dim=[0, 2]) # (channels,) + cam = self.last_cnn_output.clone() + + for i in range(cam.shape[1]): + cam[:, i, :] *= pooled_gradients[i] + + heatmap = torch.mean(cam, dim=1).detach() # (batch, time) + return heatmap + + def get_channel_importance(self): + """ + Importancia por canal: (batch, channels) + """ + if self.input.grad is None: + raise ValueError("Gradientes de la entrada no están disponibles. Llama primero a interpret().") + return self.input.grad.abs().mean(dim=2) + + def get_temporal_channel_importance(self): + """ + Importancia canal-temporal: (batch, channels, time) + """ + if self.input.grad is None: + raise ValueError("Gradients of the input are not available. Call interpret() first.") + return self.input.grad.abs().detach() + + + + + + +class ContrastiveVAE(nn.Module): + def __init__(self, in_channels=4, latent_dim=32, lstm_hidden=64, n_classes=3, use_classifier=False): + super().__init__() + self.use_classifier = use_classifier + + # Encoder + self.encoder = nn.Sequential( + nn.Conv1d(in_channels, 32, kernel_size=3, padding=1), + nn.ReLU(), + nn.Conv1d(32, 64, kernel_size=3, padding=1), + nn.ReLU() + ) + + self.global_pool = nn.AdaptiveAvgPool1d(1) # for VAE path + self.fc_mu = nn.Linear(64, latent_dim) + self.fc_logvar = nn.Linear(64, latent_dim) + + # Decoder (for reconstruction) + self.decoder_input = nn.Linear(latent_dim, 64) + self.decoder = nn.Sequential( + nn.ConvTranspose1d(64, 32, kernel_size=3, padding=1), + nn.ReLU(), + nn.ConvTranspose1d(32, in_channels, kernel_size=3, padding=1) + ) + + # LSTM + Classifier always initialized (but optionally used) + self.lstm = nn.LSTM(input_size=64, hidden_size=lstm_hidden, batch_first=True, bidirectional=True) + self.classifier = nn.Sequential( + nn.Linear(lstm_hidden * 2, 64), + nn.ReLU(), + nn.Linear(64, n_classes) + ) + + def encode(self, x): + h = self.encoder(x) # (B, 64, T) + pooled = self.global_pool(h).squeeze(-1) # (B, 64) + mu = self.fc_mu(pooled) + logvar = self.fc_logvar(pooled) + return mu, logvar, h # h is (B, 64, T) + + def reparameterize(self, mu, logvar): + std = torch.exp(0.5 * logvar) + eps = torch.randn_like(std) + return mu + eps * std + + def decode(self, z, length): + # Mejora la reconstrucción con una capa de proyección inicial + h = self.decoder_input(z).unsqueeze(-1) # (B, 64, 1) + # Usar interpolación para un escalado más suave en lugar de expand + h = F.interpolate(h, size=length, mode='linear', align_corners=False) + x_recon = self.decoder(h) + return x_recon + + def forward(self, x): + B, C, T = x.shape + mu, logvar, features = self.encode(x) # features: (B, 64, T) + z = self.reparameterize(mu, logvar) + x_recon = self.decode(z, T) + + logits = None + if self.use_classifier: + features_t = features.permute(0, 2, 1) # (B, T, 64) + lstm_out, _ = self.lstm(features_t) # (B, T, 2*hidden) + lstm_feat = lstm_out.mean(dim=1) # (B, 2*hidden) + logits = self.classifier(lstm_feat) # (B, n_classes) + + return x_recon, mu, logvar, z, logits + + def get_latents(self, x, use_mean=True): + mu, logvar, _ = self.encode(x) + return mu if use_mean else self.reparameterize(mu, logvar) + + def classify(self, x): + """Forward through the classifier only (requires use_classifier = True).""" + assert self.use_classifier, "Classifier is not enabled. Set model.use_classifier = True before calling classify." + _, _, features = self.encode(x) + features_t = features.permute(0, 2, 1) + lstm_out, _ = self.lstm(features_t) + lstm_feat = lstm_out.mean(dim=1) + return self.classifier(lstm_feat) + + +def vae_loss(recon_x, x, mu, logvar): + recon_loss = F.mse_loss(recon_x, x, reduction='mean') + kl_div = -0.5 * torch.mean(1 + logvar - mu.pow(2) - logvar.exp()) + return recon_loss + kl_div, recon_loss, kl_div + + +def contrastive_loss(z, ids, temperature=0.1): + z = F.normalize(z, dim=1) + sim = torch.mm(z, z.T) / temperature + labels = ids.view(-1, 1) + mask = torch.eq(labels, labels.T).float().to(z.device) + mask = mask - torch.eye(len(z), device=z.device) + exp_sim = torch.exp(sim) * (1 - torch.eye(len(z), device=z.device)) + log_prob = sim - torch.log(exp_sim.sum(1, keepdim=True) + 1e-8) + mean_log_prob_pos = (mask * log_prob).sum(1) / (mask.sum(1) + 1e-8) + return -mean_log_prob_pos.mean() + + +def intra_patient_loss(latents, patient_ids): + """Versión optimizada que evita bucles explícitos""" + # Convertir IDs a tensor si no lo son ya + if not isinstance(patient_ids, torch.Tensor): + patient_ids = torch.tensor(patient_ids, device=latents.device) + + # Crear matriz de similaridad de pacientes (1 donde son iguales) + patient_sim = (patient_ids.unsqueeze(1) == patient_ids.unsqueeze(0)).float() + # Quitar diagonal (mismo ejemplo) + mask = patient_sim - torch.eye(len(latents), device=latents.device) + # Calcular distancias entre latentes + latent_dists = torch.cdist(latents, latents, p=2) + # Aplicar máscara y promediar + valid_pairs = mask.sum() + if valid_pairs > 0: + return (mask * latent_dists).sum() / valid_pairs + return torch.tensor(0.0, device=latents.device) + + +def training_step(model, batch1, batch2, patient_ids, optimizer, alpha=0.1, beta=1.0): + model.train() + x1, x2 = batch1, batch2 + x1_recon, mu1, logvar1, z1, _ = model(x1) + x2_recon, mu2, logvar2, z2, _ = model(x2) + + recon1, r1, kl1 = vae_loss(x1_recon, x1, mu1, logvar1) + recon2, r2, kl2 = vae_loss(x2_recon, x2, mu2, logvar2) + vae_total = (recon1 + recon2) / 2 + + z_all = torch.cat([z1, z2], dim=0) + # Asegurar que IDs sean tensores + ids = torch.arange(len(z1), device=z1.device).repeat(2) + contrastive = contrastive_loss(z_all, ids) + + # Extender patient_ids correctamente + if isinstance(patient_ids, torch.Tensor): + p_ids = torch.cat([patient_ids, patient_ids], dim=0) + else: + p_ids = patient_ids + patient_ids # Si es una lista + + patient_reg = intra_patient_loss(z_all, p_ids) + + total = vae_total + alpha * contrastive + beta * patient_reg + + optimizer.zero_grad() + total.backward() + optimizer.step() + + return { + 'total_loss': total.item(), + 'recon_loss': vae_total.item(), + 'contrastive': contrastive.item(), + 'patient_reg': patient_reg.item(), + 'kl_loss': (kl1 + kl2).item() / 2 + } + + +def fine_tune_step(model, x, y, optimizer, criterion=nn.CrossEntropyLoss()): + model.train() + model.use_classifier = True + logits = model.classify(x) + loss = criterion(logits, y) + + optimizer.zero_grad() + loss.backward() + optimizer.step() + + preds = torch.argmax(logits, dim=1) + acc = (preds == y).float().mean().item() + + return { + 'classification_loss': loss.item(), + 'accuracy': acc + } + +class ImprovedPainClassifier(nn.Module): + def __init__(self, input_channels=3, hidden_dim=128, num_classes=3, dropout=0.4): + super(ImprovedPainClassifier, self).__init__() + + # Increased regularization and feature extraction for small datasets + self.cnn = nn.Sequential( + # Layer 1: More filters to capture diverse patterns + nn.Conv1d(input_channels, 64, kernel_size=3, padding=1), + nn.BatchNorm1d(64), + nn.LeakyReLU(0.1), # LeakyReLU helps with gradient flow + nn.MaxPool1d(kernel_size=2), + + # Layer 2: Increased complexity + nn.Conv1d(64, 128, kernel_size=3, padding=1), + nn.BatchNorm1d(128), + nn.LeakyReLU(0.1), + nn.MaxPool1d(kernel_size=2), + nn.Dropout(dropout), + + # Layer 3: Additional layer for better feature extraction + nn.Conv1d(128, 128, kernel_size=3, padding=1), + nn.BatchNorm1d(128), + nn.LeakyReLU(0.1), + nn.Dropout(dropout) + ) + + # Attention mechanism to focus on important temporal patterns + self.attention = nn.Sequential( + nn.Linear(256, 64), + nn.Tanh(), + nn.Linear(64, 1) + ) + + # Bidirectional LSTM with residual connections + self.lstm = nn.LSTM( + input_size=128, + hidden_size=hidden_dim, + num_layers=2, # Multiple layers for complex temporal patterns + batch_first=True, + bidirectional=True, + dropout=dropout # Apply dropout between LSTM layers + ) + + # Classifier with additional regularization + self.classifier = nn.Sequential( + nn.Linear(2 * hidden_dim, hidden_dim), + nn.BatchNorm1d(hidden_dim), # Normalize activations + nn.LeakyReLU(0.1), + nn.Dropout(dropout), + nn.Linear(hidden_dim, hidden_dim // 2), + nn.LeakyReLU(0.1), + nn.Dropout(dropout), + nn.Linear(hidden_dim // 2, num_classes) + ) + + def forward(self, x): + # x: (batch, channels, time) - BVP, EDA, and respiratory signals + + # Extract features with CNN + cnn_out = self.cnn(x) # (batch, 128, time') + + # Reshape for LSTM + cnn_out = cnn_out.permute(0, 2, 1) # (batch, time', 128) + + # Process with LSTM + lstm_out, (h_n, _) = self.lstm(cnn_out) # lstm_out: (batch, time', 2*hidden_dim) + + # Apply attention to focus on relevant parts of the signal + attn_weights = self.attention(lstm_out).softmax(dim=1) # (batch, time', 1) + context = torch.sum(attn_weights * lstm_out, dim=1) # (batch, 2*hidden_dim) + + # Alternative: Use concatenated hidden states from both directions + # h_n = torch.cat((h_n[-2], h_n[-1]), dim=1) # (batch, 2*hidden_dim) + + # Classify + out = self.classifier(context) # (batch, num_classes) + return out + + +import matplotlib.pyplot as plt +import numpy as np +import torch + +class ExplainabilityVisualizer: + def __init__(self, channel_names=None): + """ + channel_names: lista opcional con nombres de los canales de entrada + """ + self.channel_names = channel_names + + def plot_attention_weights(self, attention_weights, title="Atención temporal"): + attention = attention_weights.squeeze().cpu().numpy() + plt.figure(figsize=(10, 2)) + plt.plot(attention) + plt.title(title) + plt.xlabel("Timestep") + plt.ylabel("Weight") + plt.grid(True) + plt.tight_layout() + plt.show() + + def plot_gradcam_heatmap(self, heatmap, title="Grad-CAM temporal"): + heat = heatmap.squeeze().cpu().numpy() + plt.figure(figsize=(10, 2)) + plt.plot(heat) + plt.title(title) + plt.xlabel("Timestep") + plt.ylabel("Importance") + plt.grid(True) + plt.tight_layout() + plt.show() + + def plot_channel_importance(self, channel_importance, title="Importancia por canal"): + values = channel_importance.squeeze().cpu().numpy() + channels = self.channel_names if self.channel_names else [f"Channel {i}" for i in range(len(values))] + plt.figure(figsize=(6, 3)) + plt.bar(channels, values) + plt.title(title) + plt.ylabel("Importancia media") + plt.xticks(rotation=45) + plt.grid(axis='y') + plt.tight_layout() + plt.show() + + def plot_signals_with_attention_highlight(self, x, importance, threshold=0.85, title="Señal con zonas de atención"): + """ + Dibuja las señales multicanal y sombreado rojo donde la importancia temporal supera el umbral. + + Args: + x: Tensor (channels, time) + importance: Tensor (time,) + threshold: percentil (0-1) o valor absoluto + title: título del gráfico + """ + x = x.detach().cpu().numpy() + importance = importance.detach().cpu().numpy() + time = np.arange(x.shape[1]) + n_channels = x.shape[0] + + if threshold <= 1.0: + threshold_value = np.quantile(importance, threshold) + else: + threshold_value = threshold + + high_attention_mask = importance >= threshold_value + + fig, axs = plt.subplots(n_channels, 1, figsize=(12, 2.5 * n_channels), sharex=True) + if n_channels == 1: + axs = [axs] + + for i in range(n_channels): + axs[i].plot(time, x[i], label=self.channel_names[i] if self.channel_names else f"Canal {i}", color="black") + axs[i].set_ylabel("Valor") + axs[i].grid(True) + + in_high = False + start = 0 + for t in range(len(high_attention_mask)): + if high_attention_mask[t] and not in_high: + start = t + in_high = True + elif not high_attention_mask[t] and in_high: + axs[i].axvspan(start, t, color='red', alpha=0.25) + in_high = False + if in_high: + axs[i].axvspan(start, len(high_attention_mask), color='red', alpha=0.25) + + axs[i].legend(loc="upper right") + + axs[-1].set_xlabel("Tiempo (muestras)") + plt.suptitle(title) + plt.tight_layout() + plt.show() + + + +class FocalLoss(nn.Module): + """ + Focal Loss para clasificación binaria y multiclase. + + Parámetros: + - alpha: Factor de ponderación para manejar desequilibrio de clases. + Puede ser un escalar (mismo valor para todas las clases) o + un tensor (valores específicos por clase). + - gamma: Factor de modulación para enfocar en ejemplos difíciles (>= 0). + - reduction: 'none' | 'mean' | 'sum' + - eps: Pequeño valor para estabilidad numérica + + Referencias: + - Paper original: "Focal Loss for Dense Object Detection" por Lin et al. + """ + def __init__(self, alpha=0.25, gamma=2.0, reduction='mean', eps=1e-6): + super(FocalLoss, self).__init__() + self.alpha = alpha + self.gamma = gamma + self.reduction = reduction + self.eps = eps + + def forward(self, inputs, targets): + """ + Args: + inputs: Logits de forma [B, C] donde B es el tamaño del batch y C es el número de clases. + Para clasificación binaria, C puede ser 1. + targets: Etiquetas de objetivos de forma [B] para multiclase o [B, 1] para binaria. + Valores enteros para multiclase (clases indexadas desde 0 a C-1). + Valores continuos entre 0 y 1 para binaria. + """ + # Determinar si es clasificación binaria o multiclase + if inputs.shape[1] == 1 or inputs.shape[1] == 2: # Binaria + # Aplicar sigmoide para obtener probabilidades + probs = torch.sigmoid(inputs.view(-1)) + targets = targets.view(-1) + + # Calcular pt (probabilidad del objetivo correcto) + pt = probs * targets + (1 - probs) * (1 - targets) + + # Aplicar factores de ponderación + if isinstance(self.alpha, (float, int)): + alpha_t = self.alpha * targets + (1 - self.alpha) * (1 - targets) + else: + # Si alpha es un tensor, usar indexación + alpha_t = self.alpha if self.alpha is not None else torch.ones_like(pt) + + # Calcular la focal loss + focal_weight = (1 - pt).pow(self.gamma) + loss = -alpha_t * focal_weight * torch.log(pt.clamp(min=self.eps)) + + else: # Multiclase + # Convertir logits a distribución de probabilidad + log_softmax = F.log_softmax(inputs, dim=1) + + # Obtener log probabilidad para las clases objetivo + targets = targets.view(-1, 1) + log_pt = log_softmax.gather(1, targets).view(-1) + pt = log_pt.exp() # Obtener probabilidades + + # Aplicar factores de ponderación + if isinstance(self.alpha, (list, tuple, torch.Tensor)): + # Si alpha es específico por clase + alpha = torch.tensor(self.alpha, device=inputs.device) + alpha_t = alpha.gather(0, targets.view(-1)) + else: + alpha_t = self.alpha if self.alpha is not None else 1.0 + + # Calcular focal loss + focal_weight = (1 - pt).pow(self.gamma) + loss = -alpha_t * focal_weight * log_pt + + # Aplicar reduction + if self.reduction == 'mean': + return loss.mean() + elif self.reduction == 'sum': + return loss.sum() + else: # 'none' + return loss + + + + +class CNN_LSTM_Classifier_XAI_2(nn.Module): + def __init__(self, input_channels=3, hidden_dim=32, num_classes=3, dropout=0.1): + super(CNN_LSTM_Classifier_XAI_2, self).__init__() + + self.cnn_activations = [] + self.lstm_activations = None + self.attention_weights = None + self.gradients = None + self.last_cnn_output = None + self.input = None + self.input_channels = input_channels + + # CNN + self.conv1 = nn.Conv1d(input_channels, 16, kernel_size=5, padding=2) + self.bn1 = nn.BatchNorm1d(16) + self.conv2 = nn.Conv1d(16, 32, kernel_size=3, padding=1) + self.bn2 = nn.BatchNorm1d(32) + self.conv3 = nn.Conv1d(32, 64, kernel_size=3, padding=1) + self.bn3 = nn.BatchNorm1d(64) + self.pool = nn.MaxPool1d(kernel_size=2) + self.dropout = nn.Dropout(dropout) + + # LSTM + self.lstm = nn.LSTM(input_size=64, hidden_size=hidden_dim, + batch_first=True, bidirectional=True) + + # Attention + self.attention = nn.Linear(2 * hidden_dim, 1) + + # Classifier + self.classifier = nn.Sequential( + nn.Linear(2 * hidden_dim, 32), + nn.ReLU(), + nn.Dropout(dropout), + nn.Linear(32, num_classes) + ) + + def activations_hook(self, grad): + self.gradients = grad + + + + def forward(self, x, return_attention=False, track_gradients=False): + # Reset activation storage at the beginning of each forward pass + self.reset_activation_storage() + + self.input = x + + x = self.conv1(x) + x = self.bn1(x) + x = F.relu(x) + self.cnn_activations.append(x.detach()) + x = self.pool(x) + x = self.dropout(x) + + x = self.conv2(x) + x = self.bn2(x) + x = F.relu(x) + self.cnn_activations.append(x.detach()) + x = self.pool(x) + x = self.dropout(x) + + x = self.conv3(x) + x = self.bn3(x) + x = F.relu(x) + cnn_output = x + + if track_gradients and cnn_output.requires_grad: + cnn_output.register_hook(self.activations_hook) + + self.last_cnn_output = cnn_output # needed for Grad-CAM + self.cnn_activations.append(cnn_output.detach()) + + x = self.pool(cnn_output) + x = self.dropout(x) + + x = x.permute(0, 2, 1) # (batch, time, features) + lstm_out, (h_n, c_n) = self.lstm(x) + self.lstm_activations = lstm_out.detach() + + attention_scores = self.attention(lstm_out).squeeze(-1) + attention_weights = F.softmax(attention_scores, dim=1) + self.attention_weights = attention_weights.detach() + + context_vector = torch.bmm(attention_weights.unsqueeze(1), lstm_out).squeeze(1) + out = self.classifier(context_vector) + + if return_attention: + return out, attention_weights + return out + + def reset_activation_storage(self): + self.cnn_activations = [] + self.lstm_activations = None + self.attention_weights = None + self.gradients = None + self.last_cnn_output = None + + def interpret(self, x, class_idx=None, methods=None): + """ + Enhanced interpretation method with multiple explainability techniques + + Args: + x: Input data tensor + class_idx: Target class indices to explain (defaults to predicted class) + methods: List of methods to use, options: ['gradcam', 'integrated_gradients', + 'occlusion', 'shap', 'feature_ablation', 'all'] + + Returns: + Dictionary with various interpretability outputs + """ + if methods is None: + methods = ['gradcam', 'attention'] # Default methods + if 'all' in methods: + methods = ['gradcam', 'integrated_gradients', 'occlusion', 'shap', + 'feature_ablation', 'attention', 'layer_importance'] + + # Store original training state + was_training = self.training + lstm_was_training = self.lstm.training + + # Set model to evaluation mode for interpretability + self.eval() + self.lstm.train() # needed for CuDNN backward compatibility + + # Base prediction + x.requires_grad_() + self.input = x # Store input for interpretability methods + + logits, attention = self.forward(x, return_attention=True, track_gradients=True) + pred = torch.softmax(logits, dim=1) + + if class_idx is None: + class_idx = pred.argmax(dim=1) + + # Initialize results dictionary + results = { + 'prediction': pred.detach(), + 'class_idx': class_idx, + 'attention_weights': self.attention_weights, + } + + # Apply selected interpretability methods + if 'gradcam' in methods: + for i in range(x.shape[0]): + pred[i, class_idx[i]].backward(retain_graph=True if i < x.shape[0]-1 else False) + + results['feature_importance'] = self.get_feature_importance() + results['temporal_channel_importance'] = self.get_temporal_channel_importance() + results['channel_importance'] = self.get_channel_importance() + results['cnn_activations'] = self.cnn_activations + + # Integrated Gradients + if 'integrated_gradients' in methods: + ig = IntegratedGradients(self.forward_wrapper) + results['integrated_gradients'] = self._compute_integrated_gradients( + ig, x, class_idx) + + # Occlusion analysis + if 'occlusion' in methods: + occlusion = Occlusion(self.forward_wrapper) + results['occlusion'] = self._compute_occlusion(occlusion, x, class_idx) + + # SHAP (GradientSHAP implementation) + if 'shap' in methods: + gradient_shap = GradientShap(self.forward_wrapper) + results['gradient_shap'] = self._compute_gradient_shap(gradient_shap, x, class_idx) + + # Feature ablation (sensitivity analysis) + if 'feature_ablation' in methods: + results['feature_ablation'] = self._feature_ablation_analysis(x, class_idx) + + # Layer importance analysis + if 'layer_importance' in methods: + results['layer_importance'] = self._compute_layer_importance(x, class_idx) + + # Restore original training states + self.train(was_training) + self.lstm.train(lstm_was_training) + + # Clean up to avoid memory issues + self.input = None + torch.cuda.empty_cache() + + return results + + def forward_wrapper(self, x): + """Wrapper for Captum compatibility""" + return self.forward(x) + + def get_feature_importance(self): + """ + Grad-CAM temporal over the output of the last CNN block. + Returns tensor (batch, time) + """ + if self.gradients is None or self.last_cnn_output is None: + return None + + pooled_gradients = torch.mean(self.gradients, dim=[0, 2]) # (channels,) + cam = self.last_cnn_output.clone() + + for i in range(cam.shape[1]): + cam[:, i, :] *= pooled_gradients[i] + + heatmap = torch.mean(cam, dim=1).detach() # (batch, time) + + # Apply ReLU to highlight only positive influences + heatmap = F.relu(heatmap) + + # Normalize heatmap for better visualization + if heatmap.max() > 0: + heatmap = heatmap / heatmap.max() + + return heatmap + + def get_channel_importance(self): + """ + Channel importance: (batch, channels) + """ + if self.input is None or self.input.grad is None: + raise ValueError("Input gradients not available. Call interpret() first.") + return self.input.grad.abs().mean(dim=2).detach() + + def get_temporal_channel_importance(self): + """ + Temporal-channel importance: (batch, channels, time) + """ + if self.input is None or self.input.grad is None: + raise ValueError("Input gradients not available. Call interpret() first.") + return self.input.grad.abs().detach() + + def _compute_integrated_gradients(self, ig, x, class_idx): + """Compute integrated gradients attribution""" + batch_size = x.shape[0] + attributions = [] + + for i in range(batch_size): + baseline = torch.zeros_like(x[i:i+1]) + attr = ig.attribute( + x[i:i+1], baseline, target=class_idx[i].item(), n_steps=50 + ) + attributions.append(attr) + + return torch.cat(attributions).detach() + + def _compute_occlusion(self, occlusion_algo, x, class_idx): + """Compute occlusion-based feature attribution""" + batch_size = x.shape[0] + attributions = [] + + # Define sliding window parameters for temporal data + window_size = min(5, x.shape[2] // 4) # Adapt window size to input length + + for i in range(batch_size): + attr = occlusion_algo.attribute( + x[i:i+1], + sliding_window_shapes=(1, window_size), + target=class_idx[i].item(), + strides=(1, max(1, window_size // 2)) + ) + attributions.append(attr) + + return torch.cat(attributions).detach() + + def _compute_gradient_shap(self, shap_algo, x, class_idx): + """Compute GradientSHAP attributions""" + batch_size = x.shape[0] + attributions = [] + + for i in range(batch_size): + # Create random baselines (typically 10-50 for good estimates) + baselines = torch.randn(10, *x[i:i+1].shape[1:]) * 0.001 + + # Ensure baselines device matches input + baselines = baselines.to(x.device) + + attr = shap_algo.attribute( + x[i:i+1], baselines=baselines, target=class_idx[i].item() + ) + attributions.append(attr) + + return torch.cat(attributions).detach() + + def _feature_ablation_analysis(self, x, class_idx): + """Analyze model by systematically ablating input features""" + batch_size = x.shape[0] + results = [] + + for i in range(batch_size): + # Store original prediction + with torch.no_grad(): + orig_output = self.forward(x[i:i+1]) + orig_prob = torch.softmax(orig_output, dim=1)[0, class_idx[i]].item() + + # Test ablation of each channel + channel_importance = [] + for c in range(self.input_channels): + # Create ablated input (zero out one channel) + ablated_input = x[i:i+1].clone() + ablated_input[:, c, :] = 0 + + # Get prediction on ablated input + with torch.no_grad(): + ablated_output = self.forward(ablated_input) + ablated_prob = torch.softmax(ablated_output, dim=1)[0, class_idx[i]].item() + + # Impact is reduction in probability + channel_impact = orig_prob - ablated_prob + channel_importance.append(channel_impact) + + results.append(torch.tensor(channel_importance)) + + return torch.stack(results) + + def _compute_layer_importance(self, x, class_idx): + """Compute importance of each layer using Layer GradCAM""" + batch_size = x.shape[0] + layer_importance = {} + + # Define layers to analyze + layers = { + 'conv1': self.conv1, + 'conv2': self.conv2, + 'conv3': self.conv3 + } + + for layer_name, layer in layers.items(): + layer_gradcam = LayerGradCam(self.forward_wrapper, layer) + layer_attrs = [] + + for i in range(batch_size): + attr = layer_gradcam.attribute( + x[i:i+1], target=class_idx[i].item() + ) + # Process attribution to create a single importance score per sample + pooled_attr = torch.mean(attr, dim=1) + + layer_attrs.append(pooled_attr) + + layer_importance[layer_name] = torch.cat(layer_attrs).detach() + + return layer_importance + + def visualize_attributions(self, sample_idx, interpretations, time_axis=None, + channel_names=None, class_names=None): + """ + Visualize the various interpretation results + + Args: + sample_idx: Index of the sample to visualize + interpretations: Dictionary returned by interpret() method + time_axis: Optional array/list with time points for x-axis + channel_names: Optional list of channel names + class_names: Optional list of class names + """ + if not channel_names: + channel_names = [f'Channel {i}' for i in range(self.input_channels)] + + if not class_names: + class_idx = interpretations['class_idx'][sample_idx].item() + class_name = f'Class {class_idx}' + else: + class_idx = interpretations['class_idx'][sample_idx].item() + class_name = class_names[class_idx] + + # Set up figure + plt.figure(figsize=(15, 12)) + + # Original input visualization (top row, first column) + plt.subplot(3, 3, 1) + if self.input is not None: + input_data = self.input[sample_idx].cpu().detach().numpy() + if time_axis is not None: + for i in range(input_data.shape[0]): + plt.plot(time_axis, input_data[i], label=channel_names[i]) + else: + for i in range(input_data.shape[0]): + plt.plot(input_data[i], label=channel_names[i]) + plt.legend(loc='best') + plt.title('Input Signal') + plt.xlabel('Time') + plt.ylabel('Value') + + # GradCAM feature importance (top row, second column) + if 'feature_importance' in interpretations and interpretations['feature_importance'] is not None: + plt.subplot(3, 3, 2) + heatmap = interpretations['feature_importance'][sample_idx].cpu().numpy() + if time_axis is not None: + plt.plot(time_axis, heatmap) + else: + plt.plot(heatmap) + plt.title('GradCAM Feature Importance') + plt.xlabel('Time') + plt.ylabel('Importance') + + # Attention weights (top row, third column) + if 'attention_weights' in interpretations and interpretations['attention_weights'] is not None: + plt.subplot(3, 3, 3) + attention = interpretations['attention_weights'][sample_idx].cpu().numpy() + + if time_axis is not None: + # Need to match attention time axis to input time axis + # (account for pooling in the network) + x_points = np.linspace(time_axis[0], time_axis[-1], len(attention)) + plt.plot(x_points, attention) + else: + plt.plot(attention) + plt.title('Attention Weights') + plt.xlabel('Time') + plt.ylabel('Attention') + + # Channel importance (middle row, first column) + if 'channel_importance' in interpretations and interpretations['channel_importance'] is not None: + plt.subplot(3, 3, 4) + ch_importance = interpretations['channel_importance'][sample_idx].cpu().numpy() + plt.bar(channel_names, ch_importance) + plt.title('Channel Importance') + plt.ylabel('Importance') + plt.xticks(rotation=45) + + # Integrated Gradients (middle row, second column) + if 'integrated_gradients' in interpretations: + plt.subplot(3, 3, 5) + ig_attr = interpretations['integrated_gradients'][sample_idx].cpu().numpy() + ig_attr_mean = np.mean(ig_attr, axis=0) # Average across channels for visualization + + if time_axis is not None: + plt.plot(time_axis, ig_attr_mean) + else: + plt.plot(ig_attr_mean) + plt.title('Integrated Gradients') + plt.xlabel('Time') + plt.ylabel('Attribution') + + # Feature Ablation (middle row, third column) + if 'feature_ablation' in interpretations: + plt.subplot(3, 3, 6) + ablation_scores = interpretations['feature_ablation'][sample_idx].cpu().numpy() + plt.bar(channel_names, ablation_scores) + plt.title('Feature Ablation Impact') + plt.ylabel('Probability Change') + plt.xticks(rotation=45) + + # SHAP values (bottom row, first column) + if 'gradient_shap' in interpretations: + plt.subplot(3, 3, 7) + shap_attr = interpretations['gradient_shap'][sample_idx].cpu().numpy() + # Visualize average SHAP value over time + shap_avg = np.mean(shap_attr, axis=0) + + if time_axis is not None: + plt.plot(time_axis, shap_avg) + else: + plt.plot(shap_avg) + plt.title('GradientSHAP Values') + plt.xlabel('Time') + plt.ylabel('SHAP Value') + + # Occlusion analysis (bottom row, second column) + if 'occlusion' in interpretations: + plt.subplot(3, 3, 8) + occlusion_attr = interpretations['occlusion'][sample_idx].cpu().numpy() + occlusion_avg = np.mean(occlusion_attr, axis=0) + + if time_axis is not None: + plt.plot(time_axis, occlusion_avg) + else: + plt.plot(occlusion_avg) + plt.title('Occlusion Analysis') + plt.xlabel('Time') + plt.ylabel('Attribution') + + # Prediction summary (bottom row, third column) + plt.subplot(3, 3, 9) + pred_probs = interpretations['prediction'][sample_idx].cpu().numpy() + classes = list(range(len(pred_probs))) + if class_names: + classes = class_names + plt.bar(classes, pred_probs) + plt.title(f'Prediction: {class_name}') + plt.ylabel('Probability') + plt.ylim([0, 1]) + + plt.tight_layout() + return plt.gcf() + + def generate_interpretation_report(self, input_data, class_idx=None, + channel_names=None, class_names=None, + time_axis=None, methods='all'): + """ + Generate a comprehensive interpretation report for the given input + + Args: + input_data: Input tensor to analyze + class_idx: Target class indices (optional) + channel_names: Names of input channels (optional) + class_names: Names of output classes (optional) + time_axis: Time points for x-axis (optional) + methods: Explainability methods to use + + Returns: + Dictionary containing interpretations and visualization figure + """ + # Run all interpretation methods + interpretations = self.interpret(input_data, class_idx, methods=methods) + + # Generate visualizations for each sample + figures = [] + for i in range(input_data.shape[0]): + fig = self.visualize_attributions( + i, interpretations, + time_axis=time_axis, + channel_names=channel_names, + class_names=class_names + ) + figures.append(fig) + plt.close(fig) # Close to avoid display in notebooks + + return { + 'interpretations': interpretations, + 'figures': figures + } + + + +class CNN_LSTM_Classifier_XAI_2(nn.Module): + def __init__( + self, + input_channels=3, + num_classes=3, + cnn_channels=(16, 32, 64), + kernel_sizes=(5, 3, 3), + pool_type="max", # or 'avg' + dropout=0.1, + lstm_hidden_dim=32, + lstm_num_layers=1, + bidirectional=True, + classifier_hidden_dim=32, + attention_dim=None, # None = default: 2 * lstm_hidden_dim + ): + super(CNN_LSTM_Classifier_XAI_2, self).__init__() + + self.input_channels = input_channels + self.pool_type = pool_type + self.dropout_rate = dropout + self.bidirectional = bidirectional + self.num_directions = 2 if bidirectional else 1 + + self.cnn_activations = [] + self.lstm_activations = None + self.attention_weights = None + self.gradients = None + self.last_cnn_output = None + self.input = None + + # CNN Blocks + self.conv1 = nn.Conv1d(input_channels, cnn_channels[0], kernel_size=kernel_sizes[0], padding=kernel_sizes[0] // 2) + self.bn1 = nn.BatchNorm1d(cnn_channels[0]) + + self.conv2 = nn.Conv1d(cnn_channels[0], cnn_channels[1], kernel_size=kernel_sizes[1], padding=kernel_sizes[1] // 2) + self.bn2 = nn.BatchNorm1d(cnn_channels[1]) + + self.conv3 = nn.Conv1d(cnn_channels[1], cnn_channels[2], kernel_size=kernel_sizes[2], padding=kernel_sizes[2] // 2) + self.bn3 = nn.BatchNorm1d(cnn_channels[2]) + + self.pool = nn.MaxPool1d(kernel_size=2) if pool_type == "max" else nn.AvgPool1d(kernel_size=2) + self.dropout = nn.Dropout(dropout) + + # LSTM + self.lstm = nn.LSTM( + input_size=cnn_channels[2], + hidden_size=lstm_hidden_dim, + num_layers=lstm_num_layers, + batch_first=True, + bidirectional=bidirectional + ) + + # Attention + attention_dim = attention_dim or self.num_directions * lstm_hidden_dim + self.attention = nn.Linear(self.num_directions * lstm_hidden_dim, 1) + + # Classifier + self.classifier = nn.Sequential( + nn.Linear(self.num_directions * lstm_hidden_dim, classifier_hidden_dim), + nn.ReLU(), + nn.Dropout(dropout), + nn.Linear(classifier_hidden_dim, num_classes) + ) + + def activations_hook(self, grad): + self.gradients = grad + + def reset_activation_storage(self): + self.cnn_activations = [] + self.lstm_activations = None + self.attention_weights = None + self.gradients = None + self.last_cnn_output = None + + def forward(self, x, return_attention=False, track_gradients=False): + self.reset_activation_storage() + self.input = x + + x = self.pool(F.relu(self.bn1(self.conv1(x)))) + self.cnn_activations.append(x.detach()) + x = self.dropout(x) + + x = self.pool(F.relu(self.bn2(self.conv2(x)))) + self.cnn_activations.append(x.detach()) + x = self.dropout(x) + + x = F.relu(self.bn3(self.conv3(x))) + cnn_output = x + + if track_gradients and cnn_output.requires_grad: + cnn_output.register_hook(self.activations_hook) + + self.last_cnn_output = cnn_output + self.cnn_activations.append(cnn_output.detach()) + + x = self.pool(cnn_output) + x = self.dropout(x) + + x = x.permute(0, 2, 1) # (batch, time, features) + lstm_out, _ = self.lstm(x) + self.lstm_activations = lstm_out.detach() + + attention_scores = self.attention(lstm_out).squeeze(-1) + attention_weights = F.softmax(attention_scores, dim=1) + self.attention_weights = attention_weights.detach() + + context_vector = torch.bmm(attention_weights.unsqueeze(1), lstm_out).squeeze(1) + out = self.classifier(context_vector) + + if return_attention: + return out, attention_weights + return out + + +class CNN_LSTM_Classifier_Tunable(nn.Module): + def __init__( + self, + config: Optional[Dict] = None, + input_channels: int = 3, + seq_length: int = None, + num_classes: int = 3, + cnn_channels: Tuple[int, ...] = (16, 32, 64), + kernel_sizes: Tuple[int, ...] = (5, 3, 3), + pool_type: str = "max", # or 'avg' + pool_sizes: Tuple[int, ...] = (2, 2, 2), + use_batch_norm: bool = True, + activation: str = "relu", # "relu", "leaky_relu", "elu", "gelu" + dropout: float = 0.1, + cnn_dropout: Optional[float] = None, # Separate dropout for CNN + lstm_hidden_dim: int = 32, + lstm_num_layers: int = 1, + bidirectional: bool = True, + lstm_dropout: Optional[float] = None, # Separate dropout for LSTM + classifier_hidden_dims: List[int] = [32], # Multiple hidden layers + attention_dim: Optional[int] = None, # None = default: 2 * lstm_hidden_dim + attention_type: str = "basic", # "basic", "scaled_dot", "multi_head" + multi_head_num: int = 4, # For multi-head attention + residual_connections: bool = False, + layer_normalization: bool = False, + weight_init: str = "default", # "default", "xavier", "kaiming" + ): + """ + Enhanced CNN-LSTM model with attention mechanism designed for tuning flexibility. + + Args: + config: Optional dictionary with all hyperparameters to override other arguments + input_channels: Number of input channels + seq_length: Length of input sequence (needed for some operations) + num_classes: Number of output classes + cnn_channels: Tuple of CNN output channels for each layer + kernel_sizes: Tuple of kernel sizes for each CNN layer + pool_type: Pooling type ("max" or "avg") + pool_sizes: Pooling sizes for each layer + use_batch_norm: Whether to use batch normalization + activation: Activation function type + dropout: Default dropout rate + cnn_dropout: CNN-specific dropout (if None, uses dropout) + lstm_hidden_dim: LSTM hidden dimension + lstm_num_layers: Number of LSTM layers + bidirectional: Whether LSTM is bidirectional + lstm_dropout: LSTM-specific dropout (if None, uses dropout) + classifier_hidden_dims: List of hidden dimensions for classifier + attention_dim: Attention dimension + attention_type: Type of attention mechanism + multi_head_num: Number of heads for multi-head attention + residual_connections: Whether to use residual connections + layer_normalization: Whether to use layer normalization + weight_init: Weight initialization strategy + """ + super(CNN_LSTM_Classifier_Tunable, self).__init__() + + # Override with config if provided + if config is not None: + # Set all attributes from config + for key, value in config.items(): + if hasattr(self, key): + setattr(self, key, value) + elif key in locals(): + locals()[key] = value + + # Store parameters + self.input_channels = input_channels + self.seq_length = seq_length + self.num_classes = num_classes + self.cnn_channels = cnn_channels + self.kernel_sizes = kernel_sizes + self.pool_type = pool_type + self.pool_sizes = pool_sizes + self.use_batch_norm = use_batch_norm + self.activation_type = activation + self.dropout_rate = dropout + self.cnn_dropout_rate = cnn_dropout if cnn_dropout is not None else dropout + self.lstm_hidden_dim = lstm_hidden_dim + self.lstm_num_layers = lstm_num_layers + self.bidirectional = bidirectional + self.lstm_dropout_rate = lstm_dropout if lstm_dropout is not None else dropout + self.classifier_hidden_dims = classifier_hidden_dims + self.residual_connections = residual_connections + self.layer_normalization = layer_normalization + self.weight_init = weight_init + self.attention_type = attention_type + self.multi_head_num = multi_head_num + + # Calculate directions + self.num_directions = 2 if bidirectional else 1 + + # Default attention dimension if not provided + self.attention_dim = attention_dim or self.num_directions * lstm_hidden_dim + + # Precalcular dimensiones de secuencia después de las capas CNN + self.input_seq_length = seq_length + self.output_seq_length = None + + if seq_length is not None: + # Calcular reducción de secuencia por pooling + seq_reduction = 1 + current_length = seq_length + + for pool_size in self.pool_sizes: + current_length = (current_length + pool_size - 1) // pool_size # Ceil division + seq_reduction *= pool_size + + self.output_seq_length = current_length + + # Verificar dimensiones válidas + if self.output_seq_length <= 0: + raise ValueError(f"La secuencia resultante después del pooling es demasiado corta. " + f"Secuencia entrada: {seq_length}, reducción: {seq_reduction}") + + # For visualization and explanation + self.cnn_activations = [] + self.lstm_activations = None + self.attention_weights = None + self.gradients = None + self.last_cnn_output = None + self.input = None + + # Create activation function + self.activation = self._get_activation() + + # Create CNN layers + self.cnn_blocks = nn.ModuleList() + in_channels = input_channels + + for i, (out_channels, kernel_size, pool_size) in enumerate(zip(cnn_channels, kernel_sizes, pool_sizes)): + block = nn.ModuleDict() + block["conv"] = nn.Conv1d(in_channels, out_channels, kernel_size=kernel_size, padding=kernel_size // 2) + + if use_batch_norm: + block["bn"] = nn.BatchNorm1d(out_channels) + + if layer_normalization: + # Usamos LayerNorm correctamente para normalizar sobre la dimensión de características + block["ln"] = nn.LayerNorm([out_channels]) + + if pool_type == "max": + block["pool"] = nn.MaxPool1d(kernel_size=pool_size) + else: + block["pool"] = nn.AvgPool1d(kernel_size=pool_size) + + block["dropout"] = nn.Dropout(self.cnn_dropout_rate) + + # Determinar si este bloque puede usar conexión residual + # Solo si tienen mismas dimensiones de entrada y salida + if residual_connections and in_channels == out_channels: + block["has_residual"] = True + else: + block["has_residual"] = False + + self.cnn_blocks.append(block) + in_channels = out_channels + + # LSTM layer + self.lstm = nn.LSTM( + input_size=cnn_channels[-1], + hidden_size=lstm_hidden_dim, + num_layers=lstm_num_layers, + batch_first=True, + bidirectional=bidirectional, + dropout=self.lstm_dropout_rate if lstm_num_layers > 1 else 0 + ) + + # Attention mechanism + lstm_output_dim = self.num_directions * lstm_hidden_dim + if attention_type == "basic": + self.attention = nn.Linear(lstm_output_dim, 1) + elif attention_type == "scaled_dot": + self.query = nn.Linear(lstm_output_dim, self.attention_dim) + self.key = nn.Linear(lstm_output_dim, self.attention_dim) + self.value = nn.Linear(lstm_output_dim, lstm_output_dim) + elif attention_type == "multi_head": + self.mha = nn.MultiheadAttention( + embed_dim=lstm_output_dim, + num_heads=multi_head_num, + batch_first=True + ) + self.attention_ln = nn.LayerNorm(lstm_output_dim) + else: + # Caso por defecto para evitar errores + self.attention = nn.Linear(lstm_output_dim, 1) + print(f"ADVERTENCIA: Tipo de atención '{attention_type}' no reconocido. Usando 'basic'.") + + # Classifier + classifier_layers = [] + in_dim = lstm_output_dim + + for hidden_dim in classifier_hidden_dims: + classifier_layers.append(nn.Linear(in_dim, hidden_dim)) + classifier_layers.append(self.activation) + classifier_layers.append(nn.Dropout(dropout)) + in_dim = hidden_dim + + classifier_layers.append(nn.Linear(in_dim, num_classes)) + self.classifier = nn.Sequential(*classifier_layers) + + # Initialize weights + self._initialize_weights() + + + def _get_activation(self): + if self.activation_type == "relu": + return nn.ReLU() + elif self.activation_type == "leaky_relu": + return nn.LeakyReLU(0.1) + elif self.activation_type == "elu": + return nn.ELU() + elif self.activation_type == "gelu": + return nn.GELU() + else: + return nn.ReLU() + + def _initialize_weights(self): + if self.weight_init == "xavier": + for m in self.modules(): + if isinstance(m, nn.Conv1d) or isinstance(m, nn.Linear): + nn.init.xavier_uniform_(m.weight) + if m.bias is not None: + nn.init.zeros_(m.bias) + elif self.weight_init == "kaiming": + for m in self.modules(): + if isinstance(m, nn.Conv1d) or isinstance(m, nn.Linear): + nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu') + if m.bias is not None: + nn.init.zeros_(m.bias) + + def activations_hook(self, grad): + self.gradients = grad + + def reset_activation_storage(self): + self.cnn_activations = [] + self.lstm_activations = None + self.attention_weights = None + self.gradients = None + self.last_cnn_output = None + + def forward(self, x, return_attention=False, track_gradients=False): + """ + Forward pass del modelo + + Args: + x: Input tensor de forma (batch, channels, seq_length) + return_attention: Si True, devuelve también weights de atención + track_gradients: Si True, registra hook para gradientes (para explicabilidad) + + Returns: + Logits de clasificación y opcionalmente pesos de atención + """ + self.reset_activation_storage() + self.input = x + + # Guardar dimensiones originales para debugging + batch_size, channels, orig_seq_len = x.shape + + # Verificar entrada coherente con la configuración del modelo + if channels != self.input_channels: + print(f"ADVERTENCIA: Número de canales de entrada ({channels}) " + f"difiere del configurado ({self.input_channels})") + + # Almacenar dimensiones después de cada bloque para debugging + dims_after_each_block = [] + + # Process through CNN blocks + for i, block in enumerate(self.cnn_blocks): + # Guardar entrada para posible conexión residual + residual = x if block["has_residual"] else None + + # Forward pass por operaciones del bloque + x = block["conv"](x) + + if "bn" in block: + x = block["bn"](x) + + if "ln" in block: + # Transponemos correctamente para aplicar layer norm + x_transposed = x.transpose(1, 2) # (batch, seq, channels) + x_normalized = block["ln"](x_transposed) + x = x_normalized.transpose(1, 2) # Volver a (batch, channels, seq) + + x = self.activation(x) + + # Aplicar conexión residual si está disponible para este bloque + if residual is not None: + x = x + residual + + # Aplicar pooling (con la lógica corregida) + if not (i == len(self.cnn_blocks) - 1 and self.residual_connections): + x = block["pool"](x) + + x = block["dropout"](x) + self.cnn_activations.append(x.detach()) + + # Guardar dimensiones actuales + dims_after_each_block.append(tuple(x.shape)) + + cnn_output = x + + # Verificar dimensiones finales CNN + final_seq_len = x.shape[2] + if self.output_seq_length is not None and final_seq_len != self.output_seq_length: + print(f"ADVERTENCIA: Longitud secuencia después de CNN ({final_seq_len}) " + f"difiere de la esperada ({self.output_seq_length})") + + if track_gradients and cnn_output.requires_grad: + cnn_output.register_hook(self.activations_hook) + + self.last_cnn_output = cnn_output + + # Reshape for LSTM: (batch, channels, seq) -> (batch, seq, channels) + x = cnn_output.permute(0, 2, 1) + + # LSTM + lstm_out, _ = self.lstm(x) + self.lstm_activations = lstm_out.detach() + + # Declarar vectores que usaremos para todos los tipos de atención + attention_weights = None + context_vector = None + + # Apply attention mechanism según tipo configurado + if self.attention_type == "basic": + attention_scores = self.attention(lstm_out).squeeze(-1) + attention_weights = F.softmax(attention_scores, dim=1) + context_vector = torch.bmm(attention_weights.unsqueeze(1), lstm_out).squeeze(1) + + elif self.attention_type == "scaled_dot": + Q = self.query(lstm_out) + K = self.key(lstm_out) + V = self.value(lstm_out) + + scores = torch.bmm(Q, K.transpose(1, 2)) / np.sqrt(self.attention_dim) + attention_weights = F.softmax(scores, dim=-1) + context_vector = torch.bmm(attention_weights, V).mean(dim=1) + + elif self.attention_type == "multi_head": + # MultiheadAttention devuelve (attn_output, attn_output_weights) + attn_output, attn_output_weights = self.mha(lstm_out, lstm_out, lstm_out) + attention_weights = attn_output_weights + + if self.layer_normalization: + attn_output = self.attention_ln(attn_output + lstm_out) + + context_vector = attn_output.mean(dim=1) + else: + # Caso por defecto: atención uniforme + attention_weights = torch.ones(lstm_out.shape[0], lstm_out.shape[1]).to(lstm_out.device) + attention_weights = attention_weights / lstm_out.shape[1] # Normalizar + context_vector = lstm_out.mean(dim=1) + + # Verificar que attention_weights existe + if attention_weights is None: + attention_weights = torch.ones(lstm_out.shape[0], lstm_out.shape[1]).to(lstm_out.device) + attention_weights = attention_weights / lstm_out.shape[1] # Normalizar + + # Guardar pesos de atención para visualización/interpretación + self.attention_weights = attention_weights.detach() + + # Verificar dimensiones del vector de contexto + if context_vector is None: + context_vector = lstm_out.mean(dim=1) + + # Asegurar dimensionalidad correcta: (batch_size, features) + if len(context_vector.shape) > 2: + print(f"ADVERTENCIA: Vector contexto tiene forma inesperada {context_vector.shape}. " + f"Aplicando mean en dim 1.") + context_vector = context_vector.mean(dim=1) + elif len(context_vector.shape) == 1: + context_vector = context_vector.unsqueeze(0) + + # Verificación final + if len(context_vector.shape) != 2: + print(f"ERROR: Vector contexto debe ser 2D pero es {context_vector.shape}") + # Intentar corregir + if len(context_vector.shape) > 2: + context_vector = context_vector.reshape(batch_size, -1) + + # Classification + out = self.classifier(context_vector) + + if return_attention: + return out, attention_weights + return out + + def get_config(self): + """Returns the current configuration as a dictionary""" + return { + "input_channels": self.input_channels, + "seq_length": self.seq_length, + "num_classes": self.num_classes, + "cnn_channels": self.cnn_channels, + "kernel_sizes": self.kernel_sizes, + "pool_type": self.pool_type, + "pool_sizes": self.pool_sizes, + "use_batch_norm": self.use_batch_norm, + "activation": self.activation_type, + "dropout": self.dropout_rate, + "cnn_dropout": self.cnn_dropout_rate, + "lstm_hidden_dim": self.lstm_hidden_dim, + "lstm_num_layers": self.lstm_num_layers, + "bidirectional": self.bidirectional, + "lstm_dropout": self.lstm_dropout_rate, + "classifier_hidden_dims": self.classifier_hidden_dims, + "attention_dim": self.attention_dim, + "attention_type": self.attention_type, + "multi_head_num": self.multi_head_num, + "residual_connections": self.residual_connections, + "layer_normalization": self.layer_normalization, + "weight_init": self.weight_init + } + + def count_parameters(self): + """Count and return the number of trainable parameters""" + return sum(p.numel() for p in self.parameters() if p.requires_grad) + + def get_intermediate_outputs(self, x): + """Get all intermediate activations for a given input""" + _ = self.forward(x, track_gradients=True) + return { + "cnn_activations": self.cnn_activations, + "lstm_activations": self.lstm_activations, + "attention_weights": self.attention_weights + } + + def visualize_attention(self, x, return_fig=False): + """Visualize attention weights for a given input""" + try: + import matplotlib.pyplot as plt + + _, attention_weights = self.forward(x, return_attention=True) + + if attention_weights is None: + print("No attention weights available") + return None + + batch_size = attention_weights.size(0) + seq_len = attention_weights.size(1) + + fig, axes = plt.subplots(batch_size, 1, figsize=(10, 2*batch_size)) + if batch_size == 1: + axes = [axes] + + for i, ax in enumerate(axes): + weights = attention_weights[i].cpu().detach().numpy() + ax.bar(range(seq_len), weights) + ax.set_title(f"Sample {i+1}") + ax.set_xlabel("Sequence position") + ax.set_ylabel("Attention weight") + + plt.tight_layout() + + if return_fig: + return fig + plt.show() + return None + except ImportError: + print("matplotlib is required for visualization") + return None + + def interpret(self, x, class_idx=None, methods=None): + """ + Enhanced interpretation method with multiple explainability techniques + + Args: + x: Input data tensor + class_idx: Target class indices to explain (defaults to predicted class) + methods: List of methods to use, options: ['gradcam', 'integrated_gradients', + 'occlusion', 'shap', 'feature_ablation', 'all'] + + Returns: + Dictionary with various interpretability outputs + """ + try: + # Try to import Captum components + from captum.attr import IntegratedGradients, Occlusion, GradientShap, LayerGradCam + except ImportError: + raise ImportError("This method requires the 'captum' package. Install with: pip install captum") + + if methods is None: + methods = ['gradcam', 'attention'] # Default methods + if 'all' in methods: + methods = ['gradcam', 'integrated_gradients', 'occlusion', 'shap', + 'feature_ablation', 'attention', 'layer_importance'] + + # Store original training state + was_training = self.training + lstm_was_training = self.lstm.training + + # Set model to evaluation mode for interpretability + self.eval() + self.lstm.train() # needed for CuDNN backward compatibility + + # Base prediction + x.requires_grad_() + self.input = x # Store input for interpretability methods + + logits, attention = self.forward(x, return_attention=True, track_gradients=True) + pred = torch.softmax(logits, dim=1) + + if class_idx is None: + class_idx = pred.argmax(dim=1) + + # Initialize results dictionary + results = { + 'prediction': pred.detach(), + 'class_idx': class_idx, + 'attention_weights': self.attention_weights, + } + + # Apply selected interpretability methods + if 'gradcam' in methods: + for i in range(x.shape[0]): + pred[i, class_idx[i]].backward(retain_graph=True if i < x.shape[0]-1 else False) + + results['feature_importance'] = self.get_feature_importance() + results['temporal_channel_importance'] = self.get_temporal_channel_importance() + results['channel_importance'] = self.get_channel_importance() + results['cnn_activations'] = self.cnn_activations + + # Integrated Gradients + if 'integrated_gradients' in methods: + ig = IntegratedGradients(self.forward_wrapper) + results['integrated_gradients'] = self._compute_integrated_gradients( + ig, x, class_idx) + + # Occlusion analysis + if 'occlusion' in methods: + occlusion = Occlusion(self.forward_wrapper) + results['occlusion'] = self._compute_occlusion(occlusion, x, class_idx) + + # SHAP (GradientSHAP implementation) + if 'shap' in methods: + gradient_shap = GradientShap(self.forward_wrapper) + results['gradient_shap'] = self._compute_gradient_shap(gradient_shap, x, class_idx) + + # Feature ablation (sensitivity analysis) + if 'feature_ablation' in methods: + results['feature_ablation'] = self._feature_ablation_analysis(x, class_idx) + + # Layer importance analysis + if 'layer_importance' in methods: + results['layer_importance'] = self._compute_layer_importance(x, class_idx) + + # Restore original training states + self.train(was_training) + self.lstm.train(lstm_was_training) + + # Clean up to avoid memory issues + self.input = None + torch.cuda.empty_cache() + + return results + + def forward_wrapper(self, x): + """Wrapper for Captum compatibility""" + return self.forward(x) + + def get_feature_importance(self): + """ + Grad-CAM temporal over the output of the last CNN block. + Returns tensor (batch, time) + """ + if self.gradients is None or self.last_cnn_output is None: + return None + + pooled_gradients = torch.mean(self.gradients, dim=[0, 2]) # (channels,) + cam = self.last_cnn_output.clone() + + for i in range(cam.shape[1]): + cam[:, i, :] *= pooled_gradients[i] + + heatmap = torch.mean(cam, dim=1).detach() # (batch, time) + + # Apply ReLU to highlight only positive influences + heatmap = F.relu(heatmap) + + # Normalize heatmap for better visualization + if heatmap.max() > 0: + heatmap = heatmap / heatmap.max() + + return heatmap + + def get_channel_importance(self): + """ + Channel importance: (batch, channels) + """ + if self.input is None or self.input.grad is None: + raise ValueError("Input gradients not available. Call interpret() first.") + return self.input.grad.abs().mean(dim=2).detach() + + def get_temporal_channel_importance(self): + """ + Temporal-channel importance: (batch, channels, time) + """ + if self.input is None or self.input.grad is None: + raise ValueError("Input gradients not available. Call interpret() first.") + return self.input.grad.abs().detach() + + def _compute_integrated_gradients(self, ig, x, class_idx): + """Compute integrated gradients attribution""" + batch_size = x.shape[0] + attributions = [] + + for i in range(batch_size): + baseline = torch.zeros_like(x[i:i+1]) + attr = ig.attribute( + x[i:i+1], baseline, target=class_idx[i].item(), n_steps=50 + ) + attributions.append(attr) + + return torch.cat(attributions).detach() + + def _compute_occlusion(self, occlusion_algo, x, class_idx): + """Compute occlusion-based feature attribution""" + batch_size = x.shape[0] + attributions = [] + + # Define sliding window parameters for temporal data + window_size = min(5, x.shape[2] // 4) # Adapt window size to input length + + for i in range(batch_size): + attr = occlusion_algo.attribute( + x[i:i+1], + sliding_window_shapes=(1, window_size), + target=class_idx[i].item(), + strides=(1, max(1, window_size // 2)) + ) + attributions.append(attr) + + return torch.cat(attributions).detach() + + def _compute_gradient_shap(self, shap_algo, x, class_idx): + """Compute GradientSHAP attributions""" + batch_size = x.shape[0] + attributions = [] + + for i in range(batch_size): + # Create random baselines (typically 10-50 for good estimates) + baselines = torch.randn(10, *x[i:i+1].shape[1:]) * 0.001 + + # Ensure baselines device matches input + baselines = baselines.to(x.device) + + attr = shap_algo.attribute( + x[i:i+1], baselines=baselines, target=class_idx[i].item() + ) + attributions.append(attr) + + return torch.cat(attributions).detach() + + def _feature_ablation_analysis(self, x, class_idx): + """Analyze model by systematically ablating input features""" + batch_size = x.shape[0] + results = [] + + for i in range(batch_size): + # Store original prediction + with torch.no_grad(): + orig_output = self.forward(x[i:i+1]) + orig_prob = torch.softmax(orig_output, dim=1)[0, class_idx[i]].item() + + # Test ablation of each channel + channel_importance = [] + for c in range(self.input_channels): + # Create ablated input (zero out one channel) + ablated_input = x[i:i+1].clone() + ablated_input[:, c, :] = 0 + + # Get prediction on ablated input + with torch.no_grad(): + ablated_output = self.forward(ablated_input) + ablated_prob = torch.softmax(ablated_output, dim=1)[0, class_idx[i]].item() + + # Impact is reduction in probability + channel_impact = orig_prob - ablated_prob + channel_importance.append(channel_impact) + + results.append(torch.tensor(channel_importance)) + + return torch.stack(results) + + def _compute_layer_importance(self, x, class_idx): + """Compute importance of each layer using Layer GradCAM""" + try: + from captum.attr import LayerGradCam + except ImportError: + raise ImportError("This method requires the 'captum' package.") + + batch_size = x.shape[0] + layer_importance = {} + + # Define layers to analyze - adapted for our new ModuleList structure + layers = {} + for i, block in enumerate(self.cnn_blocks): + layers[f'conv{i+1}'] = block['conv'] + + for layer_name, layer in layers.items(): + layer_gradcam = LayerGradCam(self.forward_wrapper, layer) + layer_attrs = [] + + for i in range(batch_size): + attr = layer_gradcam.attribute( + x[i:i+1], target=class_idx[i].item() + ) + # Process attribution to create a single importance score per sample + pooled_attr = torch.mean(attr, dim=1) + + layer_attrs.append(pooled_attr) + + layer_importance[layer_name] = torch.cat(layer_attrs).detach() + + return layer_importance + + def visualize_attributions(self, sample_idx, interpretations, time_axis=None, + channel_names=None, class_names=None): + """ + Visualize the various interpretation results + + Args: + sample_idx: Index of the sample to visualize + interpretations: Dictionary returned by interpret() method + time_axis: Optional array/list with time points for x-axis + channel_names: Optional list of channel names + class_names: Optional list of class names + """ + try: + import matplotlib.pyplot as plt + import numpy as np + except ImportError: + raise ImportError("This method requires matplotlib and numpy for visualization") + + if not channel_names: + channel_names = [f'Channel {i}' for i in range(self.input_channels)] + + if not class_names: + class_idx = interpretations['class_idx'][sample_idx].item() + class_name = f'Class {class_idx}' + else: + class_idx = interpretations['class_idx'][sample_idx].item() + class_name = class_names[class_idx] + + # Set up figure + plt.figure(figsize=(15, 12)) + + # Original input visualization (top row, first column) + plt.subplot(3, 3, 1) + if self.input is not None: + input_data = self.input[sample_idx].cpu().detach().numpy() + if time_axis is not None: + for i in range(input_data.shape[0]): + plt.plot(time_axis, input_data[i], label=channel_names[i]) + else: + for i in range(input_data.shape[0]): + plt.plot(input_data[i], label=channel_names[i]) + plt.legend(loc='best') + plt.title('Input Signal') + plt.xlabel('Time') + plt.ylabel('Value') + + # GradCAM feature importance (top row, second column) + if 'feature_importance' in interpretations and interpretations['feature_importance'] is not None: + plt.subplot(3, 3, 2) + heatmap = interpretations['feature_importance'][sample_idx].cpu().numpy() + if time_axis is not None: + plt.plot(time_axis, heatmap) + else: + plt.plot(heatmap) + plt.title('GradCAM Feature Importance') + plt.xlabel('Time') + plt.ylabel('Importance') + + # Attention weights (top row, third column) + if 'attention_weights' in interpretations and interpretations['attention_weights'] is not None: + plt.subplot(3, 3, 3) + attention = interpretations['attention_weights'][sample_idx].cpu().numpy() + + if time_axis is not None: + # Need to match attention time axis to input time axis + # (account for pooling in the network) + x_points = np.linspace(time_axis[0], time_axis[-1], len(attention)) + plt.plot(x_points, attention) + else: + plt.plot(attention) + plt.title('Attention Weights') + plt.xlabel('Time') + plt.ylabel('Attention') + + # Channel importance (middle row, first column) + if 'channel_importance' in interpretations and interpretations['channel_importance'] is not None: + plt.subplot(3, 3, 4) + ch_importance = interpretations['channel_importance'][sample_idx].cpu().numpy() + plt.bar(channel_names, ch_importance) + plt.title('Channel Importance') + plt.ylabel('Importance') + plt.xticks(rotation=45) + + # Integrated Gradients (middle row, second column) + if 'integrated_gradients' in interpretations: + plt.subplot(3, 3, 5) + ig_attr = interpretations['integrated_gradients'][sample_idx].cpu().numpy() + ig_attr_mean = np.mean(ig_attr, axis=0) # Average across channels for visualization + + if time_axis is not None: + plt.plot(time_axis, ig_attr_mean) + else: + plt.plot(ig_attr_mean) + plt.title('Integrated Gradients') + plt.xlabel('Time') + plt.ylabel('Attribution') + + # Feature Ablation (middle row, third column) + if 'feature_ablation' in interpretations: + plt.subplot(3, 3, 6) + ablation_scores = interpretations['feature_ablation'][sample_idx].cpu().numpy() + plt.bar(channel_names, ablation_scores) + plt.title('Feature Ablation Impact') + plt.ylabel('Probability Change') + plt.xticks(rotation=45) + + # SHAP values (bottom row, first column) + if 'gradient_shap' in interpretations: + plt.subplot(3, 3, 7) + shap_attr = interpretations['gradient_shap'][sample_idx].cpu().numpy() + # Visualize average SHAP value over time + shap_avg = np.mean(shap_attr, axis=0) + + if time_axis is not None: + plt.plot(time_axis, shap_avg) + else: + plt.plot(shap_avg) + plt.title('GradientSHAP Values') + plt.xlabel('Time') + plt.ylabel('SHAP Value') + + # Occlusion analysis (bottom row, second column) + if 'occlusion' in interpretations: + plt.subplot(3, 3, 8) + occlusion_attr = interpretations['occlusion'][sample_idx].cpu().numpy() + occlusion_avg = np.mean(occlusion_attr, axis=0) + + if time_axis is not None: + plt.plot(time_axis, occlusion_avg) + else: + plt.plot(occlusion_avg) + plt.title('Occlusion Analysis') + plt.xlabel('Time') + plt.ylabel('Attribution') + + # Prediction summary (bottom row, third column) + plt.subplot(3, 3, 9) + pred_probs = interpretations['prediction'][sample_idx].cpu().numpy() + classes = list(range(len(pred_probs))) + if class_names: + classes = class_names + plt.bar(classes, pred_probs) + plt.title(f'Prediction: {class_name}') + plt.ylabel('Probability') + plt.ylim([0, 1]) + + plt.tight_layout() + return plt.gcf() + + +# Example of creating a model with custom hyperparameters +def create_model_with_config(**kwargs): + """Helper function to create a model with specified config""" + config = { + "input_channels": 3, + "num_classes": 3, + "cnn_channels": (16, 32, 64), + "kernel_sizes": (5, 3, 3), + "pool_type": "max", + "dropout": 0.1, + "lstm_hidden_dim": 32, + "lstm_num_layers": 1, + "bidirectional": True, + "classifier_hidden_dims": [32], + "attention_type": "basic", + "residual_connections": False, + "layer_normalization": False + } + + # Update config with provided kwargs + config.update(kwargs) + + return CNN_LSTM_Classifier_Tunable(config=config) + From 1aaa8f0ce6af0299807738d7ca346cb6cf9cb92c Mon Sep 17 00:00:00 2001 From: dcajal Date: Mon, 2 Mar 2026 17:48:02 +0100 Subject: [PATCH 04/93] Add some processing scripts and methods --- src/lib/EEG_functions.py | 315 ++++++++++++++++ src/lib/Resp_features.py | 221 ++++++++++++ src/lib/peakedness.py | 634 +++++++++++++++++++++++++++++++++ src/scripts/EEG_Segmenting.py | 193 ++++++++++ src/scripts/EEG_processing.py | 189 ++++++++++ src/scripts/Resp_processing.py | 114 ++++++ src/scripts/ResultsAnalysis.py | 67 ++++ 7 files changed, 1733 insertions(+) create mode 100644 src/lib/EEG_functions.py create mode 100644 src/lib/Resp_features.py create mode 100644 src/lib/peakedness.py create mode 100644 src/scripts/EEG_Segmenting.py create mode 100644 src/scripts/EEG_processing.py create mode 100644 src/scripts/Resp_processing.py create mode 100644 src/scripts/ResultsAnalysis.py diff --git a/src/lib/EEG_functions.py b/src/lib/EEG_functions.py new file mode 100644 index 0000000..5427f60 --- /dev/null +++ b/src/lib/EEG_functions.py @@ -0,0 +1,315 @@ +from scipy.signal import butter, filtfilt +import numpy as np +import plotly.graph_objects as go +from plotly.subplots import make_subplots +from scipy.signal import welch +import pandas as pd +from scipy import signal +from scipy.stats import kurtosis, entropy +import matplotlib.pyplot as plt + +def butter_bandpass_filter(data, lowcut, highcut, fs, order=4): + nyq = 0.5 * fs # Frecuencia de Nyquist + low = lowcut / nyq + high = highcut / nyq + b, a = butter(order, [low, high], btype='bandpass') + # Usamos filtfilt para que no haya desfase en la señal + # w, h = signal.freqz(b, a, worN=8000) + # frequencies = (w * fs) / (2 * np.pi) + # plt.figure(figsize=(10, 5)) + # plt.plot(frequencies, 20 * np.log10(abs(h))) + # plt.xlim(0, highcut + 20) + # plt.ylim(-40, 5) # Para ver bien la caída + # plt.title('Respuesta Frecuencial Digital (Bandpass)') + # plt.xlabel('Frecuencia [Hz]') + # plt.ylabel('Amplitud [dB]') + # plt.grid(which='both', axis='both') + # plt.axvline(lowcut, color='red', linestyle='--', label='Lowcut') + # plt.axvline(highcut, color='red', linestyle='--', label='Highcut') + # plt.legend() + # plt.show() + y = filtfilt(b, a, data) + return y + +def plot_EEG(df, columns, fs = 200): + + fig = make_subplots(rows=len(columns), cols=1, + shared_xaxes=True, + vertical_spacing=0.02, + subplot_titles=columns) + limit = int(3000 * fs) + x = np.arange(df[0].shape[0]) / fs # Asumiendo fs=100Hz, ajusta si es diferente + downsample = 10 # Factor de downsampling para mejorar rendimiento (ajusta según necesidad) + for i, col in enumerate(columns): + fig.add_trace( + go.Scattergl(x=x[:limit:downsample], y=df[i][:limit:downsample], name=col, mode='lines'), + row=i+1, col=1 + ) + fig.update_layout( + height=900, + title_text="Polisomnografía - Canales EEG", + showlegend=False, + template="plotly_white" + ) + fig.update_xaxes(title_text="Tiempo (segundos)", row=len(columns), col=1) + fig.show() + +def plot_EEG_sel(sel, name = "EEG_plot_raw.html"): + fig = make_subplots(rows=len(sel), cols=1, + shared_xaxes=True, + vertical_spacing=0.02, + subplot_titles=[ch[1].label for ch in sel]) + + for i, (idx, sig) in enumerate(sel): + # Crear eje de tiempo en segundos + fs = sig.sampling_frequency + time = np.linspace(0, len(sig.data) / fs, len(sig.data)) + + # Añadir traza (solo mostramos los primeros 30s por defecto para no saturar el navegador) + # Puedes quitar el slice [:int(30*fs)] para ver todo, pero cuidado con el rendimiento + limit = int(3000 * fs) + # limit = len(sig.data) if limit > len(sig.data) else limit + # limit = len(sig.data) + # downsample = 10 # Factor de downsampling para mejorar rendimiento (ajusta según necesidad) + fig.add_trace( + go.Scattergl(x=time[:limit], y=sig.data[:limit], name=sig.label, mode='lines'), + row=i+1, col=1 + ) + + fig.update_layout( + height=900, + title_text="Polisomnografía - Canales EEG", + showlegend=False, + template="plotly_white" + ) + + fig.update_xaxes(title_text="Tiempo (segundos)", row=len(sel), col=1) + fig.write_html(f"graphs/{name}.html") # Guardar como HTML para visualización interactiva + # fig.show() + +def filtering_and_normalization(sig, sig_fs): + b, a = signal.butter(4, 0.3, btype='highpass', fs=sig_fs) + sig_filtered = signal.filtfilt(b, a, sig) + b, a = signal.butter(4, 35, btype='lowpass', fs=sig_fs) + sig_filtered = signal.filtfilt(b, a, sig_filtered) + sig_filtered = normalize(sig_filtered) + return sig_filtered + +def normalize(x): + return (x - np.mean(x)) / np.std(x) + +def remove_impulse_artifacts(sig): + # Square of second derivative + aux = np.diff(np.diff(sig)) ** 2 + aux = np.insert(aux, 0, aux[0]) + aux = np.append(aux, aux[-1]) + + # Median filter threshold + wind = 999 + if aux.size < wind: + wind = aux.size + if (wind % 2) != 1: + wind = wind - 1 + mf = signal.medfilt(aux, wind) + + # Find impulses + margin = 20 + impulses = np.asarray(np.where(aux > mf + 0.005)).ravel() + for impulse in impulses: + impulses = np.append(impulses, np.arange(impulse - margin, min(impulse + margin+1, sig.size))) + impulses = np.sort(impulses) + impulses = np.unique(impulses) + impulses = impulses[impulses >= 0] + + # Remove impulses + output = sig + output[impulses] = np.nan + return output + +def clean_movement_artifacts(data, fs, threshold_z=10, window_ms=500): + """ + Identifica y limpia artefactos de gran amplitud. + + Args: + data: Array de la señal. + fs: Frecuencia de muestreo. + threshold_z: Umbral de desviaciones estándar para marcar como artefacto. + window_ms: Tiempo alrededor del artefacto a limpiar para asegurar + que eliminamos la subida y bajada del pico. + """ + cleaned_data = data.copy() + + # 1. Calcular Z-Score de la amplitud + z_scores = np.abs((data - np.mean(data)) / np.std(data)) + + # 2. Encontrar índices que superan el umbral + mask = z_scores > threshold_z + + # 3. Expandir la máscara (el movimiento suele durar un poco más que el pico) + padding = int((window_ms / 1000) * fs) + expanded_mask = np.convolve(mask, np.ones(padding), mode='same') > 0 + + # 4. Reemplazar artefactos con el valor medio (0 si está centrada) + cleaned_data[expanded_mask] = 0 + + artifacts_percentage = (np.sum(expanded_mask) / len(data)) * 100 + print(f"Artefactos eliminados: {artifacts_percentage:.2f}% de la señal.") + + return cleaned_data + +def adaptive_variance_cleaner(data, fs, win_size_ms=500, alpha=0.1, threshold=3.5): + """ + Filtro adaptativo que detecta artefactos cuando la varianza local + excede significativamente la varianza histórica adaptativa. + + Args: + data: Array de la señal (1D). + fs: Frecuencia de muestreo. + win_size_ms: Tamaño de la ventana para calcular la varianza local. + alpha: Factor de adaptación (0 a 1). Cuanto más alto, más rápido olvida el pasado. + threshold: Multiplicador de la varianza adaptativa para marcar artefacto. + """ + win_samples = int((win_size_ms / 1000) * fs) + n_samples = len(data) + cleaned_data = np.copy(data) + + # Inicializamos la varianza adaptativa con la varianza de la primera ventana + first_win = data[:win_samples] + adaptive_var = np.var(first_win) + + # Para guardar dónde detectamos artefactos + artifact_mask = np.zeros(n_samples, dtype=bool) + + # Iteramos por ventanas + for i in range(0, n_samples - win_samples, win_samples): + current_win_idx = slice(i, i + win_samples) + current_var = np.var(data[current_win_idx]) + + # Si la varianza actual es mucho mayor que la adaptativa, es un artefacto + if current_var > threshold * adaptive_var: + artifact_mask[current_win_idx] = True + cleaned_data[current_win_idx] = 0 # O podrías interpolar + # No actualizamos la varianza adaptativa con un artefacto para no "contaminarla" + else: + # Actualización adaptativa (Exponential Moving Average) + adaptive_var = alpha * current_var + (1 - alpha) * adaptive_var + + return cleaned_data, artifact_mask + +def create_epochs(data, fs, epoch_duration=30): + samples_per_epoch = int(fs * epoch_duration) + num_epochs = len(data) // samples_per_epoch + + # Recortamos la señal para que sea divisible exactamente + data_trimmed = data[:num_epochs * samples_per_epoch] + + # Reshape: (Número de épocas, Puntos por época) + epochs = data_trimmed.reshape(num_epochs, samples_per_epoch) + return epochs + +def extract_band_powers(epochs, fs, win_len = 2): + features = [] + complexities = [] + # Definición de las bandas + bands = { + 'Delta': (0.5, 4), + 'Theta': (4, 8), + 'Alpha': (8, 12), + 'Sigma': (11, 16), + 'Beta': (12, 30) + } + + for epoch in epochs: + # Calcular PSD + freqs, psd = welch(epoch, fs, nperseg=fs*30) # Ventanas de 2 seg para buena resolución + # Plot de PSD para verificar que las bandas se ven bien (opcional) + # plt.semilogy(freqs, psd) + # plt.show() + epoch_features = {} + for band_name, (low, high) in bands.items(): + # Encontrar índices de frecuencia para la banda actual + idx_band = np.logical_and(freqs >= low, freqs <= high) + # Calcular la potencia media en esa banda + epoch_features[band_name] = np.mean(psd[idx_band]) + + features.append(epoch_features) + + diff = np.diff(epoch) + mobility = np.sqrt(np.var(diff) / np.var(epoch)) + # 2. Complejidad de Hjorth: Qué tan similar es la señal a una onda senoidal + diff2 = np.diff(diff) + mobility_diff = np.sqrt(np.var(diff2) / np.var(diff)) + complexity = mobility_diff / mobility if mobility > 0 else 0 + complexities.append({'Hjorth_Mobility': mobility, 'Hjorth_Complexity': complexity}) + + return pd.DataFrame(features), pd.DataFrame(complexities) + +def get_patient_profile(df_features): + # 1. Calcular Potencia Total por época + total_power = df_features.sum(axis=1) + avg_p = df_features.mean() + total_avg_p = avg_p.sum() + + # 2. Variabilidad (Refleja microdespertares y fragmentación) + # Coeficiente de Variación (CV = std/mean) para normalizar por amplitud + variability = df_features.std() / df_features.mean() + variability.index = ['CV_' + col for col in variability.index] + + # 3. Curtosis (Picos súbitos de actividad) + kurt = df_features.apply(kurtosis) + kurt.index = ['Kurt_' + col for col in kurt.index] + + # 4. Índices de potencia relativa específicos + rel_delta = avg_p['Delta'] / total_avg_p + + # 5. Ratios de enlentecimiento + tar = avg_p['Theta'] / avg_p['Alpha'] # Theta-Alpha Ratio + tbr = avg_p['Theta'] / avg_p['Beta'] # Theta-Beta Ratio + + # 6. Entropía Espectral (Complejidad del perfil de potencia promedio) + # Cuanto más baja, más "pobre" es la diversidad de frecuencias del cerebro + spec_entropy = entropy(df_features) + + # 2. Calcular Potencias Relativas (promedio de toda la noche) + rel_powers = df_features.div(total_power, axis=0).mean() + rel_powers.index = ['Rel_' + col for col in rel_powers.index] + + # Calculate main frecuencies of oscilation on each band (peak frequency) + # Esto puede ser un buen indicador de cambios en la arquitectura del sueño + # peak_freqs = {} + # for band in ['Delta', 'Theta', 'Alpha', 'Sigma', 'Beta']: + # freqs, psd = welch(df_features[band], fs=1/30, nperseg=25, noverlap = 25 // 2, nfft=1024) # fs=1/30 porque cada punto es un promedio de 30s + # idx_peak = np.argmax(psd) + # peak_freqs['PeakFreq_' + band] = freqs[idx_peak] + + # 3. Calcular Ratios Críticos + # Usamos la media de las potencias absolutas para el ratio global + avg_p = df_features.mean() + ratios = { + 'Ratio_Theta_Alpha': avg_p['Theta'] / avg_p['Alpha'], + 'Ratio_Slow_Fast': (avg_p['Delta'] + avg_p['Theta']) / (avg_p['Alpha'] + avg_p['Beta']), + 'Sigma_Stability': df_features['Sigma'].std() / df_features['Sigma'].mean(), + 'Spectral_Entropy_delta': spec_entropy[0], + 'Spectral_Entropy_theta': spec_entropy[1], + 'Spectral_Entropy_alpha': spec_entropy[2], + 'Spectral_Entropy_sigma': spec_entropy[3], + 'Spectral_Entropy_beta': spec_entropy[4], + 'Theta_Alpha_Ratio': tar, + 'Theta_Beta_Ratio': tbr, + 'Relative_Delta_Power': rel_delta, + 'kurtosis_Delta': kurt['Kurt_Delta'], + 'kurtosis_Theta': kurt['Kurt_Theta'], + 'kurtosis_Alpha': kurt['Kurt_Alpha'], + 'kurtosis_Sigma': kurt['Kurt_Sigma'], + 'kurtosis_Beta': kurt['Kurt_Beta'], + 'variability_Delta': variability['CV_Delta'], + 'variability_Theta': variability['CV_Theta'], + 'variability_Alpha': variability['CV_Alpha'], + 'variability_Sigma': variability['CV_Sigma'], + 'variability_Beta': variability['CV_Beta'], + + } + + # Combinar todo en una sola fila + profile = pd.concat([rel_powers, pd.Series(ratios)]) + return profile \ No newline at end of file diff --git a/src/lib/Resp_features.py b/src/lib/Resp_features.py new file mode 100644 index 0000000..25c4706 --- /dev/null +++ b/src/lib/Resp_features.py @@ -0,0 +1,221 @@ +import pandas as pd +import numpy as np +import plotly.graph_objs as go +from lib.peakedness import peakednessCost +from scipy.interpolate import interp1d +import matplotlib.pyplot as plt +from scipy.stats import kruskal +from scipy.signal import resample, detrend +import scipy.fft as fft +from scipy.signal import butter, filtfilt + +def plot_resp(Data, subjet = 1, DownPrinting = 2): + """ + Plot resp data using Plotly. + """ + if type(Data) == dict: + Data = pd.DataFrame(Data[str(subjet)]) + Data = Data.iloc[::DownPrinting, :] + end = -1 + elif type(Data) == type(pd.DataFrame()): + Data = Data[Data['Subjet'] == str(subjet)] + Data = Data.iloc[::DownPrinting, :] + end = -2 + + # Data.reset_index(drop=True, inplace=True) + print(len(Data.columns)) + fig = go.Figure() + for c in Data.columns[:end]: + fig.add_trace(go.Line(x=Data.Time, y=Data[c], name = c)) + fig.update_layout(title_text='EDA Data', title_x=0.5) + + fig.show() + +def peakedness_application(Data, stage, plotflag = False, subjet = 1): + # print("Compute BR") + fs = 100 + Setup = {} + Setup["K"] = 5 + Setup["DT"] = 5 + Setup["Ts"] = 60 #interval length of Welch periodograms (s) + Setup["Tm"] = 20 #interval length of subintervals for Welch periodograms (s) + # Setup["d"] = 0.1 #interval length of subintervals for Welch periodograms (s) + Setup["Omega_r"] = np.array([5, 20])/60 #respiratory rate range in Hz + Setup["plotflag"] = plotflag + Setup["Nfft"] = np.power(2,14) + tsBR = np.arange(0,Data.shape[0]/fs,1/fs) + + if tsBR.shape[0] != Data.shape[0]: + # print(f"tsBR.shape[0]: {tsBR.shape[0]}, Data.shape[0]: {Data.shape[0]}") + tsBR = np.arange(0,Data.shape[0]/fs,1/fs)[:Data.shape[0]] + + hat_Br, Sk_Br, t_aver = peakednessCost(Data, tsBR, fs, Setup, title = stage, storeGraph = False, subjet = subjet) + # print(f"hat_Br: {hat_Br}, Sk_Br: {Sk_Br}, bar_Br: {bar_Br}, t_aver_Br: {t_aver_Br}, f_Br: {f_Br}, used_Br: {used_Br}") + + # print(hat_Br) + return hat_Br, Sk_Br, t_aver + +# Butterworth low-pass filter +def lowpass_filter(signal, fs, cutoff=2.0, order=4): + nyq = 0.5 * fs + normal_cutoff = cutoff / nyq + b, a = butter(order, normal_cutoff, btype='low', analog=False) + return filtfilt(b, a, signal) + +def Metrics_per_segment(Data): + """ + Compute peakedness per segment. + """ + + # Results = pd.DataFrame(columns=['Subject', 'Stage', 'Peakedness', 'Slope', 'Intercept', 'Relative Peak', 'Bocanada', 'Contraction', "TidalVolume", "Complexity", "Mobility", "Activity"]) + Results = pd.DataFrame() + + for subjet in Data['Subjet'].unique(): + sel_sujeto = Data[Data['Subjet'] == subjet] + sel_sujeto_ref = sel_sujeto.iloc[:,:-2] + Sol_subject = [] + Sol_interSubject = [] + for secc in sel_sujeto_ref.columns: + if secc == 'Time': + continue + else: + section = sel_sujeto[secc].values + section = section[~np.isnan(section)] + + hat_Br, Sk_Br, t_aver = peakedness_application(section, stage=secc, plotflag = False, subjet= subjet) + # print(f"Subjet: {subjet}, section: {secc} hat_Br: {hat_Br}, Sk_Br: {Sk_Br}") + + # Ajuste lineal + coef = np.polyfit(t_aver, hat_Br, 1) # Grado 1 = línea recta + pendiente, interseccion = coef + # print(f"Pendiente: {pendiente:.6f}, Intersección: {interseccion:.6f}") + + #Picudez relativa + rel_peak_list = [] + for ti in range(len(t_aver)): + f_max = np.argmax(Sk_Br[:,ti]) + rel_peak_list.append(np.sum(Sk_Br[f_max-1:f_max+1,ti]) / np.sum(Sk_Br[:,ti])) + real_peak = np.mean(rel_peak_list) + + # Derivada + diff = np.diff(section) + bocanada = max(np.percentile(diff, 90), np.abs(np.percentile(diff, 10))) + + Contraction = np.percentile(np.abs(diff), 10) + + #Tidal Volume + TidalVolume = max(np.percentile(section, 99), np.abs(np.percentile(section, 1))) + + # Calculate derivatives + dx = np.diff(section) + ddx = np.diff(dx) + + # Calculate variance and its derivatives + x_var = np.var(section) # = activity + dx_var = np.var(dx) + ddx_var = np.var(ddx) + + # Mobility and complexity + mobility = np.sqrt(dx_var / x_var) + complexity = np.sqrt(ddx_var / dx_var) / mobility + + + filtered_signal = lowpass_filter(section, 100, cutoff=2.0, order=4) + segment4Hz = resample(filtered_signal, int(filtered_signal.size/100*4)) # Resample to 4Hz + + fft_signal = fft.fft(detrend(segment4Hz), n=2**12) + power = np.abs(fft_signal)**2 + freqs = fft.fftfreq(2**12, d = 1/4) + max_freq_index = np.argmax(power) + max_freq = freqs[max_freq_index] + power_at_max_freq = power[max_freq_index-51:max_freq_index+51].sum() + power_ratio = power_at_max_freq / np.sum(power[:len(power)//2]) + + Sol = [subjet, secc[:secc.find('_')], np.mean(hat_Br), pendiente, interseccion, real_peak, bocanada, Contraction, TidalVolume, complexity, mobility, x_var, max_freq, power_ratio] + + Sol_subject.append(Sol) + + + Sol_subject = pd.DataFrame(Sol_subject, columns=['Subject', 'Stage', 'Peakedness', 'Slope', + 'Intercept', 'Relative Peak', 'Bocanada', + "Contraction","TidalVolume", "Complexity", + "Mobility", "Activity", "Max_freq", "Power_ratio"]) + + peakmean = Sol_subject['Peakedness'].mean() + peakmin = Sol_subject['Peakedness'].min() + peakmax = Sol_subject['Peakedness'].max() + + slopemean = Sol_subject['Slope'].mean() + slopemin = Sol_subject['Slope'].min() + slopemax = Sol_subject['Slope'].max() + + Rel_peak_mean = Sol_subject['Relative Peak'].mean() + + Bocanada_max = Sol_subject['Bocanada'].max() + Contraction_max = Sol_subject['Contraction'].max() + TidalVolume_max = Sol_subject['TidalVolume'].max() + + Rel_metrics = ['Subject', 'Stage',"Peakmean", "Peakmin", "Peakmax", "Slopemean", "Slopemin", "Slopemax", "Rel_peak_mean", "Bocanada_max", "Contraction_max", "TidalVolume_max"] + # Rel_metrics = ["Peakmean", "Peakmin", "Peakmax", "Slopemean", "Slopemin", "Slopemax", "Rel_peak_mean", "Bocanada_max", "Contraction_max", "TidalVolume_max"] + + Sol_interSubject_DF = pd.DataFrame(Sol_interSubject, columns=Rel_metrics) + Sol_interSubject_DF = pd.DataFrame(Sol_interSubject, columns=Rel_metrics[2:]) + for i in Sol_subject.index: + # Sol_interSubject_DF.at[i,Rel_metrics[0]] = Sol_subject.iloc[i,0] + # Sol_interSubject_DF.at[i,Rel_metrics[1]] = Sol_subject.at[i,'Stage'] + Sol_interSubject_DF.at[i,Rel_metrics[2]] = Sol_subject.at[i,'Peakedness']/peakmean + Sol_interSubject_DF.at[i,Rel_metrics[3]] = Sol_subject.at[i,'Peakedness']/peakmin + Sol_interSubject_DF.at[i,Rel_metrics[4]] = Sol_subject.at[i,'Peakedness']/peakmax + Sol_interSubject_DF.at[i,Rel_metrics[5]] = Sol_subject.at[i,'Slope']/slopemean + Sol_interSubject_DF.at[i,Rel_metrics[6]] = Sol_subject.at[i,'Slope']/slopemin + Sol_interSubject_DF.at[i,Rel_metrics[7]] = Sol_subject.at[i,'Slope']/slopemax + Sol_interSubject_DF.at[i,Rel_metrics[8]] = Sol_subject.at[i,'Relative Peak']/Rel_peak_mean + Sol_interSubject_DF.at[i,Rel_metrics[9]] = Sol_subject.at[i,'Bocanada']/Bocanada_max + Sol_interSubject_DF.at[i,Rel_metrics[10]] = Sol_subject.at[i,"Contraction"]/Contraction_max + Sol_interSubject_DF.at[i,Rel_metrics[11]] = Sol_subject.at[i,"TidalVolume"]/TidalVolume_max + + Sol = pd.concat([Sol_subject, Sol_interSubject_DF], axis=1) + Results = pd.concat([Results, Sol], ignore_index=True) + + + + return Results + +def Significance_tests(RespData): + """ + Compute significance tests for the features. + """ + results = {} + for metrica in RespData.columns[2:]: + # print(f"Realizando prueba de Kruskal-Wallis para la métrica: {metrica}") + # Realizar la prueba de Kruskal-Wallis + estadistico, p_valor = kruskal( + np.array(RespData[RespData.Stage == "Baseline"][metrica].reset_index(drop=True)), + np.array(RespData[RespData.Stage == "LOW"][metrica].reset_index(drop=True)), + np.array(RespData[RespData.Stage == "HIGH"][metrica].reset_index(drop=True)), + np.array(RespData[RespData.Stage == "REST"][metrica].reset_index(drop=True)) + ) + + # Imprimir resultados + + # print(f"Estadístico de Kruskal-Wallis: {estadistico}") + print(f"Metrica: "+metrica+" tiene un valor p: {p_valor}") + results[metrica] = p_valor + # if p_valor < 0.05: + # print("Se rechaza la hipótesis nula: hay diferencias significativas entre los grupos.") + # else: + # print("No se rechaza la hipótesis nula: no hay diferencias significativas entre los grupos.") + + results = pd.DataFrame.from_dict(results, orient='index', columns=['p_value']) + results = results.reset_index() + results.to_excel('./Graphs/kruskal_results.xlsx', index=False) + + plt.plot(results['index'], results['p_value']) + plt.axhline(y=0.05, color='r', linestyle='--') + plt.xlabel('Métrica') + plt.ylabel('Valor p') + plt.title('Resultados de la prueba de Kruskal-Wallis') + plt.xticks(rotation=90) + plt.tight_layout() + plt.savefig('./Graphs/kruskal_results.png') + plt.show() \ No newline at end of file diff --git a/src/lib/peakedness.py b/src/lib/peakedness.py new file mode 100644 index 0000000..6323aca --- /dev/null +++ b/src/lib/peakedness.py @@ -0,0 +1,634 @@ +import pandas as pd +import numpy as np +from numpy.fft import fftshift +from numpy.fft import fft +from scipy.signal import detrend, find_peaks +import matplotlib.pyplot as plt +import plotly.graph_objs as go +from plotly import subplots +from time import time +import os + +def setParamFr(Setup): + if 'DT' not in Setup.keys(): + Setup["DT"] = 5 + DT = 5 + else: + DT = Setup["DT"] + + if 'Ts' not in Setup.keys(): + Setup["Ts"] = 42 + Ts = 42 + else: + Ts = Setup["Ts"] + + if 'Tm' not in Setup.keys(): + Setup["Tm"] = 12 + Tm = 12 + else: + Tm = Setup["Tm"] + + if 'Nfft' not in Setup.keys(): + Setup["Nfft"] = np.power(2,12) + Nfft = np.power(2,12) + else: + Nfft = Setup["Nfft"] + + if 'K' not in Setup.keys(): + Setup["K"] = 5 + K = 5 + else: + K = Setup["K"] + + if 'Omega_r' not in Setup.keys(): + Setup["Omega_r"] = np.array([0.04, 1]) + Omega_r = np.array([0.04, 1]) + else: + Omega_r = Setup["Omega_r"] + + if 'ksi_p' not in Setup.keys(): + Setup["ksi_p"] = 45 + ksi_p = 45 + else: + ksi_p = Setup["ksi_p"] + + if 'N_k' not in Setup.keys(): + Setup["N_k"] = 4 + N_k = 4 + else: + N_k = Setup["N_k"] + + if 'ksi_a' not in Setup.keys(): + Setup["ksi_a"] = 85 + ksi_a = 85 + else: + ksi_a = Setup["ksi_a"] + + if 'd' not in Setup.keys(): + Setup["d"] = 0.125 + d = 0.125 + else: + d = Setup["d"] + + if 'b' not in Setup.keys(): + Setup["b"] = 0.8 + b = 0.8 + else: + b = Setup["b"] + + if 'a' not in Setup.keys(): + Setup["a"] = 0.5 + a = 0.5 + else: + a = Setup["a"] + + if 'plotflag' not in Setup.keys(): + Setup["plotflag"] = False + plotflag =False + else: + plotflag = Setup["plotflag"] + + + return [ DT, Ts, Tm, Nfft, K, Omega_r, ksi_p, ksi_a, d, b, a, N_k, plotflag, Setup] + +def extract_interval( x, t, int_ini, int_end ): + # EXTRACT_INTERVAL Very simple function to extract an interval from a signal + # + # Created by Jesús Lázaro in 2011 + # -------- + # Sintax: [ x_int, t_int, indexes ] = extract_interval( x, t, int_ini, int_end ) + # In: x = signal + # t = time vector + # int_ini = interval begin time (same units as 't') + # int_end = interval end time (same units as 't') + # + # Out: x_int = interval [int_ini, int_end] of 'x' + # t_int = interval [int_ini, int_end] of 't' + # indexes = indexes corresponding to returned time interval + + x_int = x[(t>=int_ini) & (t <=int_end)] + t_int = t[(t>=int_ini) & (t <=int_end)] + + return [ x_int, t_int ] + +def normalizar_PSD( PSD, f = 'default', rango = 'default'): + # NORMALIZAR_PSD Normaliza una densidad espectral de potencia en el rango + # de frecuencias requerido. + # + # Created by Jesús Lázaro in 2011 + # ------- + # Sintax: [ PSD_norm, f_PSD_norm, factor_norm ] = normalizar_PSD( PSD, f, rango ) + # In: PSD = Densidad espectral de potencia + # f = Vector de frecuencias para PSD [Por defecto: frecuencias digitales] + # rango = Rango [f1, f2] en el que se aplicar� la normalizaci�n [Por defecto: Todo f] + # + # Out: PSD_norm = Densidad espectral de potencia notmalizada + # f_PSD_norm = Vector de frecuencias para PSD_norm + # factor_norm = Factor de normalizaci�n utilizado + + if f == 'default': + f = np.arange(0,PSD.shape[0]) / PSD.shape[0] - 1/2 + + if rango == 'default': + rango = [f[0], f[-1]] + + + # Seleccionar rango de inter�s: + f_PSD_norm = f[(f>=rango[0]) & (f<=rango[1])] + PSD = PSD[(f>=rango[0]) & (f<=rango[1])] + if ~f_PSD_norm.any(): # El vector de frecuencias no estaba ordenado + print('El vector de frecuencias debe estar ordenado de forma ascendente'); + + + # Calcular factor de normalizaci�n y normalizar: + ##ADD NAN removal from PSD + factor_norm = sum(PSD) + # print("factor_norm "+ str(factor_norm)) + if factor_norm == 0: + # print("stop") // IMPORTANT MODIFICATION TODO + PSD_norm = PSD + else: + PSD_norm = PSD/factor_norm + + return [ PSD_norm, f_PSD_norm, factor_norm ] + +def init_module(kk,vars,param, plotflag): + # function vars = init_module(kk,vars,param, plotflag) + # This function is used for initialization and reinitialization of bar_fr + Skl = vars["Skl"] + t_orig = vars["t_orig"] + t_aver = vars["t_aver"] + f = vars["f"] + L = vars["L"] + + DT = param["DT"] + K = param["K"] + ksi_p = param["ksi_p"] + d = param["d"] + + # Increment of number of spectra for averaging + if kk == 0: # INITIALIZATION + N = 4*np.floor(K/2) + else: # RE-INITIALIZATION + N = 2*np.floor(K/2) + + ###### Peakedness Analysis : + # Indexes of original spectra that take part in the average + O = np.bitwise_and(t_orig>=t_aver[kk]-N*DT, t_orig<=t_aver[kk]+N*DT) + W = np.arange(O.shape[0]) + O = W[O] + # W = np.ones([O.shape[0]]) + # O1 = W[O] + # Pre-allocate + Xkl = np.empty((O.shape[0], L)) + Xkl[:] = np.nan + for k in range(O.shape[0]): + for l in range(L): + S = Skl[:, O[k], l] + # print(S.shape) + # Use as reference for Pkl calculation the absolute maximum + i_m = S.argmax() + fr_max = f[i_m] + + # Define the Omega, Omega_p bands + Omega = np.bitwise_and(f>=fr_max-d, f<=fr_max+d) + + # Modified limits for initialization (reduces the risk for 0.1 Hz) + Omega_p = np.bitwise_and(f>=max(fr_max-0.4*d,0.15), f<=min(fr_max+0.4*d,0.8)) + + # Peakedness + # print(S[Omega]) + Pkl = 100*sum(S[Omega_p])/sum(S[Omega]) + + if Pkl >= ksi_p: + Xkl[k,l] = 1 + else: + Xkl[k,l] = 0 + + # Initialization for averaged spectrum (if cannot be defined) + if L>1: + averS = np.mean(np.squeeze(np.mean(Skl[:, O, :],1)),1) + else: + averS = np.mean(np.mean(Skl[:, O, :],1),1) + + + if kk == 0: #INITIALIZATION + if np.sum(Xkl[:]) > 0: # One or more spectra were peaked enough + + # Sum all peaky spectra + averS = np.zeros((f.shape[0], 1)) + for k in range(O.shape[0]): + for l in range(L): + if Xkl[k, l] == 1: + averS = averS + Skl[:, O[k], l] + + # Select the maximum in the spectrum + i_m = averS.argmax() + + # Save in vars + vars["bar_fr"][0] = f[i_m] + + else: # RE-INITIALIZATION + # One or more spectra were peaked enough + if np.sum(Xkl[:]) > 0: + # Sum all peaky spectra + averS = np.zeros(f.shape[0]) + for k in range(O.shape[0]): + for l in range(L): + if Xkl[k,l] == 1: + averS = averS + Skl[:, O[k], l] + + # Local maxima in the averaged spectrum + j_pk = find_peaks(averS) + j_pk = j_pk[0] + pk = averS[j_pk] + # Extra restriction : consider peaks with important power + # j_del = pk<0.5*np.max(averS) # IMPORTANTE TODO + j_del = pk<0.2*np.max(averS) + pk = pk[~j_del] + j_pk = j_pk[~j_del] + + # Cost function for deviation from previous fr and maximum power + C_a = 1-np.transpose(pk)/np.max(S) + fr_prev = vars["bar_fr"][np.max(kk,0)] + C_f = abs(f[j_pk[:]]-fr_prev)/(2*d) + # C_f = abs(f(i_pk(:))-fr_prev)/(Omega_r(2)-Omega(1)); + + C = C_a +C_f + try: + j_min = C.argmin() + fj = j_pk[j_min] + vars["bar_fr"][kk] = f[fj] + except: + vars["bar_fr"][kk] = 0 + # Save in vars + # vars["bar_fr"][kk] = f[fj] + + if plotflag: + plt.plot(f, averS) + plt.plot(f[fj], averS[fj], '-') + plt.title('Initialization - Averaged Spectrum') + plt.show() + + return vars + # # No spectra fulfill the initialization + # if plotflag: + # keyboard + +def compute_Xkl( Skl, f, bar_fr, O, ksi_p, ksi_a, d): + # function [ Xkl ] = compute_Xkl( Skl, f, bar_fr, O, ksi_p, ksi_a, d) + # Created by Spyros Kontaxis in 2019 + # Computation of peakedness for a power spectrum + # Sintax: [ Xkl ] = compute_Xkl( Skl, f, bar_fr, O, ksi_p, ksi_a, d) + # Inputs: + # Skl : Welch TF maps in a 3D matrix (f x t x DR signals) + # f : frequency vector (Hz) + # bar_fr : smoothed estimate of the respiratory rate (Hz) + # O : Indexes of original spectra that take part in the average + # ksi_p : peakedness threshold based on power concentration (%) + # ksi_a : peakedness threshold based on absolute maximum (%) + # d : half bandwith of Omega centered around bar_fr (Hz) + # Outputs: + # Xkl : 1-> the o:th spectrum will be used in the average + # 0-> the o:th spectrum will not be used in the average + # + + # % Define two search window arround the estimated respiratory rate + Omega = np.bitwise_and(f>=bar_fr-d, f<=bar_fr+d) + Omega_p = np.bitwise_and(f>=bar_fr-0.4*d, f<=bar_fr+0.4*d) + + # % Get the ammount of signals + L = Skl.shape[2] + + # % Pre-allocate + Xkl = np.zeros((O.shape[0],L)) + + # % Loop over all segments + for k in range(O.shape[0]): + + # % Loop over all signals + for l in range(L): + # % Select the power spectrum of one segment + S = Skl[:, O[k], l] + + # % Define peakedness based on the power concentration + Pkl = 100*sum(S[Omega_p])/sum(S[Omega]) + + # % Define peakedness based on the absolute maximum + # print(max(S)) + Akl = 100*max(S[Omega])/max(S) + # % If the spectrum is concidered peaky by both conditions, mark as + # % peaky + if np.bitwise_and(Pkl >= ksi_p, Akl >= ksi_a): + Xkl[k,l] = 1 + else: + Xkl[k,l] = 0 + + return Xkl + +def compute_fJmin( S, f, bar_fr, d): + # function [ fJmin ] = compute_fJmin( S, f, bar_fr, d) + # Created by Spyros Kontaxis in 2019 + # Spectral peak selection based on cost function + # Sintax: [ fJmin ] = compute_fJmin( S, f, bar_fr, d) + # Inputs: + # S : Averaged Spectrum + # f : frequency vector (Hz) + # bar_fr : smoothed estimate of the respiratory rate (Hz) + # d : half bandwith of Omega centered around bar_fr (Hz) + # Outputs: + # fJmin : respiratory rate estimate + # + + # Define the search window + Omega = np.bitwise_and(f >= bar_fr-d, f <= bar_fr+d) + + # Pre-allocate + fJmin = np.nan + + # Locate peaks in the search window + [peaks, properties] = find_peaks(S[Omega]) #,'SortStr','descend' + + # Put the location in the correct perspective + lm = peaks + (Omega[:] ==1).argmax() + + # Select the frequency that corresponds to the location + fJ = f[lm] + + # print(len(lm)) + if len(lm) > 0: + # Compute the cost function for deviation from previous fr and maximum power + C_f = abs(fJ-bar_fr)/(2*d) + C_a = 1-S[lm]/max(S[Omega]) + + # Select the minimum cost + C = C_f+C_a + Jmin = C.argmin() + + # Store the frequency with the minimum cost + fJmin = fJ[Jmin] + + return fJmin + +def peakednessCost(signals, ts, fs, Setup = {}, title = "", storeGraph = False, subjet =1): + + vars = {} + # Set parameters / Arrange inputs + [ DT, Ts, Tm, Nfft, K, Omega_r, ksi_p, ksi_a, d, b, a,N_k, plotflag , Setup] = setParamFr(Setup) + + # Start the time stamps at zero + ts1 = ts[0] + ts = ts-ts1 + if type(signals) == type(pd.DataFrame()): + signals = signals.to_numpy() + + # Get the number of signals + if len(signals.shape) == 1: + signals = np.reshape(signals, (signals.shape[0],1)) + if signals.shape[0]= Omega_r[0], f < Omega_r[1]) + vars["f"] = f[f_ind] + + # Time vector for original Welch periodograms + # vars["t_orig"] = np.arange(Ts/2, ts[-1]+DT,DT) - Ts/2+DT #Es posible que sea esto lo que quieren pero no es lo que sale de MATLAB + vars["t_orig"] = np.arange(Ts/2, ts[-1]- Ts/2+DT,DT) #Esto es lo que sale de MATLAB + + # Pre-allocate + vars["Skl"] = np.empty((vars["f"].shape[0], vars["t_orig"].shape[0], vars["L"])) + vars["Skl"][:] = np.nan + + t_for1 = time() + for ii in range(vars["L"]): + t_for_L = time() + + # Select signal + signal = signals[:, ii] + # signal = np.reshape(signal, (signal.shape[0],1)) + # Compute the Welch Periodgrams + for k, ki in zip(vars["t_orig"], range(vars["t_orig"].shape[0])): + # Begin of Ts seconds interval + # Ws_begin = vars["t_orig"][k] - Ts/2 + Ws_begin = k - Ts/2 + + # End of Ts seconds interval + Ws_end = Ws_begin + Ts + [int_Ts_sig, int_Ts_t] = extract_interval(signal, ts, Ws_begin, Ws_end); # Ts seconds interval + S = np.zeros((vars["f"].shape[0])) + if int_Ts_sig.shape[0] < (Tm*100)/2: + vars["Skl"][:, ki, ii] = np.zeros((vars["f"].shape[0])) + continue + # Number of Tm length subintervals + NWm = int(np.floor(2*Ts/Tm)) + I=0 + + for i_Tm in range(NWm): + S_i = [] + + # Begin of Tm seconds interval + Wm_begin = Ws_begin + (i_Tm)*Tm/2 + + # End of Tm seconds interval + Wm_end = min(Wm_begin + Tm, Ws_end) + + # Tm seconds interval + [int_Tm_sig, int_Tm_t] = extract_interval(int_Ts_sig, int_Ts_t, Wm_begin, Wm_end) + + # Estimate the spectrum only for intervals without NaNs + if ~np.isnan((int_Ts_sig.astype(float))).any(): + S_i = abs(fftshift(fft(detrend(int_Tm_sig[:-1]), Nfft)))**2 + # S_i = abs(fftshift(fft(int_Tm_sig[:-1], Nfft)))**2 + S_i = S_i[f_ind] + [ S_i, f_PSD_norm, factor_norm ] = normalizar_PSD(S_i) + if ~np.isnan(S_i).any(): + S = S + (1/NWm)*S_i #TODO hacer una median real, que si uno falla la media se coja con los otros 3 dividido entre 3 + I=I+1 + + if I < 0.5*NWm : + vars["Skl"][:, ki, ii] = np.zeros((vars["f"].shape[0])) + else: + # Define the spectrum when enough subintervals were used + vars["Skl"][:, ki, ii] = S + + + + + ##### Peak-conditioned spectral average: ###### + # Pre-allocate + N = int(np.floor(K/2)) + vars["t_aver"] = vars["t_orig"][N:-N] + if vars["t_aver"].shape[0] == 0: + print("No hay tiempo para promediar") + return np.nan, np.nan, np.nan + vars["Sk"] = np.empty((vars["f"].shape[0], vars["t_aver"].shape[0])) + vars["Sk"][:] = np.nan + vars["bar_fr"] = np.empty(( vars["t_aver"].shape[0])) + vars["bar_fr"][:] = np.nan + vars["hat_fr"] = np.empty((vars["t_aver"].shape[0])) + vars["hat_fr"][:] = np.nan + vars["Naveraged"] = np.zeros((vars["t_aver"].shape[0])) + vars["used"] = np.zeros((vars["t_aver"].shape[0],vars["L"])) + vars["times_used"] = np.zeros((vars["t_orig"].shape[0],vars["L"])) + + # Call the initialization module + k_ini = 0 + plotFlag = False + # print(vars["t_aver"]) + vars = init_module(k_ini,vars,Setup,plotFlag); #bar_fr has been initialized + + for k in np.arange(k_ini, vars["t_aver"].shape[0]): + if k >= 1: + k_prev = k-1 + else: + k_prev = 0 + + # Re-initialization when hat_fr has not been defined for N_k time instants + N_k = 2#3+1#vars["N_k"] + N_prev = np.arange(k,max(k-N_k,-1),-1) + if np.isnan(vars["hat_fr"][N_prev]).all() and k > 2: + vars = init_module(k_prev,vars,Setup,plotFlag) # bar_fr has been re-initialized + + # Peakedness Analysis: + # Indexes of original spectra that take part in the average + O = np.bitwise_and(vars["t_orig"]>=vars["t_aver"][k]-N*DT, vars["t_orig"]<=vars["t_aver"][k]+N*DT) + W = np.arange(O.shape[0]) + O = W[O] + + # Compute the peakedness of the power spectrum (1 or 0) + Xkl = compute_Xkl(vars["Skl"], vars["f"], vars["bar_fr"][k_prev], O, ksi_p, ksi_a, d) + + + if np.sum(Xkl) == 0: # No spectrum was peaked + # Store the previous respiratory frequency + vars["bar_fr"][k] = vars["bar_fr"][k_prev] + + # Compute averaged spectrum just for visualization + if vars["L"]>1: + vars["Sk"][:, k] = np.mean(np.squeeze(np.mean(vars["Skl"][:, O, :],1)),1) + else: + try: + vars["Sk"][:,k] = np.mean(vars["Skl"][:, O, :],1)[:,0] + except: + print("Cogido en el except") + print("Cogido en el except") + print("Cogido en el except") + print("Cogido en el except") + vars["Sk"][:,k] = np.nan + + else: #One or more spectra were peaked enough + # Pre-allocate + averS = np.zeros((vars["f"].shape[0])) + + for i_Tm in range(O.shape[0]): + for ii in range(vars["L"]): + if Xkl[i_Tm,ii] == 1: # If this spectrum is considered peaky + # Sum all peaky spectra + averS = averS[:] + vars["Skl"][:, O[i_Tm], ii] + + # Store the nr of peaky spectra + vars["Naveraged"][k] = vars["Naveraged"][k] + 1 + vars["used"][k,ii] = 1 + + # Compute and store the averaged spectrum + vars["Sk"][:, k] = averS/vars["Naveraged"][k] + vars["times_used"][O,:] = vars["times_used"][O,:] + Xkl + + #Spectral peak selection + fJmin = compute_fJmin( vars["Sk"][:, k], vars["f"], vars["bar_fr"][k_prev], d) + + if ~np.isnan(fJmin).any(): # Local maxima inside Omega has been found + # Update bar_fr + + vars["bar_fr"][k] = b*vars["bar_fr"][k_prev] + (1-b)*fJmin + + # Update hat_fr + if ~np.isnan(vars["hat_fr"][k_prev]).any(): + vars["hat_fr"][k]= a*vars["hat_fr"][k_prev] + (1-a)*fJmin + else: + # Use bar_fr(k-1) that always is defined, instead of hat_fr(k-1) + vars["hat_fr"][k]= a*vars["bar_fr"][k_prev] + (1-a)*fJmin + + else: # No local maxima inside Omega + # Update bar_fr + vars["bar_fr"][k] = vars["bar_fr"][k_prev] + + # Don't Update hat_fr + + t_taver = time() + + # Extra : use bar_fr to update hat_fr when was not defined for small gaps (N_k) + # Beginning of the intervals + N_k = 0 + + int_b = np.argwhere(np.isnan(vars["hat_fr"])) + int_b1 = np.append(0,int_b) + int_b = int_b[np.diff(int_b1)>1] + + # End of the intervals + int_e = np.argwhere(np.isnan(vars["hat_fr"])) + int_e1 = np.append(int_e,np.inf) + int_e = int_e[np.diff(int_e1)>1] + + if np.isnan(vars["hat_fr"][0]) and int_e.shape[0]>1: + + int_e = int_e[1:] + try: + if (int_e[0]-int_b[0])[0] < 0: + int_e = int_e[1:] + except: + print("int vacio") + + int_small = (int_e-int_b)<=(N_k-1) + + int_b = int_b[int_small] + int_e = int_e[int_small] + for i in range(int_small.sum()): + vars["hat_fr"][int_b[i]:int_e[i]+1] = vars["hat_fr"][min(int_e[i]+1,vars["hat_fr"].shape[0])] + vars["bar_fr"][int_b[i]:int_e[i]+1] = vars["bar_fr"][min(int_e[i]+1,vars["hat_fr"].shape[0])] + + + + # # Total times a signal can be used + Ntotal = K*(vars["t_orig"].shape[0] - 2) + np.sum(np.arange(1,K)) + + # Times each signal is used + Nused = np.sum(vars["times_used"], 1) + vars["percentage_used"] = 100*Nused/Ntotal + + + vars["t_aver"] = vars["t_aver"] + ts1 + vars["t_orig"] = vars["t_orig"] + ts1 + t_fin = time() + + if plotflag: + + fig = subplots.make_subplots(rows=2,shared_xaxes=True, subplot_titles=('Peak-condition averaged EDR Spectra in '+title,"EDR/RESP signals"), row_heights=[0.7, 0.3]) + + fig.add_heatmap(x=vars["t_aver"], y=vars["f"], z=vars["Sk"]/np.max(vars["Sk"]),colorscale='jet',colorbar=dict(orientation='h')) + fig.update_layout(coloraxis_showscale=False) + fig.add_trace(go.Line(x=vars["t_aver"], y=vars["hat_fr"],name = 'f\u0302_r(k)'), row = 1, col=1) + fig.add_trace(go.Line(x=vars["t_aver"],y=vars["bar_fr"],name= 'f\u0304_r(k)'), row = 1, col=1) + + fig.add_trace(go.Line(x=vars["t_aver"],y=vars["used"]), row = 1, col=1) + # fig.axis([vars.t_aver(1), vars.t_aver(end), vars.f(1), vars.f(end)]) + for i in range(signals.shape[1]): + fig.add_trace(go.Line(x=ts+ts1,y=signals[:,i],name = 'Signal '+str(i)), row = 2, col=1) + + fig.update_layout(coloraxis_showscale=False) + fig.update_yaxes(title_text="f (Hz)", row=1, col=1) + fig.update_yaxes(title_text="(n.u.)", row=2, col=1) + fig.update_xaxes(title_text="time (s)", row=2, col=1) + if storeGraph: + os.makedirs("Graphs/Peakedness/"+str(subjet), exist_ok=True) + # fig.write_image(os.path.join("Graphs", "Peakedness",str(subjet),title+".png")) + fig.write_html(os.path.join("Graphs", "Peakedness",str(subjet),title+".html")) + # fig.write_image() + else: + fig.show() + + return vars["hat_fr"], vars["Sk"], vars["t_aver"] + # return vars["hat_fr"], vars["Sk"], vars["bar_fr"],vars["t_aver"], vars["f"], vars["used"] \ No newline at end of file diff --git a/src/scripts/EEG_Segmenting.py b/src/scripts/EEG_Segmenting.py new file mode 100644 index 0000000..6ae7406 --- /dev/null +++ b/src/scripts/EEG_Segmenting.py @@ -0,0 +1,193 @@ +from binascii import Error +import numpy as np +import pandas as pd +import sys +import os +import matplotlib.pyplot as plt +import plotly.express as px +import plotly.graph_objects as go +import scipy.signal +from plotly.subplots import make_subplots +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) +import lib.helper_code as helper_code +import lib.EEG_functions as EEG_functions + +for hospital in ['I0002','I0006', 'I0004','I0007','S0001']: + print(f"Procesando hospital: {hospital}") + + if hospital == 'I0002' or hospital == 'I0006' or hospital == "S0001": + datapath = 'data/training_set/Physiological_data/'+hospital + else: + datapath = 'data/supplementary_set/Physiological_data/'+hospital + + channels = pd.read_csv("notebooks/channel_table.csv") + selectEEG = channels[channels['Category'].isin(['eeg'])] + demographics = pd.read_csv(os.path.join('data/training_set', "demographics.csv")) + selectresp = channels[channels['Category'].isin(['resp'])] + selectECG = channels[channels['Category'].isin(['ecg'])] + + # Datos = pd.DataFrame(columns=['File', 'Channel', 'Sampling_Frequency', 'Duration_sec']) + lista_dir = os.listdir(datapath) + results = [] + + for file in lista_dir: + # Cargar el archivo (sustituye por tu ruta real) + edf = helper_code.edfio.read_edf(os.path.join(datapath, file)) + + id = file[9:-10] # Asumiendo que el ID es el nombre del archivo sin la extensión + selEEG = [] + selECG = [] + selResp = [] + labels = [] + data = [] + HayECG = False + for i, sig in enumerate(edf.signals): + for index in selectECG.index: + if sig.label.lower() in selectECG['Channel_Names'][index].lower(): + HayECG = True + print(f"Canal seleccionado: {sig.label}") + selECG.append([i,sig]) + labels.append(sig.label) + data.append(sig.data) # Guardar la señal ECG sin filtrar para su posterior procesamiento + break + + HayResp = False + for i, sig in enumerate(edf.signals): + for index in selectresp.index: + if sig.label.lower() in selectresp['Channel_Names'][index].lower(): + HayResp = True + fs = sig.sampling_frequency + if sig.label == "O2": + print(f"Warning: {sig.label} is detected as respiratory signal but has a sampling frequency higher than 100 Hz. Check the data.") + else: + print(f"Canal seleccionado: {sig.label}") + selResp.append([i,sig]) + labels.append(sig.label) + + data.append(sig.data) # Guardar la señal RESP sin filtrar para su posterior procesamiento + break + + # Listar canales para identificar los de interés (ej: C3-M2, O1-M2) + # print("Canales detectados:") + + HayEEG = False + for i, sig in enumerate(edf.signals): + # print(f"[{i}] {sig.label}") + # print length fs and duration + # print(f"Length: {len(sig.data)}, Sampling Frequency: {sig.sampling_frequency} Hz, Duration: {len(sig.data)/sig.sampling_frequency:.2f} seconds") + for index in selectEEG.index: + if sig.label.lower() in selectEEG['Channel_Names'][index].lower(): + print(f"Canal seleccionado: {sig.label}") + selEEG.append([i,sig]) + # labels.append(sig.label) + HayEEG = True + break + # for i in range(len(edf.signals)): + # print(f"Longitud: {edf.signals[i].data.shape}, Canal: {edf.signals[i].label}, Frecuencia de muestreo: {edf.signals[i].sampling_frequency} Hz, Duración: {len(edf.signals[i].data)/edf.signals[i].sampling_frequency:.2f} segundos") + + if HayEEG and HayECG and HayResp: + + Bipolar = pd.DataFrame() + if all(label in labels for label in ["F3", "F4", "M1", "M2"]): + Bipolar['F3-M2'] = edf.signals[edf.labels.index("F3")].data - edf.signals[edf.labels.index("M2")].data + Bipolar['F4-M1'] = edf.signals[edf.labels.index("F4")].data - edf.signals[edf.labels.index("M1")].data + if all(label in labels for label in ["C3", "C4", "M1", "M2"]): + Bipolar['C3-M2'] = edf.signals[edf.labels.index("C3")].data - edf.signals[edf.labels.index("M2")].data + Bipolar['C4-M1'] = edf.signals[edf.labels.index("C4")].data - edf.signals[edf.labels.index("M1")].data + if all(label in labels for label in ["O2", "O1", "M1", "M2"]): + Bipolar['O2-M2'] = edf.signals[edf.labels.index("O1")].data - edf.signals[edf.labels.index("M2")].data + Bipolar['O1-M1'] = edf.signals[edf.labels.index("O2")].data - edf.signals[edf.labels.index("M1")].data + + # print(f"Archivo {file} tiene ECG, RESP y EEG. Se procesará con canales bipolares.") + if not Bipolar.empty: + for col in Bipolar.columns: + # print(f"Archivo: {file}, Canal: {col}, Frecuencia de muestreo: {sig.sampling_frequency} Hz, Duración: {len(Bipolar[col])/sig.sampling_frequency:.2f} segundos") + fs = edf.signals[edf.labels.index("O2")].sampling_frequency # Asumimos que todos los canales tienen la misma frecuencia de muestreo + time = np.linspace(0, len(Bipolar[col]) / fs, len(Bipolar[col])) + fil = EEG_functions.butter_bandpass_filter(Bipolar[col], lowcut=0.3, highcut=35, fs=fs, order=4) + norm = (fil-np.mean(fil))/np.std(fil) + + data.append(norm) # Restar la media para centrar la señal + labels.append(col) + # columns = Bipolar.columns.tolist() + else: + for i, (idx, sig) in enumerate(selEEG): + # print(f"Archivo: {file}, Canal: {sig.label}, Frecuencia de muestreo: {sig.sampling_frequency} Hz, Duración: {len(sig.data)/sig.sampling_frequency:.2f} segundos") + fs = sig.sampling_frequency + time = np.linspace(0, len(sig.data) / fs, len(sig.data)) + fil = EEG_functions.butter_bandpass_filter(sig.data, lowcut=0.3, highcut=35, fs=fs, order=4) + norm = (fil-np.mean(fil))/np.std(fil) + labels.append(sig.label) + data.append(norm) # Restar la media para centrar la señal + + # columns = [selEEG[i][1].label for i in range(len(selEEG))] + + + # for i in range(len(selResp)): + # columns.append(selResp[i][1].label) + demographicsID = demographics[demographics['BDSPPatientID'] == int(id)] + print(demographicsID) + + columnashoras = [] + for elec in labels: + for h in np.floor(np.arange(0, len(sig.data) / fs / 3600, 1)): + columnashoras.append(elec + f"_h{int(h)}") + epochs5min = pd.DataFrame(columns= columnashoras) + + for i, elec in enumerate(labels): + # Check fs of the current channel + if elec == 'O2_resp': + fs = edf.signals[edf.labels.index('O2')].sampling_frequency + else: + fs = edf.signals[edf.labels.index(labels[i])].sampling_frequency + + if fs != 200: + # print(f"Warning: Sampling frequency for channel {elec} in file {file} is {fs} Hz, expected 200 Hz. Check the data.") + duration = len(data[i]) / fs + time_original = np.linspace(0, duration, len(data[i])) + + num_samples_target = int(duration * 200 ) + time_target = np.linspace(0, duration, num_samples_target) + data[i] = np.interp(time_target, time_original, data[i]) + fs = 200 # Update fs to the target sampling frequency after resampling + + # Plot comparison of original and resampled signals + # lim = 50000 + # factor = len(filtered_data[0]) / num_samples_target + # plt.figure(figsize=(12, 6)) + # plt.plot(time_target[:int(lim/factor)], resampled_data[:int(lim/factor)], label='Resampled Signal') + # plt.plot(time_original[:lim], filtered_data[i][:lim], label='Original Signal') + # plt.title(f'Original vs Resampled Signal - {elec} in {file}') + # plt.show() + + + epoch_length = 300 # Duración de cada época en segundos + # epochs = EEG_functions.create_epochs(df[elec].values, fs, epoch_duration=epoch_length) + epochs = EEG_functions.create_epochs(data[i], fs, epoch_duration=epoch_length) + + # Coger los primeros 5min de cada hora + for h in np.floor(np.arange(0, len(epochs)*epoch_length/3600, 1)): + start_epoch = int(h*3600/epoch_length) + end_epoch = int((h*3600 + 5*60)/epoch_length) + if end_epoch > len(epochs): + end_epoch = len(epochs) + c = elec+'_h'+str(int(h)) + epochs5min.loc[:, c] = epochs[start_epoch:end_epoch][0] + + + # del epochs, fs # Liberar memoria + + # Plotly sublot epochs5min.iloc[:,::8].plot() + # h = 6 + # fig = make_subplots(rows=int(epochs5min.shape[1]/8), cols=1, subplot_titles=epochs5min.columns[h::8]) + # for i in range(h, epochs5min.shape[1], 8): + # print(f"Plotting channel: {epochs5min.columns[i]}") + # fig.add_trace(go.Scatter(x=epochs5min.index, y=epochs5min.iloc[:,i], mode='lines', name=epochs5min.columns[i]), row=int(i/8)+1, col=1) + # fig.update_layout(height=3000, width=1200, title_text=f"Epochs de 5 minutos para el archivo {file}") + # fig.show() + if epochs5min.shape[1] > 0 and epochs5min.shape[0] == 60000: + epochs5min.to_parquet(os.path.join('X:/bsicos01/__comun/Physionet/Data5min', f"{id}.parquet")) + else: + print(f"Error: No channels in epochs5min for file {file}") + # stop program if no channels are processed + raise ValueError(f"No channels in epochs5min for file {file}") \ No newline at end of file diff --git a/src/scripts/EEG_processing.py b/src/scripts/EEG_processing.py new file mode 100644 index 0000000..127a32e --- /dev/null +++ b/src/scripts/EEG_processing.py @@ -0,0 +1,189 @@ +"""EEG_processing.py + +Este módulo contiene funciones para procesar datos EEG de los +hospitales incluidos en el desafío CincChallenge 2026. La principal +función definida es `MetricasHospitlal`, que recorre los archivos EDF +correspondientes a un hospital concreto, extrae las señales EEG, +las filtra, normaliza, crea épocas y calcula potencias de banda y +complejidades. Los resultados se guardan en un CSV resumen por +hospital. + +Características principales: + +- Soporta datos tanto del conjunto de entrenamiento como del + conjunto suplementario. +- Selección automática de canales EEG a partir de la tabla + `notebooks/channel_table.csv`. +- Creación de canales bipolares si están disponibles. +- Filtrado de banda 0.3-35 Hz y normalización de la señal. +- Re-muestreo a 200 Hz si fuese necesario. +- Cálculo de potencias de banda y complejidades usando + funciones auxiliares (`lib/EEG_functions.py`). +- Exportación de resultados en `results_summaryEEG_{hospital}.csv`. + +Uso típico: + +>>> from src.scripts.EEG_processing import MetricasHospitlal +>>> MetricasHospitlal('I0002') + +El módulo depende de `numpy`, `pandas`, `matplotlib`, `plotly` y de +las utilidades definidas en `lib/helper_code` y `lib/EEG_functions`. +""" + +import numpy as np +import pandas as pd +import sys +import os +import matplotlib.pyplot as plt +import plotly.express as px +import plotly.graph_objects as go +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) +import lib.helper_code as helper_code +import lib.EEG_functions as EEG_functions + +def MetricasHospitlal(hospital): + + print(f"Procesando hospital: {hospital}") + + if hospital == 'I0002' or hospital == 'I0006' or hospital == "S0001": + datapath = 'data/training_set/Physiological_data/'+hospital + else: + datapath = 'data/supplementary_set/Physiological_data/'+hospital + + channels = pd.read_csv("notebooks/channel_table.csv") + selectEEG = channels[channels['Category'].isin(['eeg'])] + + demographics = pd.read_csv(os.path.join('C:/BSICoS/CincChallenge2026/CincChallenge_2026/data/training_set', "demographics.csv")) + + # Datos = pd.DataFrame(columns=['File', 'Channel', 'Sampling_Frequency', 'Duration_sec']) + lista_dir = os.listdir(datapath) + results = [] + + for file in lista_dir: + # Cargar el archivo (sustituye por tu ruta real) + edf = helper_code.edfio.read_edf(os.path.join(datapath, file)) + + id = file[9:-10] # Asumiendo que el ID es el nombre del archivo sin la extensión + + selEEG = [] + labels = [] + data = [] + + # Listar canales para identificar los de interés (ej: C3-M2, O1-M2) + HayEEG = False + for i, sig in enumerate(edf.signals): + # print(f"[{i}] {sig.label}") + # print length fs and duration + # print(f"Length: {len(sig.data)}, Sampling Frequency: {sig.sampling_frequency} Hz, Duration: {len(sig.data)/sig.sampling_frequency:.2f} seconds") + for index in selectEEG.index: + if sig.label.lower() in selectEEG['Channel_Names'][index].lower(): + print(f"Canal seleccionado: {sig.label}") + selEEG.append([i,sig]) + labels.append(sig.label) + HayEEG = True + break + # for i in range(len(edf.signals)): + # print(f"Longitud: {edf.signals[i].data.shape}, Canal: {edf.signals[i].label}, Frecuencia de muestreo: {edf.signals[i].sampling_frequency} Hz, Duración: {len(edf.signals[i].data)/edf.signals[i].sampling_frequency:.2f} segundos") + + if HayEEG: + + Bipolar = pd.DataFrame() + if all(label in labels for label in ["F3", "F4", "M1", "M2"]): + Bipolar['F3-M2'] = edf.signals[edf.labels.index("F3")].data - edf.signals[edf.labels.index("M2")].data + Bipolar['F4-M1'] = edf.signals[edf.labels.index("F4")].data - edf.signals[edf.labels.index("M1")].data + labels2 = ['F3-M2', 'F4-M1'] + if all(label in labels for label in ["C3", "C4", "M1", "M2"]): + Bipolar['C3-M2'] = edf.signals[edf.labels.index("C3")].data - edf.signals[edf.labels.index("M2")].data + Bipolar['C4-M1'] = edf.signals[edf.labels.index("C4")].data - edf.signals[edf.labels.index("M1")].data + labels2.append('C3-M2') + labels2.append('C4-M1') + if all(label in labels for label in ["O2", "O1", "M1", "M2"]): + Bipolar['O2-M2'] = edf.signals[edf.labels.index("O1")].data - edf.signals[edf.labels.index("M2")].data + Bipolar['O1-M1'] = edf.signals[edf.labels.index("O2")].data - edf.signals[edf.labels.index("M1")].data + labels2.append('O1-M1') + labels2.append('O2-M2') + # print(f"Archivo {file} tiene ECG, RESP y EEG. Se procesará con canales bipolares.") + + if not Bipolar.empty: + labels = [] + for col in Bipolar.columns: + # print(f"Archivo: {file}, Canal: {col}, Frecuencia de muestreo: {sig.sampling_frequency} Hz, Duración: {len(Bipolar[col])/sig.sampling_frequency:.2f} segundos") + fs = edf.signals[edf.labels.index("M2")].sampling_frequency # Asumimos que todos los canales tienen la misma frecuencia de muestreo + time = np.linspace(0, len(Bipolar[col]) / fs, len(Bipolar[col])) + fil = EEG_functions.butter_bandpass_filter(Bipolar[col], lowcut=0.3, highcut=35, fs=fs, order=4) + norm = (fil-np.mean(fil))/np.std(fil) + + data.append(norm) # Restar la media para centrar la señal + labels.append(col) + # columns = Bipolar.columns.tolist() + else: + for i, (idx, sig) in enumerate(selEEG): + # print(f"Archivo: {file}, Canal: {sig.label}, Frecuencia de muestreo: {sig.sampling_frequency} Hz, Duración: {len(sig.data)/sig.sampling_frequency:.2f} segundos") + fs = sig.sampling_frequency + time = np.linspace(0, len(sig.data) / fs, len(sig.data)) + fil = EEG_functions.butter_bandpass_filter(sig.data, lowcut=0.3, highcut=35, fs=fs, order=4) + norm = (fil-np.mean(fil))/np.std(fil) + labels.append(sig.label) + data.append(norm) # Restar la media para centrar la señal + + # columns = [selEEG[i][1].label for i in range(len(selEEG))] + + + demographics = demographics[demographics['BDSPPatientID'] == int(id)] + print(demographics) + + for i, elec in enumerate(labels): + epoch_length = 30 # Duración de cada época en segundos + if Bipolar.empty: + fs = edf.signals[edf.labels.index(labels[i])].sampling_frequency + else: + fs = edf.signals[edf.labels.index('M1')].sampling_frequency + + if fs != 200: + # print(f"Warning: Sampling frequency for channel {elec} in file {file} is {fs} Hz, expected 200 Hz. Check the data.") + duration = len(data[i]) / fs + time_original = np.linspace(0, duration, len(data[i])) + + num_samples_target = int(duration * 200 ) + time_target = np.linspace(0, duration, num_samples_target) + data[i] = np.interp(time_target, time_original, data[i]) + fs = 200 # Update fs to the target sampling frequency after resampling + + epochs = EEG_functions.create_epochs(data[i], fs, epoch_duration=epoch_length) + + band_powers, complexities = EEG_functions.extract_band_powers(epochs, fs, win_len=15) + print(f"Band powers for file {file}:") + + band_powers = band_powers.iloc[60:] # Eliminar las primeras 60 épocas (30 min) para evitar el tiempo despierto al inicio de la grabación + print(band_powers.head()) + + + # # Convertir de formato "ancho" a "largo" para Plotly + # df_melted = band_powers.melt(var_name='Banda', value_name='Potencia') + # # Creamos el boxplot + # fig = px.box(df_melted, x='Banda', y='Potencia', + # color='Banda', + # points="outliers", # Para ver si hay épocas muy extrañas + # title=f"{id} - {elec} - {demographics.Cognitive_Impairment.values[0]}", + # log_y=True) # Usamos escala logarítmica porque Delta suele ser mucho más potente que Beta + + # fig.update_layout(template="plotly_white", showlegend=False) + # # fig.write_html(f"graphs/BandasPersona/{id}_{elec}_{demographics.Cognitive_Impairment.values[0]}.html") # Guardar como HTML para visualización interactiva + # fig.show() + + # Ejecución + patient_summar = EEG_functions.get_patient_profile(band_powers) + # print(f"Resumen del perfil del paciente {id} - {elec}:") + # print(patient_summar) + d = complexities.iloc[:].std().to_dict() + results.append({ + 'File': file, + 'Channel': elec, + 'Patient_ID': id, + **d, + **patient_summar + }) + df_results = pd.DataFrame(results) + print(df_results.head()) + return df_results + # df_results.to_csv(f"results_summaryEEG_{hospital}.csv", index=False) \ No newline at end of file diff --git a/src/scripts/Resp_processing.py b/src/scripts/Resp_processing.py new file mode 100644 index 0000000..dfe6ec2 --- /dev/null +++ b/src/scripts/Resp_processing.py @@ -0,0 +1,114 @@ +import numpy as np +import pandas as pd +import os +import matplotlib.pyplot as plt +import plotly.express as px +import sys + +import plotly.graph_objects as go +from plotly.subplots import make_subplots + +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) +import lib.helper_code as helper_code +import lib.EEG_functions as EEG_functions +import lib.Resp_features as Resp_features + +for hospital in ['I0006',"S0001",'I0004','I0007']:#'I0002', + print(f"Procesando hospital: {hospital}") + + if hospital == 'I0002' or hospital == 'I0006' or hospital == "S0001": + datapath = 'data/training_set/Physiological_data/'+hospital + else: + datapath = 'data/supplementary_set/Physiological_data/'+hospital + + channels = pd.read_csv("notebooks/channel_table.csv") + selectResp = channels[channels['Category'].isin(['resp'])] + + demographics = pd.read_csv(os.path.join('C:/BSICoS/CincChallenge2026/CincChallenge_2026/data/training_set', "demographics.csv")) + + # Datos = pd.DataFrame(columns=['File', 'Channel', 'Sampling_Frequency', 'Duration_sec']) + lista_dir = os.listdir(datapath) + results = [] + + for file in lista_dir: + # Cargar el archivo (sustituye por tu ruta real) + edf = helper_code.edfio.read_edf(os.path.join(datapath, file)) + + id = file[9:-10] # Asumiendo que el ID es el nombre del archivo sin la extensión + + selResp = [] + labels = [] + data = [] + + HayResp = False + for i, sig in enumerate(edf.signals): + for index in selectResp.index: + if sig.label.lower() in selectResp['Channel_Names'][index].lower(): + print(f"Canal seleccionado: {sig.label}") + selResp.append([i,sig]) + labels.append(sig.label) + HayResp = True + # plot en plotly la señal + go.Figure(data=go.Scattergl(x=np.arange(len(sig.data))/sig.sampling_frequency, y=sig.data, mode='lines', name=sig.label)).update_layout(title=f"Señal de {sig.label} - Archivo: {file}", xaxis_title="Tiempo (s)", yaxis_title="Amplitud").show() + # px.line(x=np.arange(len(sig.data))/sig.sampling_frequency, y=sig.data, title=f"Señal de {sig.label} - Archivo: {file}").show() + break + + if HayResp: + for i, (idx, sig) in enumerate(selResp): + print(f"Archivo: {file}, Canal: {sig.label}, Frecuencia de muestreo: {sig.sampling_frequency} Hz, Duración: {len(sig.data)/sig.sampling_frequency:.2f} segundos") + fs = sig.sampling_frequency + + if fs != 25: + duration = len(sig.data) / fs + time_original = np.linspace(0, duration, len(sig.data)) + num_samples_target = int(duration * 25 ) + time_target = np.linspace(0, duration, num_samples_target) + data = np.interp(time_target, time_original, sig.data) + fs = 25 # Update fs to the target sampling frequency after resampling + else: + data = sig.data + time_new = np.linspace(0, len(sig.data) / fs, len(sig.data)) + + # Check nan in sig.data + if np.isnan(sig.data).any(): + print(f"Warning: NaN values found in signal data for {sig.label}. Filling NaNs with zeros.") + data = np.nan_to_num(data) + + # if sig.label not in ["SpO2", "SaO2", "OSAT", "O2SAT", "O2 SAT", "O2-SAT", "O2-SATURATION"]: + # fil = EEG_functions.butter_bandpass_filter(data, lowcut=0.01, highcut=4, fs=fs, order=4) + # # norm = (fil-np.mean(fil))/np.std(fil) + # data.append(fil) # Restar la media para centrar la señal + + if sig.label.lower() in selectResp['Channel_Names'][28].lower() or sig.label.lower() in selectResp['Channel_Names'][29].lower(): + # EFFORT RESPIRATORY + elif sig.label.lower() in selectResp['Channel_Names'][30].lower() or sig.label.lower() in selectResp['Channel_Names'][31].lower(): + # RESPIRATORY Flujo + fil = EEG_functions.butter_bandpass_filter(data, lowcut=0.01, highcut=4, fs=fs, order=4) + Resp_features.peakedness_application(fil, stage=sig.label, plotflag = True, subjet =1) + elif sig.label.lower() in selectResp['Channel_Names'][32].lower() or sig.label.lower() in selectResp['Channel_Names'][33].lower(): + # CEPAP + elif sig.label.lower() in selectResp['Channel_Names'][34].lower(): + #O2 SATURATION + + + + # time_dt = pd.to_datetime(time_new, unit='s') + # # Plot raw and filtered signals + # fig = make_subplots(specs=[[{"secondary_y": True}]]) + # fig.add_trace(go.Scattergl(x=time_dt[::10], y=data[::10], name=sig.label, mode='lines'),secondary_y=False,row=1, col=1) + # fig.add_trace(go.Scattergl(x=time_dt[::10], y=fil[::10], name=f"Normalized {sig.label}", mode='lines'), secondary_y=True,row=1, col=1) + # fig.update_yaxes(title_text="Amplitud Original (uV)", secondary_y=False) + # fig.update_yaxes(title_text="Valor Normalizado (Z-score)", secondary_y=True) + # # update x axis to make time format + # fig.update_xaxes( + # tickformat="%H:%M:%S", # Formato de hora:minuto:segundo + # row=1, col=1 + # ) + # fig.show() + + # Plot spectrogram of raw and filtered signals + # fig = make_subplots(specs=[[{"secondary_y": True}]]) + # fig.add_trace(go.Scattergl(x=time_dt[::10], y=data[::10], name=sig.label, mode='lines'),secondary_y=False,row=1, col=1) + # fig.add_trace(go.Scattergl(x=time_dt[::10], y=fil[::10], name=f"Normalized {sig.label}", mode='lines'), secondary_y=True,row=1, col=1) + # fig.update_yaxes(title_text="Amplitud Original (uV)", secondary_y=False) + diff --git a/src/scripts/ResultsAnalysis.py b/src/scripts/ResultsAnalysis.py new file mode 100644 index 0000000..c2fef8b --- /dev/null +++ b/src/scripts/ResultsAnalysis.py @@ -0,0 +1,67 @@ +import numpy as np +import pandas as pd +import sys +import os +import plotly.express as px + +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) +import lib.EEG_functions as EEG_functions +import seaborn as sns +import matplotlib.pyplot as plt +import plotly.express as px + +hospital = ["I0006","I0002","I0004","I0007", "S0001"] +results = pd.DataFrame() +for h in hospital: + results = pd.concat([results, pd.read_csv(f"results_summaryEEG_{h}.csv")], ignore_index=True) + +demographics = pd.read_csv(os.path.join('C:/BSICoS/CincChallenge2026/CincChallenge_2026/data/training_set', "demographics.csv")) +demographics = pd.concat([demographics, pd.read_csv(os.path.join('C:/BSICoS/CincChallenge2026/CincChallenge_2026/data/supplementary_set', "demographics.csv"))], ignore_index=True) + +for index, row in results.iterrows(): + patient_id = row['Patient_ID'] + hospital_id = row['File'][4:9] # Asumiendo que los primeros 5 caracteres del nombre del archivo indican el hospital + demographics_row = demographics[(demographics['BDSPPatientID'] == patient_id) & (demographics['SiteID'] == hospital_id)] + if not demographics_row.empty: + cognitive_impairment = demographics_row['Cognitive_Impairment'].values[0] + time_to_event = demographics_row['Time_to_Event'].values[0] + results.at[index, 'Hospital'] = hospital_id + results.at[index, 'CognitiveImpairment'] = cognitive_impairment + results.at[index, 'Time_to_Event'] = time_to_event + else: + results.at[index, 'Hospital'] = hospital_id + results.at[index, 'CognitiveImpairment'] = np.nan # O cualquier valor que indique que no se encontró información + results.at[index, 'Time_to_Event'] = np.nan + +df = pd.DataFrame(results) +# Agrupar por electrodo +for elec in results['Channel'].unique(): + subset = results[results['Channel'] == elec] + + print(subset.Hospital.unique()) + # Hacer un boxplot de cada característica que separe entre pacientes con congnitive impairment y sin él + for col in subset.columns[3:-2]: + print(col) + + fig = px.box(subset, + x='CognitiveImpairment', + y=col, + color='CognitiveImpairment', + notched=True, + points="all", + hover_data=['Patient_ID', 'Channel'], + title=f"{elec} - Comparativa de {col} según Estado Cognitivo") + + fig.update_layout(template="plotly_white") + fig.write_html(f"graphs/ComparativaCognitiveImpairment/2segundos/PorHospital/html/{elec}_{col}.html") # Guardar como HTML para visualización interactiva + # fig.delete_traces([0]) # Eliminar la leyenda para que no se repita en cada gráfico + + # Generar el boxplot con Seaborn (Extremadamente rápido) + plt.figure(figsize=(10, 6)) + sns.boxplot(data=df, x='CognitiveImpairment', y=col, hue='CognitiveImpairment', notch=True) + sns.stripplot(data=df, x='CognitiveImpairment', y=col, color="black", alpha=0.3, size=3) # Equivalente a points="all" + + plt.title(f"Comparativa {elec} - {col}") + plt.savefig(f"graphs/ComparativaCognitiveImpairment/2segundos/PorHospital/{hospital_id}/png/{elec}_{col}.png", dpi=100) + # \graphs\ComparativaCognitiveImpairment\2segundos\PorHospital\I0006\png + plt.close() # ¡Importante! Para no saturar la memoria RAM \ No newline at end of file From 02a8229b006227596f5530bc74cc34b2b05dcfbb Mon Sep 17 00:00:00 2001 From: dcajal Date: Mon, 2 Mar 2026 18:02:19 +0100 Subject: [PATCH 05/93] Visual noise corrected --- src/scripts/{ResultsAnalysis.py => results_analysis.py} | 0 src/scripts/{EEG_Segmenting.py => segmentation.py} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename src/scripts/{ResultsAnalysis.py => results_analysis.py} (100%) rename src/scripts/{EEG_Segmenting.py => segmentation.py} (100%) diff --git a/src/scripts/ResultsAnalysis.py b/src/scripts/results_analysis.py similarity index 100% rename from src/scripts/ResultsAnalysis.py rename to src/scripts/results_analysis.py diff --git a/src/scripts/EEG_Segmenting.py b/src/scripts/segmentation.py similarity index 100% rename from src/scripts/EEG_Segmenting.py rename to src/scripts/segmentation.py From 0484d8ac91f30aa1045c5d80eba4e7c2a423472b Mon Sep 17 00:00:00 2001 From: dcajal Date: Tue, 3 Mar 2026 13:08:14 +0100 Subject: [PATCH 06/93] Refactor --- src/{scripts/EEG_processing.py => eeg_processing.py} | 0 src/{scripts/Resp_processing.py => resp_processing.py} | 0 src/{scripts => }/results_analysis.py | 0 src/{scripts => }/segmentation.py | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename src/{scripts/EEG_processing.py => eeg_processing.py} (100%) rename src/{scripts/Resp_processing.py => resp_processing.py} (100%) rename src/{scripts => }/results_analysis.py (100%) rename src/{scripts => }/segmentation.py (100%) diff --git a/src/scripts/EEG_processing.py b/src/eeg_processing.py similarity index 100% rename from src/scripts/EEG_processing.py rename to src/eeg_processing.py diff --git a/src/scripts/Resp_processing.py b/src/resp_processing.py similarity index 100% rename from src/scripts/Resp_processing.py rename to src/resp_processing.py diff --git a/src/scripts/results_analysis.py b/src/results_analysis.py similarity index 100% rename from src/scripts/results_analysis.py rename to src/results_analysis.py diff --git a/src/scripts/segmentation.py b/src/segmentation.py similarity index 100% rename from src/scripts/segmentation.py rename to src/segmentation.py From 9a73d0d2d40d71b14377c8ff730f9c2d8da7f527 Mon Sep 17 00:00:00 2001 From: dcajal Date: Tue, 3 Mar 2026 13:08:34 +0100 Subject: [PATCH 07/93] Add scripts for creating and managing smoke training datasets --- scripts/create_smoke.ps1 | 41 +++++++++++++ scripts/run.ps1 | 123 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 164 insertions(+) create mode 100644 scripts/create_smoke.ps1 create mode 100644 scripts/run.ps1 diff --git a/scripts/create_smoke.ps1 b/scripts/create_smoke.ps1 new file mode 100644 index 0000000..bd39bcd --- /dev/null +++ b/scripts/create_smoke.ps1 @@ -0,0 +1,41 @@ +# ============================================ +# Create smoke training dataset +# ============================================ + +# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> +# IMPORTANT: +# Each team member must modify this path to +# match their local dataset location. +# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> +$FULL_DATA_PATH = "data/training_set" # <-- CHANGE THIS IF NEEDED + +$SMOKE_PATH = "data/training_smoke" +$N_RECORDS = 5 + +Write-Host "Creating smoke dataset..." +Write-Host "Source: $FULL_DATA_PATH" +Write-Host "Destination: $SMOKE_PATH" + +Remove-Item -Recurse -Force $SMOKE_PATH -ErrorAction SilentlyContinue +New-Item -ItemType Directory -Force -Path $SMOKE_PATH | Out-Null + +# Copy demographics +Copy-Item "$FULL_DATA_PATH/demographics.csv" "$SMOKE_PATH/demographics.csv" + +# Select first N EDF files +$edfs = Get-ChildItem "$FULL_DATA_PATH/physiological_data" -Recurse -Filter *.edf | + Sort-Object FullName | + Select-Object -First $N_RECORDS + +foreach ($f in $edfs) { + $rel = $f.FullName.Substring((Resolve-Path $FULL_DATA_PATH).Path.Length).TrimStart('\') + $target = Join-Path $SMOKE_PATH $rel + New-Item -ItemType Directory -Force -Path (Split-Path $target) | Out-Null + Copy-Item $f.FullName $target +} + +# Copy full annotation folders (simpler and robust) +Copy-Item "$FULL_DATA_PATH/algorithmic_annotations" "$SMOKE_PATH/algorithmic_annotations" -Recurse -ErrorAction SilentlyContinue +Copy-Item "$FULL_DATA_PATH/human_annotations" "$SMOKE_PATH/human_annotations" -Recurse -ErrorAction SilentlyContinue + +Write-Host "Smoke dataset created successfully." \ No newline at end of file diff --git a/scripts/run.ps1 b/scripts/run.ps1 new file mode 100644 index 0000000..de6122d --- /dev/null +++ b/scripts/run.ps1 @@ -0,0 +1,123 @@ +param( + [Parameter(Mandatory=$true)] + [ValidateSet("build","smoke","train","train-smoke","run","run-smoke","clean")] + [string]$Command +) + +# ============================================ +# CONFIGURATION +# ============================================ + +# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> +# IMPORTANTE: +# Si tu dataset no está en data/training_set, +# modifica esta ruta. +# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> +$FULL_DATA="data/training_set" + +$SMOKE_DATA="data/training_smoke" + +$IMAGE_NAME="cinc2026" +$MODEL_FULL="model" +$MODEL_SMOKE="model_smoke" +$OUT_FULL="outputs" +$OUT_SMOKE="outputs_smoke" + +# ============================================ +# FUNCTIONS +# ============================================ + +function Build-Image { + docker build -t $IMAGE_NAME . +} + +function Create-Smoke { + Write-Host "Creating smoke dataset..." + powershell -ExecutionPolicy Bypass -File scripts/create_smoke.ps1 +} + +function Train-Full { + New-Item -ItemType Directory -Force -Path $MODEL_FULL | Out-Null + + docker run --rm ` + -v "${FULL_DATA}:/challenge/training_data:ro" ` + -v "${PWD}/${MODEL_FULL}:/challenge/model" ` + $IMAGE_NAME ` + python train_model.py -d training_data -m model -v +} + +function Train-Smoke { + New-Item -ItemType Directory -Force -Path $MODEL_SMOKE | Out-Null + + docker run --rm ` + -v "${SMOKE_DATA}:/challenge/training_data:ro" ` + -v "${PWD}/${MODEL_SMOKE}:/challenge/model" ` + $IMAGE_NAME ` + python train_model.py -d training_data -m model -v +} + +function Run-Full { + New-Item -ItemType Directory -Force -Path $OUT_FULL | Out-Null + + docker run --rm ` + -v "${FULL_DATA}:/challenge/holdout_data:ro" ` + -v "${PWD}/${MODEL_FULL}:/challenge/model:ro" ` + -v "${PWD}/${OUT_FULL}:/challenge/holdout_outputs" ` + $IMAGE_NAME ` + python run_model.py -d holdout_data -m model -o holdout_outputs -v +} + +function Run-Smoke { + New-Item -ItemType Directory -Force -Path $OUT_SMOKE | Out-Null + + docker run --rm ` + -v "${SMOKE_DATA}:/challenge/holdout_data:ro" ` + -v "${PWD}/${MODEL_SMOKE}:/challenge/model:ro" ` + -v "${PWD}/${OUT_SMOKE}:/challenge/holdout_outputs" ` + $IMAGE_NAME ` + python run_model.py -d holdout_data -m model -o holdout_outputs -v +} + +function Clean-All { + Remove-Item -Recurse -Force $MODEL_FULL -ErrorAction SilentlyContinue + Remove-Item -Recurse -Force $MODEL_SMOKE -ErrorAction SilentlyContinue + Remove-Item -Recurse -Force $OUT_FULL -ErrorAction SilentlyContinue + Remove-Item -Recurse -Force $OUT_SMOKE -ErrorAction SilentlyContinue + Write-Host "Cleaned model and output folders." +} + +# ============================================ +# COMMAND SWITCH +# ============================================ + +switch ($Command) { + + "build" { + Build-Image + } + + "smoke" { + Create-Smoke + } + + "train" { + Train-Full + } + + "train-smoke" { + Train-Smoke + } + + "run" { + Run-Full + } + + "run-smoke" { + Run-Smoke + } + + "clean" { + Clean-All + } + +} \ No newline at end of file From de17158bf7ff541ede1301d5af81b5bfbeecbe5e Mon Sep 17 00:00:00 2001 From: dcajal Date: Tue, 3 Mar 2026 13:08:45 +0100 Subject: [PATCH 08/93] Add documentation for project overview, Docker usage, smoke dataset, and unified run script --- docs/01_overview.md | 35 ++++++++++++ docs/02_docker.md | 53 ++++++++++++++++++ docs/03_smoke_dataset.md | 51 +++++++++++++++++ docs/04_run_script.md | 118 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 257 insertions(+) create mode 100644 docs/01_overview.md create mode 100644 docs/02_docker.md create mode 100644 docs/03_smoke_dataset.md create mode 100644 docs/04_run_script.md diff --git a/docs/01_overview.md b/docs/01_overview.md new file mode 100644 index 0000000..9805383 --- /dev/null +++ b/docs/01_overview.md @@ -0,0 +1,35 @@ +# CINC 2026 – Visión General del Proyecto + +Estamos participando en el Challenge 2026 de Computing in Cardiology. + +El objetivo es predecir deterioro cognitivo a partir de datos de polisomnografía (PSG). + +## Cómo nos evaluarán + +La organización: + +1. Construirá nuestra imagen Docker. +2. Ejecutará `train_model.py`. +3. Ejecutará `run_model.py`. +4. Evaluará las predicciones generadas. + +Por tanto, la reproducibilidad mediante Docker es obligatoria. + +Nuestro objetivo es garantizar que: +- El código se ejecuta sin intervención manual. +- El modelo se entrena correctamente. +- Las predicciones se generan en el formato requerido. + +## Qué se puede modificar y qué no + +❌ No modificar + +- `train_model.py` +- `run_model.py` +- `helper_code.py` +- `evaluate_model.py` + +✅ Modificar/Añadir + +- `team_code.py` <-- Toda la lógica científica y de modelado debe implementarse ahí. +- Helpers, scripts, métodos: añadir a voluntad en `src/` \ No newline at end of file diff --git a/docs/02_docker.md b/docs/02_docker.md new file mode 100644 index 0000000..eca7735 --- /dev/null +++ b/docs/02_docker.md @@ -0,0 +1,53 @@ +# Uso de Docker + +## Requisitos + +- Docker Desktop instalado (modo Linux containers) +- Dataset descargado desde Kaggle +- Se asume que el dataset está en: data/training_set/ + +Cada miembro del equipo puede tener el dataset en una ubicación diferente, pero en este repositorio asumimos que está dentro de `data/`. + +--- + +## Construir la imagen + +Desde la raíz del repositorio: + +```powershell +docker build -t cinc2026 . +``` + +## Entenar con el dataset completo + +```powershell +$DATA="data/training_set" +$MODEL="$PWD/model" + +docker run --rm ` + -v "${DATA}:/challenge/training_data:ro" ` + -v "${MODEL}:/challenge/model" ` + cinc2026 ` + python train_model.py -d training_data -m model -v +``` + +## Generar predicciones + +```powershell +$OUT="$PWD/outputs" + +docker run --rm ` + -v "${DATA}:/challenge/holdout_data:ro" ` + -v "${MODEL}:/challenge/model:ro" ` + -v "${OUT}:/challenge/holdout_outputs" ` + cinc2026 ` + python run_model.py -d holdout_data -m model -o holdout_outputs -v +``` + +## Resultado esperado + +En la carpeta `outputs/` se generará un `demographics.csv` con: + +- Columnas originales +- Cognitive_Impairment +- Cognitive_Impairment_Probability \ No newline at end of file diff --git a/docs/03_smoke_dataset.md b/docs/03_smoke_dataset.md new file mode 100644 index 0000000..a334088 --- /dev/null +++ b/docs/03_smoke_dataset.md @@ -0,0 +1,51 @@ +# Dataset Smoke (Desarrollo Rápido) + +Entrenar con el dataset completo tarda aproximadamente 30–40 minutos con el modelo de ejemplo. + +Para desarrollo utilizamos un dataset reducido (5 sujetos). + +--- + +## Crear dataset smoke + +Ejecutar: + +```powershell +powershell -ExecutionPolicy Bypass -File scripts/create_smoke.ps1 +``` + +Esto generará: `data/training_smoke/` + +## Entrenar con el smoke dataset + +```powershell +$DATA="data/training_smoke" +$MODEL="$PWD/model_smoke" + +docker run --rm ` + -v "${DATA}:/challenge/training_data:ro" ` + -v "${MODEL}:/challenge/model" ` + cinc2026 ` + python train_model.py -d training_data -m model -v +``` + +## Generar predicciones con smoke dataset + +```powershell +$OUT="$PWD/outputs_smoke" + +docker run --rm ` + -v "${DATA}:/challenge/holdout_data:ro" ` + -v "${MODEL}:/challenge/model:ro" ` + -v "${OUT}:/challenge/holdout_outputs" ` + cinc2026 ` + python run_model.py -d holdout_data -m model -o holdout_outputs -v +``` + +## ¿Cuándo usar smoke? + +- Desarrollo de nuevas features +- Comprobación rápida de que el código no rompe +- Validación de cambios en team_code.py + +Nunca usar smoke para evaluar rendimiento final. \ No newline at end of file diff --git a/docs/04_run_script.md b/docs/04_run_script.md new file mode 100644 index 0000000..694c4f3 --- /dev/null +++ b/docs/04_run_script.md @@ -0,0 +1,118 @@ +# Script unificado de ejecución (`run.ps1`) + +Para simplificar el trabajo del equipo hemos creado un único script que encapsula todos los comandos necesarios para: + +- Construir la imagen Docker +- Crear el dataset smoke +- Entrenar (completo o smoke) +- Generar predicciones +- Limpiar artefactos + +--- + +## Requisitos + +- Docker Desktop instalado +- Dataset descargado en: `data/training_set/` + + +⚠️ Si el dataset está en otra ubicación, modificar la variable `$FULL_DATA` dentro de `run.ps1`. + +--- + +# Comandos disponibles + +Desde la raíz del repositorio: + +## 1️⃣ Construir la imagen Docker + +```powershell +.\run.ps1 build +``` + +Solo es necesario hacerlo: + +- La primera vez +- Cuando cambien dependencias o el Dockerfile + +## 2️⃣ Crear dataset smoke (5 sujetos) + +```powershell +.\run.ps1 smoke +``` + +Genera: `data/training_smoke/` + +Este dataset se usa para desarrollo rápido. + +## 3️⃣ Entrenar modelo + +### Smoke (rápido) + +```powershell +.\run.ps1 train-smoke +``` + + +### Completo + +```powershell +.\run.ps1 train +``` + +El modelo se guarda en: + +- model/ (full) +- model_smoke/ (smoke) + +## 4️⃣ Generar predicciones + +### Smoke + +```powershell +.\run.ps1 run-smoke +``` + +### Completo + +```powershell +.\run.ps1 run +``` + +Los resultados se generan en: + +- outputs/ +- outputs_smoke/ + +El archivo clave es: `demographics.csv` que contiene las predicciones añadidas. + +## 5️⃣ Limpiar artefactos + +```powershell +.\run.ps1 clean +``` + +Elimina: + +- model/ +- model_smoke/ +- outputs/ +- outputs_smoke/ + +No elimina datasets. + +# Flujo recomendado para desarrollo + +```powershell +.\run.ps1 build +.\run.ps1 smoke +.\run.ps1 train-smoke +.\run.ps1 run-smoke +``` + +Solo cuando el modelo esté estable: + +```powershell +.\run.ps1 train +.\run.ps1 run +``` \ No newline at end of file From fb6216cca9fef188cb66b7617d88631370e608be Mon Sep 17 00:00:00 2001 From: dcajal Date: Tue, 3 Mar 2026 13:25:43 +0100 Subject: [PATCH 09/93] Add .dockerignore file to exclude datasets, artifacts, and OS/IDE files --- .dockerignore | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..d8e589d --- /dev/null +++ b/.dockerignore @@ -0,0 +1,23 @@ +# Datasets +data/ +**/training_set/ +**/supplementary_set/ +**/*.edf + +# Artifacts +model/ +model_smoke/ +model_full_smoke/ +outputs/ +outputs_smoke/ +__pycache__/ +*.pyc +*.pkl +*.sav +*.joblib + +# OS / IDE +.DS_Store +Thumbs.db +.vscode/ +.idea/ \ No newline at end of file From 689fd97c9f55421d76cbd425df9d6b0866d817a1 Mon Sep 17 00:00:00 2001 From: dcajal Date: Tue, 3 Mar 2026 13:29:29 +0100 Subject: [PATCH 10/93] Update .gitignore to include datasets, model artifacts, and output directories --- .gitignore | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index f298c64..7cc07cb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,29 @@ +# Dataset +data/ + +# Model artifacts +model/ +model_smoke/ +*.pkl +*.sav +*.joblib + +# Outputs +outputs/ +outputs_smoke/ + +# Python +__pycache__/ +*.pyc + +# OS +.DS_Store +Thumbs.db + +# IDE +.vscode/ +.idea/ + # Byte-compiled / optimized / DLL files __pycache__/ *.py[codz] @@ -49,7 +75,6 @@ coverage.xml .hypothesis/ .pytest_cache/ cover/ -data/ graphs/ graphs From d3a10dbcb9786b8de71e2dd9553a4cb4b89831b6 Mon Sep 17 00:00:00 2001 From: dcajal Date: Tue, 3 Mar 2026 13:47:18 +0100 Subject: [PATCH 11/93] Fix paths for smoke dataset training in documentation --- docs/03_smoke_dataset.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/03_smoke_dataset.md b/docs/03_smoke_dataset.md index a334088..f91249d 100644 --- a/docs/03_smoke_dataset.md +++ b/docs/03_smoke_dataset.md @@ -19,8 +19,8 @@ Esto generará: `data/training_smoke/` ## Entrenar con el smoke dataset ```powershell -$DATA="data/training_smoke" -$MODEL="$PWD/model_smoke" +$DATA="$PWD\data\training_smoke" +$MODEL="$PWD\model_smoke" docker run --rm ` -v "${DATA}:/challenge/training_data:ro" ` From 2f63033b3cd0145a60b8a5873f77f555871f208d Mon Sep 17 00:00:00 2001 From: dcajal Date: Tue, 3 Mar 2026 13:49:42 +0100 Subject: [PATCH 12/93] Add progress bar functionality and print filtering for model execution --- team_code.py | 75 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/team_code.py b/team_code.py index a055222..e8152a8 100644 --- a/team_code.py +++ b/team_code.py @@ -12,6 +12,9 @@ import joblib import numpy as np import os +import atexit +import builtins +import re from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor import sys from tqdm import tqdm @@ -28,6 +31,46 @@ # Build the absolute path to the CSV file relative to the script location DEFAULT_CSV_PATH = os.path.join(SCRIPT_DIR, 'channel_table.csv') +# Progress bar state for run_model (initialized lazily) +RUN_MODEL_PBAR = None +RUN_MODEL_PBAR_TOTAL = None +ORIGINAL_PRINT = builtins.print +PRINT_FILTER_ACTIVE = False +RUN_PROGRESS_LINE_RE = re.compile(r'^-\s+\d+/\d+:\s') + + +def _close_run_model_pbar(): + global RUN_MODEL_PBAR + if RUN_MODEL_PBAR is not None: + RUN_MODEL_PBAR.close() + RUN_MODEL_PBAR = None + + +def _install_run_print_filter(): + global PRINT_FILTER_ACTIVE + if PRINT_FILTER_ACTIVE: + return + + def _filtered_print(*args, **kwargs): + message = kwargs.get('sep', ' ').join(str(a) for a in args) if args else '' + if RUN_PROGRESS_LINE_RE.match(message): + return + return ORIGINAL_PRINT(*args, **kwargs) + + builtins.print = _filtered_print + PRINT_FILTER_ACTIVE = True + + +def _restore_print(): + global PRINT_FILTER_ACTIVE + if PRINT_FILTER_ACTIVE: + builtins.print = ORIGINAL_PRINT + PRINT_FILTER_ACTIVE = False + + +atexit.register(_close_run_model_pbar) +atexit.register(_restore_print) + ################################################################################ # @@ -150,6 +193,9 @@ def train_model(data_folder, model_folder, verbose, csv_path=DEFAULT_CSV_PATH): # Load your trained models. This function is *required*. You should edit this function to add your code, but do *not* change the # arguments of this function. If you do not train one of the models, then you can return None for the model. def load_model(model_folder, verbose): + if verbose: + _install_run_print_filter() + model_filename = os.path.join(model_folder, 'model.sav') model = joblib.load(model_filename) return model @@ -157,6 +203,8 @@ def load_model(model_folder, verbose): # Run your trained model. This function is *required*. You should edit this function to add your code, but do *not* change the # arguments of this function. def run_model(model, record, data_folder, verbose): + global RUN_MODEL_PBAR, RUN_MODEL_PBAR_TOTAL + # Load the model. model = model['model'] @@ -165,6 +213,27 @@ def run_model(model, record, data_folder, verbose): site_id = record[HEADERS['site_id']] session_id = record[HEADERS['session_id']] + # Initialize tqdm progress bar lazily so it advances across run_model calls. + if verbose and RUN_MODEL_PBAR is None: + patient_data_file = os.path.join(data_folder, DEMOGRAPHICS_FILE) + try: + RUN_MODEL_PBAR_TOTAL = len(find_patients(patient_data_file)) + except Exception: + RUN_MODEL_PBAR_TOTAL = None + + RUN_MODEL_PBAR = tqdm( + total=RUN_MODEL_PBAR_TOTAL, + desc="Running Model", + unit="record", + leave=True, + file=sys.stdout, + delay=0.5, + disable=not verbose + ) + + if verbose and RUN_MODEL_PBAR is not None: + RUN_MODEL_PBAR.set_postfix({"patient": patient_id}) + # Load the patient data. patient_data_file = os.path.join(data_folder, DEMOGRAPHICS_FILE) patient_data = load_demographics(patient_data_file, patient_id, session_id) @@ -195,6 +264,12 @@ def run_model(model, record, data_folder, verbose): binary_output = model.predict(features)[0] probability_output = model.predict_proba(features)[0][1] + if verbose and RUN_MODEL_PBAR is not None: + RUN_MODEL_PBAR.update(1) + if RUN_MODEL_PBAR_TOTAL is not None and RUN_MODEL_PBAR.n >= RUN_MODEL_PBAR_TOTAL: + RUN_MODEL_PBAR.close() + RUN_MODEL_PBAR = None + return binary_output, probability_output ################################################################################ From 5e05482d486213ccc5b55165c46ac30f8ade7ddf Mon Sep 17 00:00:00 2001 From: dcajal Date: Tue, 3 Mar 2026 13:56:15 +0100 Subject: [PATCH 13/93] Update run script and documentation to include new commands for development mode --- docs/04_run_script.md | 32 ++++---- scripts/run.ps1 | 169 +++++++++++++++++++++++++++++------------- 2 files changed, 136 insertions(+), 65 deletions(-) diff --git a/docs/04_run_script.md b/docs/04_run_script.md index 694c4f3..361df34 100644 --- a/docs/04_run_script.md +++ b/docs/04_run_script.md @@ -22,12 +22,18 @@ Para simplificar el trabajo del equipo hemos creado un único script que encapsu # Comandos disponibles +Nota: Si PowerShell bloquea la ejecución, ejecutar primero: + +```powershell +Set-ExecutionPolicy -Scope Process -ExecutionPolicy Bypass +``` + Desde la raíz del repositorio: ## 1️⃣ Construir la imagen Docker ```powershell -.\run.ps1 build +.\scripts\run.ps1 build ``` Solo es necesario hacerlo: @@ -38,7 +44,7 @@ Solo es necesario hacerlo: ## 2️⃣ Crear dataset smoke (5 sujetos) ```powershell -.\run.ps1 smoke +.\scripts\run.ps1 smoke ``` Genera: `data/training_smoke/` @@ -50,14 +56,14 @@ Este dataset se usa para desarrollo rápido. ### Smoke (rápido) ```powershell -.\run.ps1 train-smoke +.\scripts\run.ps1 train-smoke ``` ### Completo ```powershell -.\run.ps1 train +.\scripts\run.ps1 train ``` El modelo se guarda en: @@ -70,13 +76,13 @@ El modelo se guarda en: ### Smoke ```powershell -.\run.ps1 run-smoke +.\scripts\run.ps1 run-smoke ``` ### Completo ```powershell -.\run.ps1 run +.\scripts\run.ps1 run ``` Los resultados se generan en: @@ -89,7 +95,7 @@ El archivo clave es: `demographics.csv` que contiene las predicciones añadidas. ## 5️⃣ Limpiar artefactos ```powershell -.\run.ps1 clean +.\scripts\run.ps1 clean ``` Elimina: @@ -104,15 +110,15 @@ No elimina datasets. # Flujo recomendado para desarrollo ```powershell -.\run.ps1 build -.\run.ps1 smoke -.\run.ps1 train-smoke -.\run.ps1 run-smoke +.\scripts\run.ps1 build +.\scripts\run.ps1 smoke +.\scripts\run.ps1 train-smoke +.\scripts\run.ps1 run-smoke ``` Solo cuando el modelo esté estable: ```powershell -.\run.ps1 train -.\run.ps1 run +.\scripts\run.ps1 train +.\scripts\run.ps1 run ``` \ No newline at end of file diff --git a/scripts/run.ps1 b/scripts/run.ps1 index de6122d..41a9df7 100644 --- a/scripts/run.ps1 +++ b/scripts/run.ps1 @@ -1,11 +1,21 @@ param( [Parameter(Mandatory=$true)] - [ValidateSet("build","smoke","train","train-smoke","run","run-smoke","clean")] + [ValidateSet( + "build", + "smoke", + "train", + "train-smoke", + "run", + "run-smoke", + "train-dev", + "run-dev", + "clean" + )] [string]$Command ) # ============================================ -# CONFIGURATION +# CONFIGURACIÓN # ============================================ # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> @@ -13,18 +23,33 @@ param( # Si tu dataset no está en data/training_set, # modifica esta ruta. # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> -$FULL_DATA="data/training_set" +$FULL_DATA_REL = "data/training_set" +$SMOKE_DATA_REL = "data/training_smoke" -$SMOKE_DATA="data/training_smoke" +$IMAGE_NAME = "cinc2026" -$IMAGE_NAME="cinc2026" -$MODEL_FULL="model" -$MODEL_SMOKE="model_smoke" -$OUT_FULL="outputs" -$OUT_SMOKE="outputs_smoke" +$MODEL_FULL_REL = "model" +$MODEL_SMOKE_REL = "model_smoke" + +$OUT_FULL_REL = "outputs" +$OUT_SMOKE_REL = "outputs_smoke" + +# ============================================ +# FUNCIONES AUXILIARES +# ============================================ + +function Get-AbsolutePath($relativePath) { + return (Resolve-Path $relativePath).Path +} + +function Ensure-Directory($path) { + if (!(Test-Path $path)) { + New-Item -ItemType Directory -Force -Path $path | Out-Null + } +} # ============================================ -# FUNCTIONS +# COMANDOS # ============================================ function Build-Image { @@ -32,92 +57,132 @@ function Build-Image { } function Create-Smoke { - Write-Host "Creating smoke dataset..." + Write-Host "Creando dataset smoke..." powershell -ExecutionPolicy Bypass -File scripts/create_smoke.ps1 } function Train-Full { - New-Item -ItemType Directory -Force -Path $MODEL_FULL | Out-Null + + $FULL_DATA = Get-AbsolutePath $FULL_DATA_REL + $MODEL_FULL = Join-Path (Get-AbsolutePath ".") $MODEL_FULL_REL + + Ensure-Directory $MODEL_FULL docker run --rm ` -v "${FULL_DATA}:/challenge/training_data:ro" ` - -v "${PWD}/${MODEL_FULL}:/challenge/model" ` + -v "${MODEL_FULL}:/challenge/model" ` $IMAGE_NAME ` python train_model.py -d training_data -m model -v } function Train-Smoke { - New-Item -ItemType Directory -Force -Path $MODEL_SMOKE | Out-Null + + $SMOKE_DATA = Get-AbsolutePath $SMOKE_DATA_REL + $MODEL_SMOKE = Join-Path (Get-AbsolutePath ".") $MODEL_SMOKE_REL + + Ensure-Directory $MODEL_SMOKE docker run --rm ` -v "${SMOKE_DATA}:/challenge/training_data:ro" ` - -v "${PWD}/${MODEL_SMOKE}:/challenge/model" ` + -v "${MODEL_SMOKE}:/challenge/model" ` $IMAGE_NAME ` python train_model.py -d training_data -m model -v } function Run-Full { - New-Item -ItemType Directory -Force -Path $OUT_FULL | Out-Null + + $FULL_DATA = Get-AbsolutePath $FULL_DATA_REL + $MODEL_FULL = Get-AbsolutePath $MODEL_FULL_REL + $OUT_FULL = Join-Path (Get-AbsolutePath ".") $OUT_FULL_REL + + Ensure-Directory $OUT_FULL docker run --rm ` -v "${FULL_DATA}:/challenge/holdout_data:ro" ` - -v "${PWD}/${MODEL_FULL}:/challenge/model:ro" ` - -v "${PWD}/${OUT_FULL}:/challenge/holdout_outputs" ` + -v "${MODEL_FULL}:/challenge/model:ro" ` + -v "${OUT_FULL}:/challenge/holdout_outputs" ` $IMAGE_NAME ` python run_model.py -d holdout_data -m model -o holdout_outputs -v } function Run-Smoke { - New-Item -ItemType Directory -Force -Path $OUT_SMOKE | Out-Null + + $SMOKE_DATA = Get-AbsolutePath $SMOKE_DATA_REL + $MODEL_SMOKE = Get-AbsolutePath $MODEL_SMOKE_REL + $OUT_SMOKE = Join-Path (Get-AbsolutePath ".") $OUT_SMOKE_REL + + Ensure-Directory $OUT_SMOKE docker run --rm ` -v "${SMOKE_DATA}:/challenge/holdout_data:ro" ` - -v "${PWD}/${MODEL_SMOKE}:/challenge/model:ro" ` - -v "${PWD}/${OUT_SMOKE}:/challenge/holdout_outputs" ` + -v "${MODEL_SMOKE}:/challenge/model:ro" ` + -v "${OUT_SMOKE}:/challenge/holdout_outputs" ` $IMAGE_NAME ` python run_model.py -d holdout_data -m model -o holdout_outputs -v } -function Clean-All { - Remove-Item -Recurse -Force $MODEL_FULL -ErrorAction SilentlyContinue - Remove-Item -Recurse -Force $MODEL_SMOKE -ErrorAction SilentlyContinue - Remove-Item -Recurse -Force $OUT_FULL -ErrorAction SilentlyContinue - Remove-Item -Recurse -Force $OUT_SMOKE -ErrorAction SilentlyContinue - Write-Host "Cleaned model and output folders." +# ====================== +# MODO DESARROLLO (SIN REBUILD) +# ====================== + +function Train-Dev { + + $CODE_PATH = Get-AbsolutePath "." + $SMOKE_DATA = Get-AbsolutePath $SMOKE_DATA_REL + $MODEL_SMOKE = Join-Path $CODE_PATH $MODEL_SMOKE_REL + + Ensure-Directory $MODEL_SMOKE + + docker run --rm ` + -v "${CODE_PATH}:/challenge" ` + -v "${SMOKE_DATA}:/challenge/training_data:ro" ` + -v "${MODEL_SMOKE}:/challenge/model" ` + $IMAGE_NAME ` + python train_model.py -d training_data -m model -v } -# ============================================ -# COMMAND SWITCH -# ============================================ +function Run-Dev { -switch ($Command) { + $CODE_PATH = Get-AbsolutePath "." + $SMOKE_DATA = Get-AbsolutePath $SMOKE_DATA_REL + $MODEL_SMOKE = Get-AbsolutePath $MODEL_SMOKE_REL + $OUT_SMOKE = Join-Path $CODE_PATH $OUT_SMOKE_REL - "build" { - Build-Image - } + Ensure-Directory $OUT_SMOKE - "smoke" { - Create-Smoke - } + docker run --rm ` + -v "${CODE_PATH}:/challenge" ` + -v "${SMOKE_DATA}:/challenge/holdout_data:ro" ` + -v "${MODEL_SMOKE}:/challenge/model:ro" ` + -v "${OUT_SMOKE}:/challenge/holdout_outputs" ` + $IMAGE_NAME ` + python run_model.py -d holdout_data -m model -o holdout_outputs -v +} - "train" { - Train-Full - } +function Clean-All { - "train-smoke" { - Train-Smoke - } + Remove-Item -Recurse -Force $MODEL_FULL_REL -ErrorAction SilentlyContinue + Remove-Item -Recurse -Force $MODEL_SMOKE_REL -ErrorAction SilentlyContinue + Remove-Item -Recurse -Force $OUT_FULL_REL -ErrorAction SilentlyContinue + Remove-Item -Recurse -Force $OUT_SMOKE_REL -ErrorAction SilentlyContinue - "run" { - Run-Full - } + Write-Host "Modelos y outputs eliminados." +} - "run-smoke" { - Run-Smoke - } +# ============================================ +# SWITCH PRINCIPAL +# ============================================ - "clean" { - Clean-All - } +switch ($Command) { + + "build" { Build-Image } + "smoke" { Create-Smoke } + "train" { Train-Full } + "train-smoke" { Train-Smoke } + "run" { Run-Full } + "run-smoke" { Run-Smoke } + "train-dev" { Train-Dev } + "run-dev" { Run-Dev } + "clean" { Clean-All } } \ No newline at end of file From 4d080a199be4b15bcca201f1d04b23cd8eb040b0 Mon Sep 17 00:00:00 2001 From: dcajal Date: Tue, 3 Mar 2026 14:06:31 +0100 Subject: [PATCH 14/93] Add unified run script with commands for building, training, and running models --- docs/04_run_script.md | 169 ++++++++++++++++++++++++------------- scripts/run.ps1 => run.ps1 | 0 2 files changed, 112 insertions(+), 57 deletions(-) rename scripts/run.ps1 => run.ps1 (100%) diff --git a/docs/04_run_script.md b/docs/04_run_script.md index 361df34..61cccd3 100644 --- a/docs/04_run_script.md +++ b/docs/04_run_script.md @@ -1,124 +1,179 @@ # Script unificado de ejecución (`run.ps1`) -Para simplificar el trabajo del equipo hemos creado un único script que encapsula todos los comandos necesarios para: +Este script centraliza todos los comandos necesarios para trabajar en el proyecto: -- Construir la imagen Docker -- Crear el dataset smoke -- Entrenar (completo o smoke) -- Generar predicciones -- Limpiar artefactos +- Construir la imagen Docker +- Crear el dataset smoke +- Entrenar (modo desarrollo o completo) +- Generar predicciones +- Limpiar artefactos --- -## Requisitos +# Requisitos -- Docker Desktop instalado -- Dataset descargado en: `data/training_set/` +- Docker Desktop instalado +- Dataset descargado en: +``` +data/training_set/ +``` -⚠️ Si el dataset está en otra ubicación, modificar la variable `$FULL_DATA` dentro de `run.ps1`. +⚠️ Si el dataset está en otra ubicación, modificar la variable `$FULL_DATA_REL` +dentro de `run.ps1`. --- # Comandos disponibles -Nota: Si PowerShell bloquea la ejecución, ejecutar primero: - -```powershell -Set-ExecutionPolicy -Scope Process -ExecutionPolicy Bypass -``` - Desde la raíz del repositorio: +--- + ## 1️⃣ Construir la imagen Docker ```powershell -.\scripts\run.ps1 build +.\run.ps1 build ``` Solo es necesario hacerlo: -- La primera vez -- Cuando cambien dependencias o el Dockerfile +- La primera vez +- Cuando cambien `requirements.txt` +- Cuando cambie el `Dockerfile` + +No es necesario al modificar `team_code.py` en modo desarrollo. + +--- ## 2️⃣ Crear dataset smoke (5 sujetos) ```powershell -.\scripts\run.ps1 smoke +.\run.ps1 smoke +``` + +Genera: + +``` +data/training_smoke/ ``` -Genera: `data/training_smoke/` +Este dataset se utiliza exclusivamente para desarrollo rápido. + +--- + +# 🚀 Modo desarrollo (rápido) -Este dataset se usa para desarrollo rápido. +Estos comandos: -## 3️⃣ Entrenar modelo +- Usan el dataset smoke +- Montan el código como volumen +- No requieren rebuild al modificar Python -### Smoke (rápido) +--- + +## 3️⃣ Entrenar en modo desarrollo ```powershell -.\scripts\run.ps1 train-smoke +.\run.ps1 train-dev ``` +Utiliza: + +- `data/training_smoke` +- `model_smoke/` -### Completo +--- + +## 4️⃣ Generar predicciones en modo desarrollo ```powershell -.\scripts\run.ps1 train +.\run.ps1 run-dev ``` -El modelo se guarda en: +Genera resultados en: -- model/ (full) -- model_smoke/ (smoke) +``` +outputs_smoke/ +``` -## 4️⃣ Generar predicciones +--- -### Smoke +## 🔁 Flujo recomendado de desarrollo ```powershell -.\scripts\run.ps1 run-smoke +.\run.ps1 build # solo la primera vez +.\run.ps1 smoke # solo si no existe +.\run.ps1 train-dev +.\run.ps1 run-dev ``` -### Completo +Este flujo debe usarse para: + +- Probar nuevas features +- Ajustar el modelo +- Depurar errores +- Iterar rápidamente + +--- + +# 🧪 Entrenamiento completo + +Solo cuando el modelo esté estable. + +--- + +## 5️⃣ Entrenar con dataset completo ```powershell -.\scripts\run.ps1 run +.\run.ps1 train ``` -Los resultados se generan en: +Guarda el modelo en: -- outputs/ -- outputs_smoke/ +``` +model/ +``` -El archivo clave es: `demographics.csv` que contiene las predicciones añadidas. +--- -## 5️⃣ Limpiar artefactos +## 6️⃣ Generar predicciones completas ```powershell -.\scripts\run.ps1 clean +.\run.ps1 run ``` -Elimina: +Genera resultados en: -- model/ -- model_smoke/ -- outputs/ -- outputs_smoke/ +``` +outputs/ +``` -No elimina datasets. +--- -# Flujo recomendado para desarrollo +# 🧹 Limpiar artefactos ```powershell -.\scripts\run.ps1 build -.\scripts\run.ps1 smoke -.\scripts\run.ps1 train-smoke -.\scripts\run.ps1 run-smoke +.\run.ps1 clean ``` -Solo cuando el modelo esté estable: +Elimina: -```powershell -.\scripts\run.ps1 train -.\scripts\run.ps1 run -``` \ No newline at end of file +- `model/` +- `model_smoke/` +- `outputs/` +- `outputs_smoke/` + +No elimina datasets. + +# Estrategia recomendada del equipo + +1. Desarrollar siempre en modo `*-dev`. +2. Entrenar en full solo antes de: + - Hacer merge a `main` + - Generar submission +3. Antes de enviar al challenge: + - Ejecutar `build` + - Ejecutar `train` + - Ejecutar `run` + - Verificar que funciona sin modo dev diff --git a/scripts/run.ps1 b/run.ps1 similarity index 100% rename from scripts/run.ps1 rename to run.ps1 From a2a52f33cdc79e849d534d8efab4c195247f193b Mon Sep 17 00:00:00 2001 From: dcajal Date: Tue, 3 Mar 2026 14:10:48 +0100 Subject: [PATCH 15/93] Add note for PowerShell script execution policy in run script documentation --- docs/04_run_script.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/04_run_script.md b/docs/04_run_script.md index 61cccd3..c7aba9e 100644 --- a/docs/04_run_script.md +++ b/docs/04_run_script.md @@ -22,6 +22,11 @@ data/training_set/ ⚠️ Si el dataset está en otra ubicación, modificar la variable `$FULL_DATA_REL` dentro de `run.ps1`. +⚠️ Si PowerShell bloquea la ejecución de scripts, ejecutar (aplica solo para la sesión actual): + +```powershell +Set-ExecutionPolicy -Scope Process -ExecutionPolicy Bypass +``` --- # Comandos disponibles From ec74b589319e6a6ecf3768e8e1239d09774dad02 Mon Sep 17 00:00:00 2001 From: dcajal Date: Tue, 3 Mar 2026 16:46:26 +0100 Subject: [PATCH 16/93] Add bash run script and create smoke dataset script for model training and execution --- run.sh | 173 ++++++++++++++++++++++++++++++++++++++++ scripts/create_smoke.sh | 47 +++++++++++ 2 files changed, 220 insertions(+) create mode 100644 run.sh create mode 100644 scripts/create_smoke.sh diff --git a/run.sh b/run.sh new file mode 100644 index 0000000..eadf715 --- /dev/null +++ b/run.sh @@ -0,0 +1,173 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -lt 1 ]]; then + echo "Usage: $0 " + exit 1 +fi + +COMMAND="$1" + +# ============================================ +# CONFIGURATION +# ============================================ + +FULL_DATA_REL="data/training_set" +SMOKE_DATA_REL="data/training_smoke" + +IMAGE_NAME="cinc2026" + +MODEL_FULL_REL="model" +MODEL_SMOKE_REL="model_smoke" + +OUT_FULL_REL="outputs" +OUT_SMOKE_REL="outputs_smoke" + +# ============================================ +# HELPERS +# ============================================ + +get_absolute_path() { + local rel_path="$1" + (cd "$rel_path" && pwd) +} + +ensure_directory() { + local dir_path="$1" + mkdir -p "$dir_path" +} + +build_image() { + docker build -t "$IMAGE_NAME" . +} + +create_smoke() { + echo "Creating smoke dataset..." + bash scripts/create_smoke.sh +} + +train_full() { + local full_data model_full + + full_data="$(get_absolute_path "$FULL_DATA_REL")" + model_full="$(get_absolute_path ".")/${MODEL_FULL_REL}" + + ensure_directory "$model_full" + + docker run --rm \ + -v "${full_data}:/challenge/training_data:ro" \ + -v "${model_full}:/challenge/model" \ + "$IMAGE_NAME" \ + python train_model.py -d training_data -m model -v +} + +train_smoke() { + local smoke_data model_smoke + + smoke_data="$(get_absolute_path "$SMOKE_DATA_REL")" + model_smoke="$(get_absolute_path ".")/${MODEL_SMOKE_REL}" + + ensure_directory "$model_smoke" + + docker run --rm \ + -v "${smoke_data}:/challenge/training_data:ro" \ + -v "${model_smoke}:/challenge/model" \ + "$IMAGE_NAME" \ + python train_model.py -d training_data -m model -v +} + +run_full() { + local full_data model_full out_full + + full_data="$(get_absolute_path "$FULL_DATA_REL")" + model_full="$(get_absolute_path "$MODEL_FULL_REL")" + out_full="$(get_absolute_path ".")/${OUT_FULL_REL}" + + ensure_directory "$out_full" + + docker run --rm \ + -v "${full_data}:/challenge/holdout_data:ro" \ + -v "${model_full}:/challenge/model:ro" \ + -v "${out_full}:/challenge/holdout_outputs" \ + "$IMAGE_NAME" \ + python run_model.py -d holdout_data -m model -o holdout_outputs -v +} + +run_smoke() { + local smoke_data model_smoke out_smoke + + smoke_data="$(get_absolute_path "$SMOKE_DATA_REL")" + model_smoke="$(get_absolute_path "$MODEL_SMOKE_REL")" + out_smoke="$(get_absolute_path ".")/${OUT_SMOKE_REL}" + + ensure_directory "$out_smoke" + + docker run --rm \ + -v "${smoke_data}:/challenge/holdout_data:ro" \ + -v "${model_smoke}:/challenge/model:ro" \ + -v "${out_smoke}:/challenge/holdout_outputs" \ + "$IMAGE_NAME" \ + python run_model.py -d holdout_data -m model -o holdout_outputs -v +} + +# ===================== +# DEVELOPMENT MODE (NO REBUILD) +# ===================== + +train_dev() { + local code_path smoke_data model_smoke + + code_path="$(get_absolute_path ".")" + smoke_data="$(get_absolute_path "$SMOKE_DATA_REL")" + model_smoke="${code_path}/${MODEL_SMOKE_REL}" + + ensure_directory "$model_smoke" + + docker run --rm \ + -v "${code_path}:/challenge" \ + -v "${smoke_data}:/challenge/training_data:ro" \ + -v "${model_smoke}:/challenge/model" \ + "$IMAGE_NAME" \ + python train_model.py -d training_data -m model -v +} + +run_dev() { + local code_path smoke_data model_smoke out_smoke + + code_path="$(get_absolute_path ".")" + smoke_data="$(get_absolute_path "$SMOKE_DATA_REL")" + model_smoke="$(get_absolute_path "$MODEL_SMOKE_REL")" + out_smoke="${code_path}/${OUT_SMOKE_REL}" + + ensure_directory "$out_smoke" + + docker run --rm \ + -v "${code_path}:/challenge" \ + -v "${smoke_data}:/challenge/holdout_data:ro" \ + -v "${model_smoke}:/challenge/model:ro" \ + -v "${out_smoke}:/challenge/holdout_outputs" \ + "$IMAGE_NAME" \ + python run_model.py -d holdout_data -m model -o holdout_outputs -v +} + +clean_all() { + rm -rf "$MODEL_FULL_REL" "$MODEL_SMOKE_REL" "$OUT_FULL_REL" "$OUT_SMOKE_REL" + echo "Models and outputs removed." +} + +case "$COMMAND" in + build) build_image ;; + smoke) create_smoke ;; + train) train_full ;; + train-smoke) train_smoke ;; + run) run_full ;; + run-smoke) run_smoke ;; + train-dev) train_dev ;; + run-dev) run_dev ;; + clean) clean_all ;; + *) + echo "Invalid command: $COMMAND" + echo "Valid commands: build, smoke, train, train-smoke, run, run-smoke, train-dev, run-dev, clean" + exit 1 + ;; +esac diff --git a/scripts/create_smoke.sh b/scripts/create_smoke.sh new file mode 100644 index 0000000..34bd020 --- /dev/null +++ b/scripts/create_smoke.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +set -euo pipefail + +# ============================================ +# Create smoke training dataset +# ============================================ + +# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> +# IMPORTANT: +# Each team member can modify this path to +# match their local dataset location. +# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> +FULL_DATA_PATH="${FULL_DATA_PATH:-data/training_set}" # Override with env var if needed + +SMOKE_PATH="data/training_smoke" +N_RECORDS="${N_RECORDS:-5}" + +echo "Creating smoke dataset..." +echo "Source: ${FULL_DATA_PATH}" +echo "Destination: ${SMOKE_PATH}" + +rm -rf "${SMOKE_PATH}" +mkdir -p "${SMOKE_PATH}" + +# Copy demographics +cp "${FULL_DATA_PATH}/demographics.csv" "${SMOKE_PATH}/demographics.csv" + +# Select first N EDF files +while IFS= read -r file_path; do + rel_path="${file_path#${FULL_DATA_PATH}/}" + target_path="${SMOKE_PATH}/${rel_path}" + mkdir -p "$(dirname "${target_path}")" + cp "${file_path}" "${target_path}" +done < <( + find "${FULL_DATA_PATH}/physiological_data" -type f -name "*.edf" | sort | head -n "${N_RECORDS}" +) + +# Copy full annotation folders (simpler and robust) +if [[ -d "${FULL_DATA_PATH}/algorithmic_annotations" ]]; then + cp -R "${FULL_DATA_PATH}/algorithmic_annotations" "${SMOKE_PATH}/algorithmic_annotations" +fi + +if [[ -d "${FULL_DATA_PATH}/human_annotations" ]]; then + cp -R "${FULL_DATA_PATH}/human_annotations" "${SMOKE_PATH}/human_annotations" +fi + +echo "Smoke dataset created successfully." From 67d9b636afe1f2bfaad8a5019ed7e96ac13e7621 Mon Sep 17 00:00:00 2001 From: dcajal Date: Tue, 3 Mar 2026 16:52:17 +0100 Subject: [PATCH 17/93] Update Docker and smoke dataset documentation for clarity and structure --- docs/02_docker.md | 55 ++++++------ docs/03_smoke_dataset.md | 51 +++++------ docs/04_run_script.md | 177 +++++++++++---------------------------- 3 files changed, 91 insertions(+), 192 deletions(-) diff --git a/docs/02_docker.md b/docs/02_docker.md index eca7735..c24ced2 100644 --- a/docs/02_docker.md +++ b/docs/02_docker.md @@ -1,53 +1,46 @@ # Uso de Docker +Este documento define el contexto de ejecución con Docker. + ## Requisitos - Docker Desktop instalado (modo Linux containers) - Dataset descargado desde Kaggle -- Se asume que el dataset está en: data/training_set/ +- Dataset completo disponible en `data/training_set/` (ruta por defecto del proyecto) -Cada miembro del equipo puede tener el dataset en una ubicación diferente, pero en este repositorio asumimos que está dentro de `data/`. +Si tu dataset está en otra ubicación, actualiza la variable de ruta en el script de ejecución. ---- +## Estructura de trabajo -## Construir la imagen +Entradas: -Desde la raíz del repositorio: +- `data/training_set/` (dataset completo) +- `data/training_smoke/` (dataset reducido para modo desarrollo (smoke)) -```powershell -docker build -t cinc2026 . -``` +Salidas: -## Entenar con el dataset completo +- `model/` y `outputs/` (flujo completo) +- `model_smoke/` y `outputs_smoke/` (flujo smoke/desarrollo) -```powershell -$DATA="data/training_set" -$MODEL="$PWD/model" +## Orden recomendado de ejecución -docker run --rm ` - -v "${DATA}:/challenge/training_data:ro" ` - -v "${MODEL}:/challenge/model" ` - cinc2026 ` - python train_model.py -d training_data -m model -v -``` +1. Construir imagen Docker (`build`) +2. Preparar dataset smoke (`smoke`) +3. Iterar en modo desarrollo (smoke) (`train-dev` / `run-dev`) +4. Ejecutar validación completa (`train` / `run`) +5. Limpiar artefactos cuando corresponda (`clean`) -## Generar predicciones +La guía paso a paso está en `docs/04_run_script.md`. -```powershell -$OUT="$PWD/outputs" +## Compatibilidad de scripts -docker run --rm ` - -v "${DATA}:/challenge/holdout_data:ro" ` - -v "${MODEL}:/challenge/model:ro" ` - -v "${OUT}:/challenge/holdout_outputs" ` - cinc2026 ` - python run_model.py -d holdout_data -m model -o holdout_outputs -v -``` +El flujo principal del equipo está documentado con `run.sh` (Git Bash). +También existen equivalentes en PowerShell: `run.ps1` y `scripts/create_smoke.ps1`. ## Resultado esperado -En la carpeta `outputs/` se generará un `demographics.csv` con: +Tras ejecutar la generación de predicciones (inferencia) completa, en `outputs/` se genera un `demographics.csv` con: - Columnas originales -- Cognitive_Impairment -- Cognitive_Impairment_Probability \ No newline at end of file +- `Cognitive_Impairment` +- `Cognitive_Impairment_Probability` \ No newline at end of file diff --git a/docs/03_smoke_dataset.md b/docs/03_smoke_dataset.md index f91249d..5ba827b 100644 --- a/docs/03_smoke_dataset.md +++ b/docs/03_smoke_dataset.md @@ -1,51 +1,40 @@ -# Dataset Smoke (Desarrollo Rápido) +# Dataset smoke (Modo desarrollo) Entrenar con el dataset completo tarda aproximadamente 30–40 minutos con el modelo de ejemplo. Para desarrollo utilizamos un dataset reducido (5 sujetos). ---- - -## Crear dataset smoke +Este documento describe cuándo y por qué usar smoke. +Los comandos de ejecución están centralizados en `docs/04_run_script.md`. -Ejecutar: +--- -```powershell -powershell -ExecutionPolicy Bypass -File scripts/create_smoke.ps1 -``` +## Qué incluye -Esto generará: `data/training_smoke/` +- Muestra reducida del dataset (5 sujetos) +- Estructura compatible con el flujo oficial del proyecto +- Directorio de salida en `data/training_smoke/` -## Entrenar con el smoke dataset +## Para qué se usa -```powershell -$DATA="$PWD\data\training_smoke" -$MODEL="$PWD\model_smoke" +- Validar cambios de código rápidamente +- Detectar errores de integración antes del entrenamiento completo +- Iterar en modo desarrollo (smoke) sin esperar ciclos largos -docker run --rm ` - -v "${DATA}:/challenge/training_data:ro" ` - -v "${MODEL}:/challenge/model" ` - cinc2026 ` - python train_model.py -d training_data -m model -v -``` +## Artefactos asociados -## Generar predicciones con smoke dataset +- Entrenamiento smoke: `model_smoke/` +- Predicciones (inferencia) smoke: `outputs_smoke/` -```powershell -$OUT="$PWD/outputs_smoke" +## Relación con el flujo principal -docker run --rm ` - -v "${DATA}:/challenge/holdout_data:ro" ` - -v "${MODEL}:/challenge/model:ro" ` - -v "${OUT}:/challenge/holdout_outputs" ` - cinc2026 ` - python run_model.py -d holdout_data -m model -o holdout_outputs -v -``` +El dataset smoke se crea al inicio del ciclo de desarrollo y se usa junto con `train-dev` y `run-dev`. +El orden detallado de ejecución está en `docs/04_run_script.md`. ## ¿Cuándo usar smoke? -- Desarrollo de nuevas features +- Desarrollo de nuevas funcionalidades - Comprobación rápida de que el código no rompe -- Validación de cambios en team_code.py +- Validación de cambios en `team_code.py` Nunca usar smoke para evaluar rendimiento final. \ No newline at end of file diff --git a/docs/04_run_script.md b/docs/04_run_script.md index c7aba9e..2ee0517 100644 --- a/docs/04_run_script.md +++ b/docs/04_run_script.md @@ -1,12 +1,7 @@ -# Script unificado de ejecución (`run.ps1`) +# Script unificado de ejecución (`run.sh`) -Este script centraliza todos los comandos necesarios para trabajar en el proyecto: - -- Construir la imagen Docker -- Crear el dataset smoke -- Entrenar (modo desarrollo o completo) -- Generar predicciones -- Limpiar artefactos +Este documento es la guía operativa única para ejecutar el proyecto. +Aquí se define el orden recomendado y los comandos asociados. --- @@ -20,165 +15,87 @@ data/training_set/ ``` ⚠️ Si el dataset está en otra ubicación, modificar la variable `$FULL_DATA_REL` -dentro de `run.ps1`. - -⚠️ Si PowerShell bloquea la ejecución de scripts, ejecutar (aplica solo para la sesión actual): - -```powershell -Set-ExecutionPolicy -Scope Process -ExecutionPolicy Bypass -``` ---- +dentro de `run.sh`. -# Comandos disponibles +⚠️ Ejecutar los comandos desde Git Bash. -Desde la raíz del repositorio: +ℹ️ Existen scripts equivalentes en PowerShell (`run.ps1` y `scripts/create_smoke.ps1`) para quienes prefieran ese entorno. +ℹ️ Para contexto general y definición de artefactos, ver `docs/02_docker.md` y `docs/03_smoke_dataset.md`. --- -## 1️⃣ Construir la imagen Docker - -```powershell -.\run.ps1 build -``` - -Solo es necesario hacerlo: - -- La primera vez -- Cuando cambien `requirements.txt` -- Cuando cambie el `Dockerfile` +# Orden de ejecución recomendado -No es necesario al modificar `team_code.py` en modo desarrollo. +Desde la raíz del repositorio. ---- +## 1) Preparar entorno -## 2️⃣ Crear dataset smoke (5 sujetos) +### Construir imagen Docker -```powershell -.\run.ps1 smoke +```bash +./run.sh build ``` -Genera: +Ejecutar la primera vez y cada vez que cambien `requirements.txt` o `Dockerfile`. -``` -data/training_smoke/ -``` +### Crear dataset smoke (5 sujetos) -Este dataset se utiliza exclusivamente para desarrollo rápido. - ---- - -# 🚀 Modo desarrollo (rápido) - -Estos comandos: - -- Usan el dataset smoke -- Montan el código como volumen -- No requieren rebuild al modificar Python - ---- - -## 3️⃣ Entrenar en modo desarrollo - -```powershell -.\run.ps1 train-dev +```bash +./run.sh smoke ``` -Utiliza: - -- `data/training_smoke` -- `model_smoke/` - ---- - -## 4️⃣ Generar predicciones en modo desarrollo +Genera `data/training_smoke/`. -```powershell -.\run.ps1 run-dev -``` +## 2) Ciclo en modo desarrollo (smoke) -Genera resultados en: +### Entrenar en modo desarrollo (smoke) -``` -outputs_smoke/ +```bash +./run.sh train-dev ``` ---- +Usa `data/training_smoke/` y guarda modelo en `model_smoke/`. -## 🔁 Flujo recomendado de desarrollo +### Generar predicciones (inferencia) en modo desarrollo (smoke) -```powershell -.\run.ps1 build # solo la primera vez -.\run.ps1 smoke # solo si no existe -.\run.ps1 train-dev -.\run.ps1 run-dev +```bash +./run.sh run-dev ``` -Este flujo debe usarse para: - -- Probar nuevas features -- Ajustar el modelo -- Depurar errores -- Iterar rápidamente - ---- - -# 🧪 Entrenamiento completo - -Solo cuando el modelo esté estable. - ---- +Genera resultados en `outputs_smoke/`. -## 5️⃣ Entrenar con dataset completo +### Secuencia típica en modo desarrollo (smoke) -```powershell -.\run.ps1 train +```bash +./run.sh build # solo la primera vez +./run.sh smoke # solo si no existe +./run.sh train-dev +./run.sh run-dev ``` -Guarda el modelo en: +## 3) Validación completa -``` -model/ -``` - ---- - -## 6️⃣ Generar predicciones completas +### Entrenar con dataset completo -```powershell -.\run.ps1 run +```bash +./run.sh train ``` -Genera resultados en: +Guarda el modelo en `model/`. -``` -outputs/ +### Generar predicciones (inferencia) completas + +```bash +./run.sh run ``` ---- +Genera resultados en `outputs/`. -# 🧹 Limpiar artefactos +## 4) Limpieza de artefactos -```powershell -.\run.ps1 clean +```bash +./run.sh clean ``` -Elimina: - -- `model/` -- `model_smoke/` -- `outputs/` -- `outputs_smoke/` - +Elimina `model/`, `model_smoke/`, `outputs/` y `outputs_smoke/`. No elimina datasets. - -# Estrategia recomendada del equipo - -1. Desarrollar siempre en modo `*-dev`. -2. Entrenar en full solo antes de: - - Hacer merge a `main` - - Generar submission -3. Antes de enviar al challenge: - - Ejecutar `build` - - Ejecutar `train` - - Ejecutar `run` - - Verificar que funciona sin modo dev From 819ca187fbc500381eff1a511fe3c58fcebc983d Mon Sep 17 00:00:00 2001 From: dcajal Date: Tue, 3 Mar 2026 18:28:37 +0100 Subject: [PATCH 18/93] Refactor Docker commands in run script for improved path handling and consistency --- run.sh | 86 +++++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 58 insertions(+), 28 deletions(-) diff --git a/run.sh b/run.sh index eadf715..fed6e06 100644 --- a/run.sh +++ b/run.sh @@ -37,8 +37,22 @@ ensure_directory() { mkdir -p "$dir_path" } +to_docker_path() { + local host_path="$1" + + if command -v cygpath >/dev/null 2>&1; then + cygpath -m "$host_path" + else + echo "$host_path" + fi +} + +docker_cli() { + MSYS_NO_PATHCONV=1 MSYS2_ARG_CONV_EXCL="*" docker "$@" +} + build_image() { - docker build -t "$IMAGE_NAME" . + docker_cli build -t "$IMAGE_NAME" . } create_smoke() { @@ -48,64 +62,78 @@ create_smoke() { train_full() { local full_data model_full + local full_data_docker model_full_docker full_data="$(get_absolute_path "$FULL_DATA_REL")" model_full="$(get_absolute_path ".")/${MODEL_FULL_REL}" + full_data_docker="$(to_docker_path "$full_data")" + model_full_docker="$(to_docker_path "$model_full")" ensure_directory "$model_full" - docker run --rm \ - -v "${full_data}:/challenge/training_data:ro" \ - -v "${model_full}:/challenge/model" \ + docker_cli run --rm \ + -v "${full_data_docker}:/challenge/training_data:ro" \ + -v "${model_full_docker}:/challenge/model" \ "$IMAGE_NAME" \ python train_model.py -d training_data -m model -v } train_smoke() { local smoke_data model_smoke + local smoke_data_docker model_smoke_docker smoke_data="$(get_absolute_path "$SMOKE_DATA_REL")" model_smoke="$(get_absolute_path ".")/${MODEL_SMOKE_REL}" + smoke_data_docker="$(to_docker_path "$smoke_data")" + model_smoke_docker="$(to_docker_path "$model_smoke")" ensure_directory "$model_smoke" - docker run --rm \ - -v "${smoke_data}:/challenge/training_data:ro" \ - -v "${model_smoke}:/challenge/model" \ + docker_cli run --rm \ + -v "${smoke_data_docker}:/challenge/training_data:ro" \ + -v "${model_smoke_docker}:/challenge/model" \ "$IMAGE_NAME" \ python train_model.py -d training_data -m model -v } run_full() { local full_data model_full out_full + local full_data_docker model_full_docker out_full_docker full_data="$(get_absolute_path "$FULL_DATA_REL")" model_full="$(get_absolute_path "$MODEL_FULL_REL")" out_full="$(get_absolute_path ".")/${OUT_FULL_REL}" + full_data_docker="$(to_docker_path "$full_data")" + model_full_docker="$(to_docker_path "$model_full")" + out_full_docker="$(to_docker_path "$out_full")" ensure_directory "$out_full" - docker run --rm \ - -v "${full_data}:/challenge/holdout_data:ro" \ - -v "${model_full}:/challenge/model:ro" \ - -v "${out_full}:/challenge/holdout_outputs" \ + docker_cli run --rm \ + -v "${full_data_docker}:/challenge/holdout_data:ro" \ + -v "${model_full_docker}:/challenge/model:ro" \ + -v "${out_full_docker}:/challenge/holdout_outputs" \ "$IMAGE_NAME" \ python run_model.py -d holdout_data -m model -o holdout_outputs -v } run_smoke() { local smoke_data model_smoke out_smoke + local smoke_data_docker model_smoke_docker out_smoke_docker smoke_data="$(get_absolute_path "$SMOKE_DATA_REL")" model_smoke="$(get_absolute_path "$MODEL_SMOKE_REL")" out_smoke="$(get_absolute_path ".")/${OUT_SMOKE_REL}" + smoke_data_docker="$(to_docker_path "$smoke_data")" + model_smoke_docker="$(to_docker_path "$model_smoke")" + out_smoke_docker="$(to_docker_path "$out_smoke")" ensure_directory "$out_smoke" - docker run --rm \ - -v "${smoke_data}:/challenge/holdout_data:ro" \ - -v "${model_smoke}:/challenge/model:ro" \ - -v "${out_smoke}:/challenge/holdout_outputs" \ + docker_cli run --rm \ + -v "${smoke_data_docker}:/challenge/holdout_data:ro" \ + -v "${model_smoke_docker}:/challenge/model:ro" \ + -v "${out_smoke_docker}:/challenge/holdout_outputs" \ "$IMAGE_NAME" \ python run_model.py -d holdout_data -m model -o holdout_outputs -v } @@ -116,38 +144,40 @@ run_smoke() { train_dev() { local code_path smoke_data model_smoke + local code_path_docker smoke_data_docker code_path="$(get_absolute_path ".")" smoke_data="$(get_absolute_path "$SMOKE_DATA_REL")" model_smoke="${code_path}/${MODEL_SMOKE_REL}" + code_path_docker="$(to_docker_path "$code_path")" + smoke_data_docker="$(to_docker_path "$smoke_data")" ensure_directory "$model_smoke" - docker run --rm \ - -v "${code_path}:/challenge" \ - -v "${smoke_data}:/challenge/training_data:ro" \ - -v "${model_smoke}:/challenge/model" \ + docker_cli run --rm \ + -v "${code_path_docker}:/challenge" \ + -v "${smoke_data_docker}:/challenge/data_smoke:ro" \ "$IMAGE_NAME" \ - python train_model.py -d training_data -m model -v + python train_model.py -d /challenge/data_smoke -m /challenge/model_smoke -v } run_dev() { - local code_path smoke_data model_smoke out_smoke + local code_path smoke_data out_smoke + local code_path_docker smoke_data_docker code_path="$(get_absolute_path ".")" smoke_data="$(get_absolute_path "$SMOKE_DATA_REL")" - model_smoke="$(get_absolute_path "$MODEL_SMOKE_REL")" out_smoke="${code_path}/${OUT_SMOKE_REL}" + code_path_docker="$(to_docker_path "$code_path")" + smoke_data_docker="$(to_docker_path "$smoke_data")" ensure_directory "$out_smoke" - docker run --rm \ - -v "${code_path}:/challenge" \ - -v "${smoke_data}:/challenge/holdout_data:ro" \ - -v "${model_smoke}:/challenge/model:ro" \ - -v "${out_smoke}:/challenge/holdout_outputs" \ + docker_cli run --rm \ + -v "${code_path_docker}:/challenge" \ + -v "${smoke_data_docker}:/challenge/data_smoke:ro" \ "$IMAGE_NAME" \ - python run_model.py -d holdout_data -m model -o holdout_outputs -v + python run_model.py -d /challenge/data_smoke -m /challenge/model_smoke -o /challenge/outputs_smoke -v } clean_all() { From 1ebe7a86a58f5c4e891bfaefd1374cd5933cb5cc Mon Sep 17 00:00:00 2001 From: dcajal Date: Tue, 3 Mar 2026 18:29:24 +0100 Subject: [PATCH 19/93] Move lolai_models to src --- {models => src}/lolai_models.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {models => src}/lolai_models.py (100%) diff --git a/models/lolai_models.py b/src/lolai_models.py similarity index 100% rename from models/lolai_models.py rename to src/lolai_models.py From b7cc2ea87c41fe29de934b0c32dbbc6369122a22 Mon Sep 17 00:00:00 2001 From: dcajal Date: Mon, 9 Mar 2026 13:12:46 +0100 Subject: [PATCH 20/93] Add evaluation commands for existing predictions in run scripts and documentation --- docs/04_run_script.md | 29 ++++++++++++++-- run.ps1 | 55 +++++++++++++++++++++++++++++ run.sh | 80 +++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 160 insertions(+), 4 deletions(-) diff --git a/docs/04_run_script.md b/docs/04_run_script.md index 2ee0517..e5f5907 100644 --- a/docs/04_run_script.md +++ b/docs/04_run_script.md @@ -62,7 +62,15 @@ Usa `data/training_smoke/` y guarda modelo en `model_smoke/`. ./run.sh run-dev ``` -Genera resultados en `outputs_smoke/`. +Genera resultados en `outputs_smoke/` y luego imprime métricas de evaluación en consola. + +### Evaluar predicciones existentes en modo desarrollo (smoke) + +```bash +./run.sh eval-dev +``` + +Reutiliza `outputs_smoke/demographics.csv` y muestra AUROC, AUPRC, Accuracy y F-measure sin volver a ejecutar inferencia. ### Secuencia típica en modo desarrollo (smoke) @@ -71,6 +79,7 @@ Genera resultados en `outputs_smoke/`. ./run.sh smoke # solo si no existe ./run.sh train-dev ./run.sh run-dev +./run.sh eval-dev # opcional: reevaluar sin correr inferencia ``` ## 3) Validación completa @@ -89,7 +98,23 @@ Guarda el modelo en `model/`. ./run.sh run ``` -Genera resultados en `outputs/`. +Genera resultados en `outputs/` y luego imprime métricas de evaluación en consola. + +### Evaluar predicciones existentes completas + +```bash +./run.sh eval +``` + +Reutiliza `outputs/demographics.csv` y muestra AUROC, AUPRC, Accuracy y F-measure sin volver a ejecutar inferencia. + +### Evaluar predicciones existentes del dataset smoke + +```bash +./run.sh eval-smoke +``` + +Reutiliza `outputs_smoke/demographics.csv` y muestra AUROC, AUPRC, Accuracy y F-measure sin volver a ejecutar inferencia. ## 4) Limpieza de artefactos diff --git a/run.ps1 b/run.ps1 index 41a9df7..439eade 100644 --- a/run.ps1 +++ b/run.ps1 @@ -7,8 +7,11 @@ param( "train-smoke", "run", "run-smoke", + "eval", + "eval-smoke", "train-dev", "run-dev", + "eval-dev", "clean" )] [string]$Command @@ -33,6 +36,7 @@ $MODEL_SMOKE_REL = "model_smoke" $OUT_FULL_REL = "outputs" $OUT_SMOKE_REL = "outputs_smoke" +$DEMOGRAPHICS_FILE = "demographics.csv" # ============================================ # FUNCIONES AUXILIARES @@ -48,6 +52,24 @@ function Ensure-Directory($path) { } } +function Invoke-Evaluation($DataPath, $OutputPath, $Label) { + Write-Host "Evaluating $Label predictions..." + docker run --rm ` + -v "${DataPath}:/challenge/eval_data:ro" ` + -v "${OutputPath}:/challenge/eval_outputs:ro" ` + $IMAGE_NAME ` + python evaluate_model.py -d "/challenge/eval_data/$DEMOGRAPHICS_FILE" -o "/challenge/eval_outputs/$DEMOGRAPHICS_FILE" +} + +function Invoke-EvaluationDev($CodePath, $DataPath, $OutputPath, $Label) { + Write-Host "Evaluating $Label predictions..." + docker run --rm ` + -v "${CodePath}:/challenge" ` + -v "${DataPath}:/challenge/eval_data:ro" ` + $IMAGE_NAME ` + python evaluate_model.py -d "/challenge/eval_data/$DEMOGRAPHICS_FILE" -o "$OutputPath/$DEMOGRAPHICS_FILE" +} + # ============================================ # COMANDOS # ============================================ @@ -103,6 +125,8 @@ function Run-Full { -v "${OUT_FULL}:/challenge/holdout_outputs" ` $IMAGE_NAME ` python run_model.py -d holdout_data -m model -o holdout_outputs -v + + Invoke-Evaluation $FULL_DATA $OUT_FULL "full-dataset" } function Run-Smoke { @@ -119,6 +143,24 @@ function Run-Smoke { -v "${OUT_SMOKE}:/challenge/holdout_outputs" ` $IMAGE_NAME ` python run_model.py -d holdout_data -m model -o holdout_outputs -v + + Invoke-Evaluation $SMOKE_DATA $OUT_SMOKE "smoke" +} + +function Eval-Full { + + $FULL_DATA = Get-AbsolutePath $FULL_DATA_REL + $OUT_FULL = Get-AbsolutePath $OUT_FULL_REL + + Invoke-Evaluation $FULL_DATA $OUT_FULL "full-dataset" +} + +function Eval-Smoke { + + $SMOKE_DATA = Get-AbsolutePath $SMOKE_DATA_REL + $OUT_SMOKE = Get-AbsolutePath $OUT_SMOKE_REL + + Invoke-Evaluation $SMOKE_DATA $OUT_SMOKE "smoke" } # ====================== @@ -157,6 +199,16 @@ function Run-Dev { -v "${OUT_SMOKE}:/challenge/holdout_outputs" ` $IMAGE_NAME ` python run_model.py -d holdout_data -m model -o holdout_outputs -v + + Invoke-EvaluationDev $CODE_PATH $SMOKE_DATA "/challenge/holdout_outputs" "development smoke" +} + +function Eval-Dev { + + $CODE_PATH = Get-AbsolutePath "." + $SMOKE_DATA = Get-AbsolutePath $SMOKE_DATA_REL + + Invoke-EvaluationDev $CODE_PATH $SMOKE_DATA "/challenge/holdout_outputs" "development smoke" } function Clean-All { @@ -181,8 +233,11 @@ switch ($Command) { "train-smoke" { Train-Smoke } "run" { Run-Full } "run-smoke" { Run-Smoke } + "eval" { Eval-Full } + "eval-smoke" { Eval-Smoke } "train-dev" { Train-Dev } "run-dev" { Run-Dev } + "eval-dev" { Eval-Dev } "clean" { Clean-All } } \ No newline at end of file diff --git a/run.sh b/run.sh index fed6e06..df73527 100644 --- a/run.sh +++ b/run.sh @@ -2,7 +2,7 @@ set -euo pipefail if [[ $# -lt 1 ]]; then - echo "Usage: $0 " + echo "Usage: $0 " exit 1 fi @@ -22,6 +22,7 @@ MODEL_SMOKE_REL="model_smoke" OUT_FULL_REL="outputs" OUT_SMOKE_REL="outputs_smoke" +DEMOGRAPHICS_FILE="demographics.csv" # ============================================ # HELPERS @@ -51,6 +52,45 @@ docker_cli() { MSYS_NO_PATHCONV=1 MSYS2_ARG_CONV_EXCL="*" docker "$@" } +evaluate_predictions() { + local data_dir="$1" + local output_dir="$2" + local label="$3" + local data_dir_docker output_dir_docker + + data_dir_docker="$(to_docker_path "$data_dir")" + output_dir_docker="$(to_docker_path "$output_dir")" + + echo "Evaluating ${label} predictions..." + docker_cli run --rm \ + -v "${data_dir_docker}:/challenge/eval_data:ro" \ + -v "${output_dir_docker}:/challenge/eval_outputs:ro" \ + "$IMAGE_NAME" \ + python evaluate_model.py \ + -d "/challenge/eval_data/${DEMOGRAPHICS_FILE}" \ + -o "/challenge/eval_outputs/${DEMOGRAPHICS_FILE}" +} + +evaluate_predictions_dev() { + local code_path="$1" + local data_path="$2" + local output_path="$3" + local label="$4" + local code_path_docker data_path_docker + + code_path_docker="$(to_docker_path "$code_path")" + data_path_docker="$(to_docker_path "$data_path")" + + echo "Evaluating ${label} predictions..." + docker_cli run --rm \ + -v "${code_path_docker}:/challenge" \ + -v "${data_path_docker}:/challenge/eval_data:ro" \ + "$IMAGE_NAME" \ + python evaluate_model.py \ + -d "/challenge/eval_data/${DEMOGRAPHICS_FILE}" \ + -o "$output_path/${DEMOGRAPHICS_FILE}" +} + build_image() { docker_cli build -t "$IMAGE_NAME" . } @@ -115,6 +155,8 @@ run_full() { -v "${out_full_docker}:/challenge/holdout_outputs" \ "$IMAGE_NAME" \ python run_model.py -d holdout_data -m model -o holdout_outputs -v + + evaluate_predictions "$full_data" "$out_full" "full-dataset" } run_smoke() { @@ -136,6 +178,26 @@ run_smoke() { -v "${out_smoke_docker}:/challenge/holdout_outputs" \ "$IMAGE_NAME" \ python run_model.py -d holdout_data -m model -o holdout_outputs -v + + evaluate_predictions "$smoke_data" "$out_smoke" "smoke" +} + +eval_full() { + local full_data out_full + + full_data="$(get_absolute_path "$FULL_DATA_REL")" + out_full="$(get_absolute_path "$OUT_FULL_REL")" + + evaluate_predictions "$full_data" "$out_full" "full-dataset" +} + +eval_smoke() { + local smoke_data out_smoke + + smoke_data="$(get_absolute_path "$SMOKE_DATA_REL")" + out_smoke="$(get_absolute_path "$OUT_SMOKE_REL")" + + evaluate_predictions "$smoke_data" "$out_smoke" "smoke" } # ===================== @@ -178,6 +240,17 @@ run_dev() { -v "${smoke_data_docker}:/challenge/data_smoke:ro" \ "$IMAGE_NAME" \ python run_model.py -d /challenge/data_smoke -m /challenge/model_smoke -o /challenge/outputs_smoke -v + + evaluate_predictions_dev "$code_path" "$smoke_data" "/challenge/outputs_smoke" "development smoke" +} + +eval_dev() { + local code_path smoke_data + + code_path="$(get_absolute_path ".")" + smoke_data="$(get_absolute_path "$SMOKE_DATA_REL")" + + evaluate_predictions_dev "$code_path" "$smoke_data" "/challenge/outputs_smoke" "development smoke" } clean_all() { @@ -192,12 +265,15 @@ case "$COMMAND" in train-smoke) train_smoke ;; run) run_full ;; run-smoke) run_smoke ;; + eval) eval_full ;; + eval-smoke) eval_smoke ;; train-dev) train_dev ;; run-dev) run_dev ;; + eval-dev) eval_dev ;; clean) clean_all ;; *) echo "Invalid command: $COMMAND" - echo "Valid commands: build, smoke, train, train-smoke, run, run-smoke, train-dev, run-dev, clean" + echo "Valid commands: build, smoke, train, train-smoke, run, run-smoke, eval, eval-smoke, train-dev, run-dev, eval-dev, clean" exit 1 ;; esac From 03fc272d88c6b4b171894593d353ccf74b1363b0 Mon Sep 17 00:00:00 2001 From: dcajal Date: Fri, 27 Mar 2026 11:00:43 +0100 Subject: [PATCH 21/93] Refactor dataset handling in run scripts to support supplementary data and improve evaluation logic --- docs/04_run_script.md | 8 ++++++-- run.ps1 | 37 ++++++++++++++++++++++++++++--------- run.sh | 38 +++++++++++++++++++++++++++----------- 3 files changed, 61 insertions(+), 22 deletions(-) diff --git a/docs/04_run_script.md b/docs/04_run_script.md index e5f5907..310bb5c 100644 --- a/docs/04_run_script.md +++ b/docs/04_run_script.md @@ -12,9 +12,10 @@ Aquí se define el orden recomendado y los comandos asociados. ``` data/training_set/ +data/supplementary_set/ ``` -⚠️ Si el dataset está en otra ubicación, modificar la variable `$FULL_DATA_REL` +⚠️ Si el dataset está en otra ubicación, modificar las variables `$TRAIN_DATA_REL` y `$RUN_DATA_REL` dentro de `run.sh`. ⚠️ Ejecutar los comandos desde Git Bash. @@ -98,7 +99,8 @@ Guarda el modelo en `model/`. ./run.sh run ``` -Genera resultados en `outputs/` y luego imprime métricas de evaluación en consola. +Genera resultados en `outputs/` usando `data/supplementary_set/`. +Si el dataset no tiene etiquetas (como en `supplementary_set`), el script omite la evaluación automáticamente. ### Evaluar predicciones existentes completas @@ -107,6 +109,8 @@ Genera resultados en `outputs/` y luego imprime métricas de evaluación en cons ``` Reutiliza `outputs/demographics.csv` y muestra AUROC, AUPRC, Accuracy y F-measure sin volver a ejecutar inferencia. +Evalúa contra `data/supplementary_set/`. +Si no hay etiquetas en ese set, el script omite la evaluación automáticamente. ### Evaluar predicciones existentes del dataset smoke diff --git a/run.ps1 b/run.ps1 index 439eade..b480a55 100644 --- a/run.ps1 +++ b/run.ps1 @@ -23,10 +23,11 @@ param( # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> # IMPORTANTE: -# Si tu dataset no está en data/training_set, -# modifica esta ruta. +# Si tu dataset no está en data/training_set o data/supplementary_set, +# modifica estas rutas. # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> -$FULL_DATA_REL = "data/training_set" +$TRAIN_DATA_REL = "data/training_set" +$RUN_DATA_REL = "data/supplementary_set" $SMOKE_DATA_REL = "data/training_smoke" $IMAGE_NAME = "cinc2026" @@ -70,6 +71,16 @@ function Invoke-EvaluationDev($CodePath, $DataPath, $OutputPath, $Label) { python evaluate_model.py -d "/challenge/eval_data/$DEMOGRAPHICS_FILE" -o "$OutputPath/$DEMOGRAPHICS_FILE" } +function Test-DatasetHasLabels($DataPath) { + $demographicsPath = Join-Path $DataPath $DEMOGRAPHICS_FILE + if (!(Test-Path $demographicsPath)) { + return $false + } + + $header = Get-Content -Path $demographicsPath -TotalCount 1 + return $header -match "Cognitive_Impairment" +} + # ============================================ # COMANDOS # ============================================ @@ -85,7 +96,7 @@ function Create-Smoke { function Train-Full { - $FULL_DATA = Get-AbsolutePath $FULL_DATA_REL + $FULL_DATA = Get-AbsolutePath $TRAIN_DATA_REL $MODEL_FULL = Join-Path (Get-AbsolutePath ".") $MODEL_FULL_REL Ensure-Directory $MODEL_FULL @@ -113,20 +124,24 @@ function Train-Smoke { function Run-Full { - $FULL_DATA = Get-AbsolutePath $FULL_DATA_REL + $RUN_DATA = Get-AbsolutePath $RUN_DATA_REL $MODEL_FULL = Get-AbsolutePath $MODEL_FULL_REL $OUT_FULL = Join-Path (Get-AbsolutePath ".") $OUT_FULL_REL Ensure-Directory $OUT_FULL docker run --rm ` - -v "${FULL_DATA}:/challenge/holdout_data:ro" ` + -v "${RUN_DATA}:/challenge/holdout_data:ro" ` -v "${MODEL_FULL}:/challenge/model:ro" ` -v "${OUT_FULL}:/challenge/holdout_outputs" ` $IMAGE_NAME ` python run_model.py -d holdout_data -m model -o holdout_outputs -v - Invoke-Evaluation $FULL_DATA $OUT_FULL "full-dataset" + if (Test-DatasetHasLabels $RUN_DATA) { + Invoke-Evaluation $RUN_DATA $OUT_FULL "run-dataset" + } else { + Write-Host "Skipping evaluation for run dataset (labels not present in $RUN_DATA_REL/$DEMOGRAPHICS_FILE)." + } } function Run-Smoke { @@ -149,10 +164,14 @@ function Run-Smoke { function Eval-Full { - $FULL_DATA = Get-AbsolutePath $FULL_DATA_REL + $RUN_DATA = Get-AbsolutePath $RUN_DATA_REL $OUT_FULL = Get-AbsolutePath $OUT_FULL_REL - Invoke-Evaluation $FULL_DATA $OUT_FULL "full-dataset" + if (Test-DatasetHasLabels $RUN_DATA) { + Invoke-Evaluation $RUN_DATA $OUT_FULL "run-dataset" + } else { + Write-Host "Skipping evaluation for run dataset (labels not present in $RUN_DATA_REL/$DEMOGRAPHICS_FILE)." + } } function Eval-Smoke { diff --git a/run.sh b/run.sh index df73527..18b7e1e 100644 --- a/run.sh +++ b/run.sh @@ -12,7 +12,8 @@ COMMAND="$1" # CONFIGURATION # ============================================ -FULL_DATA_REL="data/training_set" +TRAIN_DATA_REL="data/training_set" +RUN_DATA_REL="data/supplementary_set" SMOKE_DATA_REL="data/training_smoke" IMAGE_NAME="cinc2026" @@ -91,6 +92,13 @@ evaluate_predictions_dev() { -o "$output_path/${DEMOGRAPHICS_FILE}" } +dataset_has_labels() { + local data_dir="$1" + local demographics_path="$data_dir/$DEMOGRAPHICS_FILE" + + [[ -f "$demographics_path" ]] && head -n 1 "$demographics_path" | grep -q "Cognitive_Impairment" +} + build_image() { docker_cli build -t "$IMAGE_NAME" . } @@ -104,7 +112,7 @@ train_full() { local full_data model_full local full_data_docker model_full_docker - full_data="$(get_absolute_path "$FULL_DATA_REL")" + full_data="$(get_absolute_path "$TRAIN_DATA_REL")" model_full="$(get_absolute_path ".")/${MODEL_FULL_REL}" full_data_docker="$(to_docker_path "$full_data")" model_full_docker="$(to_docker_path "$model_full")" @@ -137,26 +145,30 @@ train_smoke() { } run_full() { - local full_data model_full out_full - local full_data_docker model_full_docker out_full_docker + local run_data model_full out_full + local run_data_docker model_full_docker out_full_docker - full_data="$(get_absolute_path "$FULL_DATA_REL")" + run_data="$(get_absolute_path "$RUN_DATA_REL")" model_full="$(get_absolute_path "$MODEL_FULL_REL")" out_full="$(get_absolute_path ".")/${OUT_FULL_REL}" - full_data_docker="$(to_docker_path "$full_data")" + run_data_docker="$(to_docker_path "$run_data")" model_full_docker="$(to_docker_path "$model_full")" out_full_docker="$(to_docker_path "$out_full")" ensure_directory "$out_full" docker_cli run --rm \ - -v "${full_data_docker}:/challenge/holdout_data:ro" \ + -v "${run_data_docker}:/challenge/holdout_data:ro" \ -v "${model_full_docker}:/challenge/model:ro" \ -v "${out_full_docker}:/challenge/holdout_outputs" \ "$IMAGE_NAME" \ python run_model.py -d holdout_data -m model -o holdout_outputs -v - evaluate_predictions "$full_data" "$out_full" "full-dataset" + if dataset_has_labels "$run_data"; then + evaluate_predictions "$run_data" "$out_full" "run-dataset" + else + echo "Skipping evaluation for run dataset (labels not present in ${RUN_DATA_REL}/${DEMOGRAPHICS_FILE})." + fi } run_smoke() { @@ -183,12 +195,16 @@ run_smoke() { } eval_full() { - local full_data out_full + local run_data out_full - full_data="$(get_absolute_path "$FULL_DATA_REL")" + run_data="$(get_absolute_path "$RUN_DATA_REL")" out_full="$(get_absolute_path "$OUT_FULL_REL")" - evaluate_predictions "$full_data" "$out_full" "full-dataset" + if dataset_has_labels "$run_data"; then + evaluate_predictions "$run_data" "$out_full" "run-dataset" + else + echo "Skipping evaluation for run dataset (labels not present in ${RUN_DATA_REL}/${DEMOGRAPHICS_FILE})." + fi } eval_smoke() { From 922c1093ec97aad29dd6afbccad9e808848cd434 Mon Sep 17 00:00:00 2001 From: dcajal Date: Fri, 27 Mar 2026 12:52:56 +0100 Subject: [PATCH 22/93] Update run scripts to replace references from supplementary_set to test_set --- docs/04_run_script.md | 6 +++--- run.ps1 | 4 ++-- run.sh | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/04_run_script.md b/docs/04_run_script.md index 310bb5c..87eef6d 100644 --- a/docs/04_run_script.md +++ b/docs/04_run_script.md @@ -99,8 +99,8 @@ Guarda el modelo en `model/`. ./run.sh run ``` -Genera resultados en `outputs/` usando `data/supplementary_set/`. -Si el dataset no tiene etiquetas (como en `supplementary_set`), el script omite la evaluación automáticamente. +Genera resultados en `outputs/` usando `data/test_set/`. +Si el dataset no tiene etiquetas (como en `test_set`), el script omite la evaluación automáticamente. ### Evaluar predicciones existentes completas @@ -109,7 +109,7 @@ Si el dataset no tiene etiquetas (como en `supplementary_set`), el script omite ``` Reutiliza `outputs/demographics.csv` y muestra AUROC, AUPRC, Accuracy y F-measure sin volver a ejecutar inferencia. -Evalúa contra `data/supplementary_set/`. +Evalúa contra `data/test_set/`. Si no hay etiquetas en ese set, el script omite la evaluación automáticamente. ### Evaluar predicciones existentes del dataset smoke diff --git a/run.ps1 b/run.ps1 index b480a55..8d2141b 100644 --- a/run.ps1 +++ b/run.ps1 @@ -23,11 +23,11 @@ param( # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> # IMPORTANTE: -# Si tu dataset no está en data/training_set o data/supplementary_set, +# Si tu dataset no está en data/training_set o data/test_set, # modifica estas rutas. # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> $TRAIN_DATA_REL = "data/training_set" -$RUN_DATA_REL = "data/supplementary_set" +$RUN_DATA_REL = "data/test_set" $SMOKE_DATA_REL = "data/training_smoke" $IMAGE_NAME = "cinc2026" diff --git a/run.sh b/run.sh index 18b7e1e..dcabdbf 100644 --- a/run.sh +++ b/run.sh @@ -13,7 +13,7 @@ COMMAND="$1" # ============================================ TRAIN_DATA_REL="data/training_set" -RUN_DATA_REL="data/supplementary_set" +RUN_DATA_REL="data/test_set" SMOKE_DATA_REL="data/training_smoke" IMAGE_NAME="cinc2026" From ebe970c1d37eb5f530d993fa901cd4f1f17ac919 Mon Sep 17 00:00:00 2001 From: dcajal Date: Fri, 27 Mar 2026 13:08:56 +0100 Subject: [PATCH 23/93] Add optimization tracking documentation and improve training performance with caching and parallel processing --- docs/05_optimization_tracking.md | 128 ++++++++++++++++++++++++++ team_code.py | 153 +++++++++++++++++++------------ 2 files changed, 224 insertions(+), 57 deletions(-) create mode 100644 docs/05_optimization_tracking.md diff --git a/docs/05_optimization_tracking.md b/docs/05_optimization_tracking.md new file mode 100644 index 0000000..db352fa --- /dev/null +++ b/docs/05_optimization_tracking.md @@ -0,0 +1,128 @@ +# Seguimiento De Optimizaciones + +Este documento registra las optimizaciones de tiempo de entrenamiento aplicadas sobre el código de ejemplo original del PhysioNet Challenge en `team_code.py`. + +## Objetivo + +Mantener un registro claro de los cambios respecto a la base proporcionada por la organización para que el equipo pueda: + +- entender qué optimizaciones se probaron; +- medir su efecto sobre el flujo smoke; +- identificar qué cambios merece la pena conservar; +- revertir cambios concretos si la submission se comporta distinto en el entorno del Challenge. + +## Línea Base + +- Fuente de la línea base: implementación de ejemplo proporcionada por la organización en `team_code.py`. +- Tiempo observado de entrenamiento smoke con `./run.sh train-dev`: alrededor de 22 segundos. +- Comportamiento base: extracción secuencial de features, lecturas repetidas de CSV, recarga repetida de reglas de renombrado de canales y carga no utilizada de anotaciones humanas durante entrenamiento. + +## Cambios Aplicados + +### 1. Eliminación de la carga no utilizada de anotaciones humanas en entrenamiento + +Cambio: + +- Se eliminó la carga de `human_annotations` dentro de `train_model`. +- Se dejó intacta la función auxiliar `extract_human_annotations_features`. + +Motivo: + +- El vector final de entrenamiento solo concatenaba features demográficas, fisiológicas y algorítmicas. +- Las features de anotaciones humanas se calculaban, pero nunca se incluían en el `np.hstack(...)` que se pasaba al clasificador. + +Efecto observado: + +- El tiempo de entrenamiento smoke pasó de unos 22.0 s a 21.891 s. +- Conclusión: la limpieza es correcta a nivel lógico, pero su impacto en tiempo es despreciable en el dataset smoke. + +Riesgo: + +- Bajo. Solo elimina trabajo muerto. + +### 2. Caché de reglas de renombrado de canales + +Cambio: + +- Se añadió una caché en proceso para las reglas de renombrado cargadas desde `channel_table.csv`. +- Se sustituyeron las llamadas repetidas a `load_rename_rules(os.path.abspath(csv_path))` por una consulta a la caché. + +Motivo: + +- `extract_physiological_features` estaba cargando y parseando el mismo CSV para cada registro. + +Efecto observado: + +- El tiempo smoke medido en la siguiente ejecución fue 22.040 s. +- Conclusión: la optimización es correcta, pero no ataca un cuello de botella relevante en smoke. + +Riesgo: + +- Bajo. El comportamiento no cambia salvo por reutilizar reglas ya parseadas. + +### 3. Caché de demographics y etiquetas para entrenamiento + +Cambio: + +- Se añadió una lectura única de `demographics.csv` al inicio de `train_model`. +- Se construyeron: + - una caché de demographics indexada por `(patient_id, session_id)`; + - una caché de diagnósticos indexada por `patient_id`. +- Se reemplazaron las llamadas por registro a `load_demographics(...)` y `load_diagnoses(...)` durante entrenamiento. + +Motivo: + +- El bucle original de entrenamiento releía el mismo CSV para cada registro. + +Efecto observado: + +- El tiempo smoke bajó a 20.837 s. +- Conclusión: es una mejora real, aunque moderada. + +Riesgo: + +- Bajo a medio. +- Asume que las etiquetas de entrenamiento son estables a nivel de paciente cuando se cachean por `patient_id`, igual que hacía el comportamiento original de `load_diagnoses(...)`. + +### 4. Paralelización de la extracción de features en entrenamiento + +Cambio: + +- Se añadió procesamiento paralelo por registro con `ThreadPoolExecutor` dentro de `train_model`. +- Se movió la lógica de extracción por registro a `process_training_record(...)`. +- Se limitó el número de workers con: + +```python +MAX_TRAIN_WORKERS = max(1, min(4, os.cpu_count() or 1)) +``` + +Motivo: + +- Cada registro de entrenamiento se procesa de forma independiente. +- El pipeline mezcla lecturas de archivos EDF y trabajo con NumPy, así que un pool pequeño de hilos puede reducir el tiempo total. + +Efecto observado: + +- El tiempo smoke bajó a 9.578 s en la primera ejecución tras paralelizar. +- Las ejecuciones de seguimiento midieron 9.762 s y 9.655 s. +- Conclusión: esta es la optimización dominante. + +Riesgo: + +- Medio. +- El acceso paralelo a archivos puede comportarse distinto en discos más lentos o en una infraestructura más limitada del Challenge. + +## Plan De Rollback + +Si la submission se comporta distinto en el entorno del Challenge, revertir en este orden: + +1. Eliminar la extracción con hilos y restaurar el bucle secuencial original en `train_model`. +2. Eliminar las cachés de metadata de entrenamiento y volver a `load_demographics(...)` / `load_diagnoses(...)`. +3. Eliminar la caché de reglas de renombrado y volver a las llamadas directas a `load_rename_rules(...)`. +4. Rehabilitar la carga de anotaciones humanas solo si el vector de entrenamiento se modifica explícitamente para usar esas features. + +Este orden de rollback elimina primero la optimización de mayor riesgo y deja para el final los cambios de comportamiento más pequeños. + +## Archivos Modificados + +- `team_code.py` \ No newline at end of file diff --git a/team_code.py b/team_code.py index 2b18023..2185a52 100644 --- a/team_code.py +++ b/team_code.py @@ -14,7 +14,9 @@ import os import atexit import builtins +import pandas as pd import re +from concurrent.futures import ThreadPoolExecutor from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor import sys from tqdm import tqdm @@ -37,6 +39,77 @@ ORIGINAL_PRINT = builtins.print PRINT_FILTER_ACTIVE = False RUN_PROGRESS_LINE_RE = re.compile(r'^-\s+\d+/\d+:\s') +RENAME_RULES_CACHE = {} +MAX_TRAIN_WORKERS = max(1, min(4, os.cpu_count() or 1)) + + +def build_training_metadata_cache(patient_data_file): + metadata = pd.read_csv(patient_data_file) + demographics_cache = {} + diagnosis_cache = {} + + for row in metadata.to_dict('records'): + patient_id = row[HEADERS['bids_folder']] + session_id = row[HEADERS['session_id']] + demographics_cache[(patient_id, session_id)] = row + diagnosis_cache[patient_id] = load_label(row) + + return demographics_cache, diagnosis_cache + + +def get_rename_rules(csv_path): + normalized_csv_path = os.path.abspath(csv_path) + rename_rules = RENAME_RULES_CACHE.get(normalized_csv_path) + if rename_rules is None: + rename_rules = load_rename_rules(normalized_csv_path) + RENAME_RULES_CACHE[normalized_csv_path] = rename_rules + return rename_rules + + +def process_training_record(record, data_folder, demographics_cache, diagnosis_cache, csv_path): + patient_id = record[HEADERS['bids_folder']] + site_id = record[HEADERS['site_id']] + session_id = record[HEADERS['session_id']] + + try: + patient_data = demographics_cache.get((patient_id, session_id), {}) + demographic_features = extract_demographic_features(patient_data) + + physiological_data_file = os.path.join( + data_folder, + PHYSIOLOGICAL_DATA_SUBFOLDER, + site_id, + f"{patient_id}_ses-{session_id}.edf" + ) + if not os.path.exists(physiological_data_file): + return patient_id, None, None, f"Missing physiological data for {patient_id}. Skipping..." + + physiological_data, physiological_fs = load_signal_data(physiological_data_file) + physiological_features = extract_physiological_features( + physiological_data, + physiological_fs, + csv_path=csv_path + ) + + algorithmic_annotations_file = os.path.join( + data_folder, + ALGORITHMIC_ANNOTATIONS_SUBFOLDER, + site_id, + f"{patient_id}_ses-{session_id}_caisr_annotations.edf" + ) + algorithmic_annotations, algorithmic_fs = load_signal_data(algorithmic_annotations_file) + algorithmic_features = extract_algorithmic_annotations_features(algorithmic_annotations) + + label = diagnosis_cache.get(patient_id) + + if label == 0 or label == 1: + feature_vector = np.hstack([demographic_features, physiological_features, algorithmic_features]) + return patient_id, feature_vector, label, None + + return patient_id, None, None, f"Invalid label for {patient_id}. Skipping..." + + except Exception as e: + return patient_id, None, None, f"Error processing {patient_id}: {e}" def _close_run_model_pbar(): @@ -89,6 +162,7 @@ def train_model(data_folder, model_folder, verbose, csv_path=DEFAULT_CSV_PATH): patient_data_file = os.path.join(data_folder, DEMOGRAPHICS_FILE) patient_metadata_list = find_patients(patient_data_file) + demographics_cache, diagnosis_cache = build_training_metadata_cache(patient_data_file) num_records = len(patient_metadata_list) if num_records == 0: @@ -98,69 +172,34 @@ def train_model(data_folder, model_folder, verbose, csv_path=DEFAULT_CSV_PATH): if verbose: print('Extracting features and labels from the data...') - # Iterate over the records to extract the features and labels. features = list() labels = list() - - pbar = tqdm(range(num_records), desc="Extracting Features", unit="record", disable=not verbose) - for i in pbar: - try: - # Extract identifiers for this specific record - record = patient_metadata_list[i] - patient_id = record[HEADERS['bids_folder']] - site_id = record[HEADERS['site_id']] - session_id = record[HEADERS['session_id']] + with ThreadPoolExecutor(max_workers=MAX_TRAIN_WORKERS) as executor: + results = executor.map( + lambda record: process_training_record( + record, + data_folder, + demographics_cache, + diagnosis_cache, + csv_path + ), + patient_metadata_list + ) + + pbar = tqdm(results, total=num_records, desc="Extracting Features", unit="record", disable=not verbose) + for patient_id, feature_vector, label, message in pbar: if verbose: pbar.set_postfix({"patient": patient_id}) - # Load the patient data. - patient_data_file = os.path.join(data_folder, DEMOGRAPHICS_FILE) - patient_data = load_demographics(patient_data_file, patient_id, session_id) - demographic_features = extract_demographic_features(patient_data) - - # Load signal data. - - # Load the physiological signal. - physiological_data_file = os.path.join(data_folder, PHYSIOLOGICAL_DATA_SUBFOLDER, site_id, f"{patient_id}_ses-{session_id}.edf") - # --- Check if the file actually exists before proceeding --- - if not os.path.exists(physiological_data_file): - if verbose: - print(f" ! Missing physiological data for {patient_id}. Skipping...") - continue # skip record - physiological_data, physiological_fs = load_signal_data(physiological_data_file) - physiological_features = extract_physiological_features(physiological_data, physiological_fs, csv_path=csv_path) # This function can rename, re-reference, resample, etc. the signal data. - - # Load the algorithmic annotations. - algorithmic_annotations_file = os.path.join(data_folder, ALGORITHMIC_ANNOTATIONS_SUBFOLDER, site_id, f"{patient_id}_ses-{session_id}_caisr_annotations.edf") - algorithmic_annotations, algorithmic_fs = load_signal_data(algorithmic_annotations_file) - algorithmic_features = extract_algorithmic_annotations_features(algorithmic_annotations) - - # Load the human annotations; these data will not be available in the hidden validation and test sets. - human_annotations_file = os.path.join(data_folder, HUMAN_ANNOTATIONS_SUBFOLDER, site_id, f"{patient_id}_ses-{session_id}_expert_annotations.edf") - human_annotations, human_fs = load_signal_data(human_annotations_file) - human_features = extract_human_annotations_features(human_annotations) - - # Load the diagnoses; these data will not be available in the hidden validation and test sets. - diagnosis_file = os.path.join(data_folder, DEMOGRAPHICS_FILE) - label = load_diagnoses(diagnosis_file, patient_id) - - # Store the features and labels, but - # the human annotations are not available on the hidden validation and test sets, but you - # may want to consider how to use them for training. - if label == 0 or label == 1: - features.append(np.hstack([demographic_features, physiological_features, algorithmic_features])) - labels.append(label) - - if 'physiological_data' in locals(): del physiological_data - if 'algorithmic_annotations' in locals(): del algorithmic_annotations - - except Exception as e: - # If an error occurs (e.g., a record is corrupted), log it and move to the next - tqdm.write(f" !!! Error processing record {i+1} ({patient_id}): {e}") - continue + if message is not None: + tqdm.write(f" ! {message}") + continue + + features.append(feature_vector) + labels.append(label) - pbar.close() + pbar.close() features = np.asarray(features, dtype=np.float32) labels = np.asarray(labels, dtype=bool) @@ -332,7 +371,7 @@ def extract_physiological_features(physiological_data, physiological_fs, csv_pat # Step 1: Load rules and standardize names # Note: Use script-relative path or absolute path for robustness - rename_rules = load_rename_rules(os.path.abspath(csv_path)) + rename_rules = get_rename_rules(csv_path) rename_map, cols_to_drop = standardize_channel_names_rename_only(original_labels, rename_rules) # Step 2: Apply renaming to BOTH signals and their corresponding FS From 8f121b784bca971fa6b8d105d7c9aa3cd526e817 Mon Sep 17 00:00:00 2001 From: rolopu1 <718694@unizar.es> Date: Fri, 27 Mar 2026 13:52:07 +0100 Subject: [PATCH 24/93] Refactor respiratory processing to improve signal handling and feature extraction, including updates to sampling frequency and NaN handling. --- src/eeg_processing.py | 226 +++++++++++++++---------------------- src/lib/Resp_features.py | 92 ++++++++++++++- src/resp_processing.py | 236 +++++++++++++++++++++------------------ 3 files changed, 306 insertions(+), 248 deletions(-) diff --git a/src/eeg_processing.py b/src/eeg_processing.py index 127a32e..760b474 100644 --- a/src/eeg_processing.py +++ b/src/eeg_processing.py @@ -29,161 +29,113 @@ El módulo depende de `numpy`, `pandas`, `matplotlib`, `plotly` y de las utilidades definidas en `lib/helper_code` y `lib/EEG_functions`. """ - -import numpy as np -import pandas as pd -import sys +import sys import os -import matplotlib.pyplot as plt -import plotly.express as px -import plotly.graph_objects as go -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -import lib.helper_code as helper_code +import pandas as pd +import numpy as np +import helper_code as helper_code import lib.EEG_functions as EEG_functions -def MetricasHospitlal(hospital): - - print(f"Procesando hospital: {hospital}") - if hospital == 'I0002' or hospital == 'I0006' or hospital == "S0001": - datapath = 'data/training_set/Physiological_data/'+hospital - else: - datapath = 'data/supplementary_set/Physiological_data/'+hospital - - channels = pd.read_csv("notebooks/channel_table.csv") - selectEEG = channels[channels['Category'].isin(['eeg'])] +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) - demographics = pd.read_csv(os.path.join('C:/BSICoS/CincChallenge2026/CincChallenge_2026/data/training_set', "demographics.csv")) +def processEEG(physiological_data, physiological_fs, csv_path): - # Datos = pd.DataFrame(columns=['File', 'Channel', 'Sampling_Frequency', 'Duration_sec']) - lista_dir = os.listdir(datapath) - results = [] + channels = pd.read_csv(csv_path) + selectEEG = channels[channels['Category'].isin(['eeg'])] - for file in lista_dir: - # Cargar el archivo (sustituye por tu ruta real) - edf = helper_code.edfio.read_edf(os.path.join(datapath, file)) + for label in original_labels: + fs = physiological_fs[label] - id = file[9:-10] # Asumiendo que el ID es el nombre del archivo sin la extensión - - selEEG = [] - labels = [] data = [] + original_labels = list(physiological_data.keys()) # Listar canales para identificar los de interés (ej: C3-M2, O1-M2) HayEEG = False - for i, sig in enumerate(edf.signals): - # print(f"[{i}] {sig.label}") - # print length fs and duration - # print(f"Length: {len(sig.data)}, Sampling Frequency: {sig.sampling_frequency} Hz, Duration: {len(sig.data)/sig.sampling_frequency:.2f} seconds") + for i, label in enumerate(original_labels): for index in selectEEG.index: - if sig.label.lower() in selectEEG['Channel_Names'][index].lower(): - print(f"Canal seleccionado: {sig.label}") - selEEG.append([i,sig]) - labels.append(sig.label) + if label.lower() in selectEEG['Channel_Names'][index].lower(): + print(f"Canal seleccionado: {label}") + labels.append(label) HayEEG = True break - # for i in range(len(edf.signals)): - # print(f"Longitud: {edf.signals[i].data.shape}, Canal: {edf.signals[i].label}, Frecuencia de muestreo: {edf.signals[i].sampling_frequency} Hz, Duración: {len(edf.signals[i].data)/edf.signals[i].sampling_frequency:.2f} segundos") + + results = [] + labels2 = [] + if HayEEG: + Bipolar = pd.DataFrame() + if all(label in labels for label in ["F3", "F4", "M1", "M2"]): + Bipolar['F3-M2'] = physiological_data["F3"] - physiological_data["M2"] + Bipolar['F4-M1'] = physiological_data["F4"] - physiological_data["M1"] + labels2.append('F3-M2') + labels2.append('F4-M1') + if all(label in labels for label in ["C3", "C4", "M1", "M2"]): + Bipolar['C3-M2'] = physiological_data["C3"] - physiological_data["M2"] + Bipolar['C4-M1'] = physiological_data["C4"] - physiological_data["M1"] + labels2.append('C3-M2') + labels2.append('C4-M1') + if all(label in labels for label in ["O2", "O1", "M1", "M2"]): + Bipolar['O2-M2'] = physiological_data["O1"] - physiological_data["M2"] + Bipolar['O1-M1'] = physiological_data["O2"] - physiological_data["M1"] + labels2.append('O1-M1') + labels2.append('O2-M2') + # print(f"Archivo {file} tiene ECG, RESP y EEG. Se procesará con canales bipolares.") - if HayEEG: - - Bipolar = pd.DataFrame() - if all(label in labels for label in ["F3", "F4", "M1", "M2"]): - Bipolar['F3-M2'] = edf.signals[edf.labels.index("F3")].data - edf.signals[edf.labels.index("M2")].data - Bipolar['F4-M1'] = edf.signals[edf.labels.index("F4")].data - edf.signals[edf.labels.index("M1")].data - labels2 = ['F3-M2', 'F4-M1'] - if all(label in labels for label in ["C3", "C4", "M1", "M2"]): - Bipolar['C3-M2'] = edf.signals[edf.labels.index("C3")].data - edf.signals[edf.labels.index("M2")].data - Bipolar['C4-M1'] = edf.signals[edf.labels.index("C4")].data - edf.signals[edf.labels.index("M1")].data - labels2.append('C3-M2') - labels2.append('C4-M1') - if all(label in labels for label in ["O2", "O1", "M1", "M2"]): - Bipolar['O2-M2'] = edf.signals[edf.labels.index("O1")].data - edf.signals[edf.labels.index("M2")].data - Bipolar['O1-M1'] = edf.signals[edf.labels.index("O2")].data - edf.signals[edf.labels.index("M1")].data - labels2.append('O1-M1') - labels2.append('O2-M2') - # print(f"Archivo {file} tiene ECG, RESP y EEG. Se procesará con canales bipolares.") - - if not Bipolar.empty: - labels = [] - for col in Bipolar.columns: - # print(f"Archivo: {file}, Canal: {col}, Frecuencia de muestreo: {sig.sampling_frequency} Hz, Duración: {len(Bipolar[col])/sig.sampling_frequency:.2f} segundos") - fs = edf.signals[edf.labels.index("M2")].sampling_frequency # Asumimos que todos los canales tienen la misma frecuencia de muestreo - time = np.linspace(0, len(Bipolar[col]) / fs, len(Bipolar[col])) - fil = EEG_functions.butter_bandpass_filter(Bipolar[col], lowcut=0.3, highcut=35, fs=fs, order=4) - norm = (fil-np.mean(fil))/np.std(fil) - - data.append(norm) # Restar la media para centrar la señal - labels.append(col) - # columns = Bipolar.columns.tolist() + if not Bipolar.empty: + labels = [] + for col in Bipolar.columns: + # print(f"Archivo: {file}, Canal: {col}, Frecuencia de muestreo: {sig.sampling_frequency} Hz, Duración: {len(Bipolar[col])/sig.sampling_frequency:.2f} segundos") + fs = physiological_data["M2"].sampling_frequency # Asumimos que todos los canales tienen la misma frecuencia de muestreo + fil = EEG_functions.butter_bandpass_filter(Bipolar[col], lowcut=0.3, highcut=35, fs=fs, order=4) + norm = (fil-np.mean(fil))/np.std(fil) + + data.append(norm) # Restar la media para centrar la señal + labels.append(col) + # columns = Bipolar.columns.tolist() + else: + labels = [] + for l in labels: + # print(f"Archivo: {file}, Canal: {sig.label}, Frecuencia de muestreo: {sig.sampling_frequency} Hz, Duración: {len(sig.data)/sig.sampling_frequency:.2f} segundos") + fs = physiological_fs[l] + fil = EEG_functions.butter_bandpass_filter(physiological_data[l], lowcut=0.3, highcut=35, fs=fs, order=4) + norm = (fil-np.mean(fil))/np.std(fil) + labels.append(l) + data.append(norm) # Restar la media para centrar la señal + + # columns = [selEEG[i][1].label for i in range(len(selEEG))] + + + for i, elec in enumerate(labels): + epoch_length = 30 # Duración de cada época en segundos + if Bipolar.empty: + fs = physiological_fs[l] else: - for i, (idx, sig) in enumerate(selEEG): - # print(f"Archivo: {file}, Canal: {sig.label}, Frecuencia de muestreo: {sig.sampling_frequency} Hz, Duración: {len(sig.data)/sig.sampling_frequency:.2f} segundos") - fs = sig.sampling_frequency - time = np.linspace(0, len(sig.data) / fs, len(sig.data)) - fil = EEG_functions.butter_bandpass_filter(sig.data, lowcut=0.3, highcut=35, fs=fs, order=4) - norm = (fil-np.mean(fil))/np.std(fil) - labels.append(sig.label) - data.append(norm) # Restar la media para centrar la señal - - # columns = [selEEG[i][1].label for i in range(len(selEEG))] - + fs = physiological_fs['M1'] + + if fs != 200: + # print(f"Warning: Sampling frequency for channel {elec} in file {file} is {fs} Hz, expected 200 Hz. Check the data.") + duration = len(data[i]) / fs + time_original = np.linspace(0, duration, len(data[i])) - demographics = demographics[demographics['BDSPPatientID'] == int(id)] - print(demographics) + num_samples_target = int(duration * 200 ) + time_target = np.linspace(0, duration, num_samples_target) + data[i] = np.interp(time_target, time_original, data[i]) + fs = 200 # Update fs to the target sampling frequency after resampling - for i, elec in enumerate(labels): - epoch_length = 30 # Duración de cada época en segundos - if Bipolar.empty: - fs = edf.signals[edf.labels.index(labels[i])].sampling_frequency - else: - fs = edf.signals[edf.labels.index('M1')].sampling_frequency - - if fs != 200: - # print(f"Warning: Sampling frequency for channel {elec} in file {file} is {fs} Hz, expected 200 Hz. Check the data.") - duration = len(data[i]) / fs - time_original = np.linspace(0, duration, len(data[i])) - - num_samples_target = int(duration * 200 ) - time_target = np.linspace(0, duration, num_samples_target) - data[i] = np.interp(time_target, time_original, data[i]) - fs = 200 # Update fs to the target sampling frequency after resampling - - epochs = EEG_functions.create_epochs(data[i], fs, epoch_duration=epoch_length) + epochs = EEG_functions.create_epochs(data[i], fs, epoch_duration=epoch_length) - band_powers, complexities = EEG_functions.extract_band_powers(epochs, fs, win_len=15) - print(f"Band powers for file {file}:") - - band_powers = band_powers.iloc[60:] # Eliminar las primeras 60 épocas (30 min) para evitar el tiempo despierto al inicio de la grabación - print(band_powers.head()) - - - # # Convertir de formato "ancho" a "largo" para Plotly - # df_melted = band_powers.melt(var_name='Banda', value_name='Potencia') - # # Creamos el boxplot - # fig = px.box(df_melted, x='Banda', y='Potencia', - # color='Banda', - # points="outliers", # Para ver si hay épocas muy extrañas - # title=f"{id} - {elec} - {demographics.Cognitive_Impairment.values[0]}", - # log_y=True) # Usamos escala logarítmica porque Delta suele ser mucho más potente que Beta - - # fig.update_layout(template="plotly_white", showlegend=False) - # # fig.write_html(f"graphs/BandasPersona/{id}_{elec}_{demographics.Cognitive_Impairment.values[0]}.html") # Guardar como HTML para visualización interactiva - # fig.show() - - # Ejecución - patient_summar = EEG_functions.get_patient_profile(band_powers) - # print(f"Resumen del perfil del paciente {id} - {elec}:") - # print(patient_summar) - d = complexities.iloc[:].std().to_dict() - results.append({ - 'File': file, - 'Channel': elec, - 'Patient_ID': id, - **d, - **patient_summar - }) + band_powers, complexities = EEG_functions.extract_band_powers(epochs, fs, win_len=15) + band_powers = band_powers.iloc[60:] # Eliminar las primeras 60 épocas (30 min) para evitar el tiempo despierto al inicio de la grabación + + + # Ejecución + patient_summar = EEG_functions.get_patient_profile(band_powers) + + d = complexities.iloc[:].std().to_dict() + results.append({ + 'Channel': elec, + **d, + **patient_summar + }) df_results = pd.DataFrame(results) - print(df_results.head()) - return df_results - # df_results.to_csv(f"results_summaryEEG_{hospital}.csv", index=False) \ No newline at end of file + return df_results \ No newline at end of file diff --git a/src/lib/Resp_features.py b/src/lib/Resp_features.py index 25c4706..553c9fa 100644 --- a/src/lib/Resp_features.py +++ b/src/lib/Resp_features.py @@ -33,27 +33,109 @@ def plot_resp(Data, subjet = 1, DownPrinting = 2): def peakedness_application(Data, stage, plotflag = False, subjet = 1): # print("Compute BR") - fs = 100 + fs = 25 Setup = {} Setup["K"] = 5 Setup["DT"] = 5 Setup["Ts"] = 60 #interval length of Welch periodograms (s) Setup["Tm"] = 20 #interval length of subintervals for Welch periodograms (s) # Setup["d"] = 0.1 #interval length of subintervals for Welch periodograms (s) - Setup["Omega_r"] = np.array([5, 20])/60 #respiratory rate range in Hz + Setup["Omega_r"] = np.array([5, 25])/60 #respiratory rate range in Hz Setup["plotflag"] = plotflag - Setup["Nfft"] = np.power(2,14) + Setup["Nfft"] = np.power(2,13) tsBR = np.arange(0,Data.shape[0]/fs,1/fs) if tsBR.shape[0] != Data.shape[0]: # print(f"tsBR.shape[0]: {tsBR.shape[0]}, Data.shape[0]: {Data.shape[0]}") tsBR = np.arange(0,Data.shape[0]/fs,1/fs)[:Data.shape[0]] - hat_Br, Sk_Br, t_aver = peakednessCost(Data, tsBR, fs, Setup, title = stage, storeGraph = False, subjet = subjet) + hat_Br, Sk_Br, t_aver, used = peakednessCost(Data, tsBR, fs, Setup, title = stage, storeGraph = False, subjet = subjet) # print(f"hat_Br: {hat_Br}, Sk_Br: {Sk_Br}, bar_Br: {bar_Br}, t_aver_Br: {t_aver_Br}, f_Br: {f_Br}, used_Br: {used_Br}") # print(hat_Br) - return hat_Br, Sk_Br, t_aver + return hat_Br, Sk_Br, t_aver, used + +def ODI_application(data, fs, plotflag=True, subjet=1): + """Detecta desaturaciones de más del 3 % en la señal de saturación de + oxígeno (SpO2) y devuelve estadísticas básicas de los eventos. + + El índice de desaturación de oxígeno (ODI) se define como el número de + episodios en los que la saturación cae al menos un 3 % respecto a una + línea de base móvil, normalizado por hora de grabación. Aquí se calcula + una línea base mediante la mediana móvil de 60 segundos y se agrupan + los índices consecutivos que cumplen el criterio en eventos únicos. + + Args: + data (array-like): valores de SpO2 (0‑100). + fs (float): frecuencia de muestreo en Hz. + plotflag (bool): si True, dibuja la señal y marca los eventos. + subjet (int): identificador de sujeto (utilizado en títulos de gráficas). + + Returns: + tuple: + * odi_mean (float): número de desaturaciones normalizado por hora. + * odi_std (float): desviación estándar de las magnitudes de caída + entre eventos (en porcentaje). + """ + # convertir a serie para comodidad + sp = pd.Series(data) + if len(sp) == 0 or fs <= 0: + return 0.0, 0.0 + + # base móvil de 60 segundos (median para ser robusto). ventana en muestras + window = int(fs * 60) + if window < 1: + window = 1 + baseline = sp.rolling(window, min_periods=1, center=True).median() + + # diferencia de base menos señal; buscamos caídas >=3 + diff = baseline - sp + mask = diff >= 3 + + # juntar índices contiguos en eventos + events = [] # lista de (start_idx, end_idx) + in_event = False + for idx, flag in mask.items(): + if flag and not in_event: + start = idx + in_event = True + elif not flag and in_event: + end = prev_idx + events.append((start, end)) + in_event = False + prev_idx = idx + if in_event: + events.append((start, prev_idx)) + + num_events = len(events) + duration_hours = len(sp) / fs / 3600.0 + odi_mean = num_events / duration_hours if duration_hours > 0 else 0.0 + + # calcular magnitudes de caída en cada evento (tomando el valor más bajo) + magnitudes = [] + for start, end in events: + mag = diff.loc[start:end].max() + magnitudes.append(mag) + odi_deepness = np.mean(magnitudes) if magnitudes else 0.0 + + if plotflag: + import matplotlib.pyplot as plt + times = np.arange(len(sp)) / fs / 60.0 # minutos + plt.figure(figsize=(10, 4)) + plt.plot(times, sp.values, label='SpO2') + plt.plot(times, baseline.values, label='Baseline (60s med)') + for (start, end) in events: + t0 = start / fs / 60.0 + t1 = end / fs / 60.0 + plt.axvspan(t0, t1, color='red', alpha=0.3) + plt.xlabel('Tiempo (min)') + plt.ylabel('SpO2 (%)') + plt.title(f'Sujeto {subjet} - ODI detectado: {odi_mean:.2f} eventos/h') + plt.legend() + plt.tight_layout() + plt.show() + + return odi_mean, odi_deepness # Butterworth low-pass filter def lowpass_filter(signal, fs, cutoff=2.0, order=4): diff --git a/src/resp_processing.py b/src/resp_processing.py index dfe6ec2..d8fa8bf 100644 --- a/src/resp_processing.py +++ b/src/resp_processing.py @@ -1,114 +1,138 @@ -import numpy as np -import pandas as pd -import os -import matplotlib.pyplot as plt -import plotly.express as px +import lib.Resp_features as Resp_features import sys - -import plotly.graph_objects as go -from plotly.subplots import make_subplots +import os +import pandas as pd +import numpy as np sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -import lib.helper_code as helper_code -import lib.EEG_functions as EEG_functions -import lib.Resp_features as Resp_features -for hospital in ['I0006',"S0001",'I0004','I0007']:#'I0002', - print(f"Procesando hospital: {hospital}") +def processResp(physiological_data, physiological_fs, csv_path): - if hospital == 'I0002' or hospital == 'I0006' or hospital == "S0001": - datapath = 'data/training_set/Physiological_data/'+hospital - else: - datapath = 'data/supplementary_set/Physiological_data/'+hospital - - channels = pd.read_csv("notebooks/channel_table.csv") + channels = pd.read_csv(csv_path) selectResp = channels[channels['Category'].isin(['resp'])] - demographics = pd.read_csv(os.path.join('C:/BSICoS/CincChallenge2026/CincChallenge_2026/data/training_set', "demographics.csv")) - - # Datos = pd.DataFrame(columns=['File', 'Channel', 'Sampling_Frequency', 'Duration_sec']) - lista_dir = os.listdir(datapath) - results = [] - - for file in lista_dir: - # Cargar el archivo (sustituye por tu ruta real) - edf = helper_code.edfio.read_edf(os.path.join(datapath, file)) - - id = file[9:-10] # Asumiendo que el ID es el nombre del archivo sin la extensión - - selResp = [] - labels = [] - data = [] - - HayResp = False - for i, sig in enumerate(edf.signals): - for index in selectResp.index: - if sig.label.lower() in selectResp['Channel_Names'][index].lower(): - print(f"Canal seleccionado: {sig.label}") - selResp.append([i,sig]) - labels.append(sig.label) - HayResp = True - # plot en plotly la señal - go.Figure(data=go.Scattergl(x=np.arange(len(sig.data))/sig.sampling_frequency, y=sig.data, mode='lines', name=sig.label)).update_layout(title=f"Señal de {sig.label} - Archivo: {file}", xaxis_title="Tiempo (s)", yaxis_title="Amplitud").show() - # px.line(x=np.arange(len(sig.data))/sig.sampling_frequency, y=sig.data, title=f"Señal de {sig.label} - Archivo: {file}").show() - break - - if HayResp: - for i, (idx, sig) in enumerate(selResp): - print(f"Archivo: {file}, Canal: {sig.label}, Frecuencia de muestreo: {sig.sampling_frequency} Hz, Duración: {len(sig.data)/sig.sampling_frequency:.2f} segundos") - fs = sig.sampling_frequency - - if fs != 25: - duration = len(sig.data) / fs - time_original = np.linspace(0, duration, len(sig.data)) - num_samples_target = int(duration * 25 ) - time_target = np.linspace(0, duration, num_samples_target) - data = np.interp(time_target, time_original, sig.data) - fs = 25 # Update fs to the target sampling frequency after resampling + resultados = {} + UsedFlow = 0 + UsedChest = 0 + UsedAbdomen = 0 + UsedSpO2 = 0 + UsedNasal = 0 + UsedCepap = 0 + + data = [] + original_labels = list(physiological_data.keys()) + + for label in original_labels: + fs = physiological_fs[label] + sig = physiological_data[label] + if fs != 25: + duration = len(sig) / fs + time_original = np.linspace(0, duration, len(sig)) + num_samples_target = int(duration * 25 ) + time_target = np.linspace(0, duration, num_samples_target) + data = np.interp(time_target, time_original, sig) + fs = 25 # Update fs to the target sampling frequency after resampling + else: + data = sig + + # Check nan in sig.data + if np.isnan(sig).any(): + print(f"Warning: NaN values found in signal data for {label}. Filling NaNs with zeros.") + data = np.nan_to_num(data) + + name = "" + if label.lower() not in selectResp['Channel_Names'][34].lower(): + d = Resp_features.peakedness_application(data, stage=label, plotflag = False, subjet =label) + if label.lower() in selectResp['Channel_Names'][28].lower(): + name = "Chest" + # EFFORT RESPIRATORY Chest + elif label.lower() in selectResp['Channel_Names'][29].lower(): + # EFFORT RESPIRATORY Abdomen + name = "Abdomen" + elif label.lower() in selectResp['Channel_Names'][30].lower(): + # RESPIRATORY NASAL + name = "Nasal" + elif label.lower() in selectResp['Channel_Names'][31].lower(): + # RESPIRATORY FLOW + name = "Flow" + elif label.lower() in selectResp['Channel_Names'][32].lower(): + # CEPAP + if np.all(data == 0) or np.std(data) < 5: + print(f"Warning: All values in the signal data for {label} are zero. Skipping feature extraction for this channel.") else: - data = sig.data - time_new = np.linspace(0, len(sig.data) / fs, len(sig.data)) - - # Check nan in sig.data - if np.isnan(sig.data).any(): - print(f"Warning: NaN values found in signal data for {sig.label}. Filling NaNs with zeros.") - data = np.nan_to_num(data) - - # if sig.label not in ["SpO2", "SaO2", "OSAT", "O2SAT", "O2 SAT", "O2-SAT", "O2-SATURATION"]: - # fil = EEG_functions.butter_bandpass_filter(data, lowcut=0.01, highcut=4, fs=fs, order=4) - # # norm = (fil-np.mean(fil))/np.std(fil) - # data.append(fil) # Restar la media para centrar la señal - - if sig.label.lower() in selectResp['Channel_Names'][28].lower() or sig.label.lower() in selectResp['Channel_Names'][29].lower(): - # EFFORT RESPIRATORY - elif sig.label.lower() in selectResp['Channel_Names'][30].lower() or sig.label.lower() in selectResp['Channel_Names'][31].lower(): - # RESPIRATORY Flujo - fil = EEG_functions.butter_bandpass_filter(data, lowcut=0.01, highcut=4, fs=fs, order=4) - Resp_features.peakedness_application(fil, stage=sig.label, plotflag = True, subjet =1) - elif sig.label.lower() in selectResp['Channel_Names'][32].lower() or sig.label.lower() in selectResp['Channel_Names'][33].lower(): - # CEPAP - elif sig.label.lower() in selectResp['Channel_Names'][34].lower(): - #O2 SATURATION - - - - # time_dt = pd.to_datetime(time_new, unit='s') - # # Plot raw and filtered signals - # fig = make_subplots(specs=[[{"secondary_y": True}]]) - # fig.add_trace(go.Scattergl(x=time_dt[::10], y=data[::10], name=sig.label, mode='lines'),secondary_y=False,row=1, col=1) - # fig.add_trace(go.Scattergl(x=time_dt[::10], y=fil[::10], name=f"Normalized {sig.label}", mode='lines'), secondary_y=True,row=1, col=1) - # fig.update_yaxes(title_text="Amplitud Original (uV)", secondary_y=False) - # fig.update_yaxes(title_text="Valor Normalizado (Z-score)", secondary_y=True) - # # update x axis to make time format - # fig.update_xaxes( - # tickformat="%H:%M:%S", # Formato de hora:minuto:segundo - # row=1, col=1 - # ) - # fig.show() - - # Plot spectrogram of raw and filtered signals - # fig = make_subplots(specs=[[{"secondary_y": True}]]) - # fig.add_trace(go.Scattergl(x=time_dt[::10], y=data[::10], name=sig.label, mode='lines'),secondary_y=False,row=1, col=1) - # fig.add_trace(go.Scattergl(x=time_dt[::10], y=fil[::10], name=f"Normalized {sig.label}", mode='lines'), secondary_y=True,row=1, col=1) - # fig.update_yaxes(title_text="Amplitud Original (uV)", secondary_y=False) - + name = "" + elif label.lower() in selectResp['Channel_Names'][33].lower(): + # CEPAP + name = "" + + if name != "": + DSinNan = d[0][~np.isnan(d[0])] # Eliminar NaN antes de calcular min y max + if len(DSinNan) != 0: + maximo = DSinNan.max() + minimo = DSinNan.min() + media = np.mean(DSinNan) + mediana = np.median(DSinNan) + std = DSinNan.std() + write = False + if name == "Nasal" and UsedNasal< d[-1]: + UsedNasal = d[-1] + write = True + elif name == "Chest" and UsedChest< d[-1]: + UsedChest = d[-1] + write = True + elif name == "Abdomen" and UsedAbdomen< d[-1]: + UsedAbdomen = d[-1] + write = True + elif name == "Flow" and UsedFlow< d[-1]: + UsedFlow = d[-1] + write = True + elif name == "SpO2" and UsedSpO2< d[-1]: + UsedSpO2 = d[-1] + write = True + elif name == "CEPAP" and UsedCepap < d[-1]: + UsedCepap = d[-1] + write = True + if write: + resultados.update({ + name+"_Peakedness_Max": maximo, + name+"_Peakedness_Min": minimo, + name+"_Peakedness_Mean": media, + name+"_Peakedness_Median": mediana, + name+"_Peakedness_Std": std + }) + + elif label.lower() in selectResp['Channel_Names'][34].lower(): + #O2 SATURATION + if np.max(data) < 2: + data = np.round((data/1.055)*100) + + lim = 0.7 + # Quitar los valores por debajo de lim y sus 10 valores anteriores y posteriores para quedarnos solo con los eventos de desaturación + dataReal = data.copy() + for i in range(len(data)): + if data[i] < lim: + start = int(max(0, i-fs*2)) + end = int(min(len(data), i+fs*2)) + dataReal[start:end] = np.nan # Marcar los valores por debajo del límite y sus alrededores como NaN + + CET90 = dataReal[dataReal < 90] + # CET90SinNan = CET90[~np.isnan(CET90)] # Eliminar NaN antes de calcular min y max + CET90 = len(CET90)/len(data) + dataRealSinNan = dataReal[~np.isnan(dataReal)] # Eliminar NaN antes de calcular min y max + if len(dataRealSinNan)>0: + maximo = dataRealSinNan.max() + minimo = dataRealSinNan.min() + std = dataRealSinNan.std() + media = dataRealSinNan.mean() + ODI_mean, ODI_deepness = Resp_features.ODI_application(dataReal, fs, plotflag=False, subjet=1) + + resultados.update({"SpO2_Max": maximo, + "SpO2_Min": minimo, + "SpO2_Mean": media, + "SpO2_Std": std, + "CET90": CET90, + "ODI_Mean": ODI_mean, + "ODI_deepness": ODI_deepness, + }) + + return pd.DataFrame(resultados) From de1a5cfbf2da2151cd2cbdae825b08fa2f9013e5 Mon Sep 17 00:00:00 2001 From: rolopu1 <718694@unizar.es> Date: Fri, 27 Mar 2026 13:58:12 +0100 Subject: [PATCH 25/93] Cambio de tipo de datos a np.array y concatenar features --- src/eeg_processing.py | 2 +- src/resp_processing.py | 2 +- team_code.py | 15 +++++++++++++-- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/eeg_processing.py b/src/eeg_processing.py index 760b474..df1303f 100644 --- a/src/eeg_processing.py +++ b/src/eeg_processing.py @@ -137,5 +137,5 @@ def processEEG(physiological_data, physiological_fs, csv_path): **d, **patient_summar }) - df_results = pd.DataFrame(results) + df_results = np.array(results) return df_results \ No newline at end of file diff --git a/src/resp_processing.py b/src/resp_processing.py index d8fa8bf..8bc284c 100644 --- a/src/resp_processing.py +++ b/src/resp_processing.py @@ -135,4 +135,4 @@ def processResp(physiological_data, physiological_fs, csv_path): "ODI_deepness": ODI_deepness, }) - return pd.DataFrame(resultados) + return np.array(resultados) diff --git a/team_code.py b/team_code.py index 2185a52..7296cc1 100644 --- a/team_code.py +++ b/team_code.py @@ -22,7 +22,8 @@ from tqdm import tqdm from helper_code import * - +from src.resp_processing import processResp +from src.eeg_processing import processEEG ################################################################################ # Path & Constant Configuration (Added for Robustness) ################################################################################ @@ -90,7 +91,17 @@ def process_training_record(record, data_folder, demographics_cache, diagnosis_c physiological_fs, csv_path=csv_path ) - + resp_features = processResp( + physiological_data, + physiological_fs, + csv_path=csv_path + ) + eeg_features = processEEG( + physiological_data, + physiological_fs, + csv_path=csv_path + ) + physiological_features = np.concatenate([physiological_features, resp_features, eeg_features], axis = 1) algorithmic_annotations_file = os.path.join( data_folder, ALGORITHMIC_ANNOTATIONS_SUBFOLDER, From 6ea8ddf4caf583b377cdcb66c7dfb74521a0b7d7 Mon Sep 17 00:00:00 2001 From: rolopu1 <718694@unizar.es> Date: Fri, 27 Mar 2026 14:14:44 +0100 Subject: [PATCH 26/93] =?UTF-8?q?A=C3=B1adir=20ficheros=20Sofia?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/lib/compute_hrv_hrf.py | 155 +++++++++++++++++++++++++++ src/lib/interpolate_NN.py | 40 +++++++ src/lib/openECGfunction.py | 39 +++++++ src/lib/pan_tompkins.py | 184 +++++++++++++++++++++++++++++++++ src/lib/remove_ectopic_beat.py | 41 ++++++++ src/main_ECG_ver2.py | 152 +++++++++++++++++++++++++++ 6 files changed, 611 insertions(+) create mode 100644 src/lib/compute_hrv_hrf.py create mode 100644 src/lib/interpolate_NN.py create mode 100644 src/lib/openECGfunction.py create mode 100644 src/lib/pan_tompkins.py create mode 100644 src/lib/remove_ectopic_beat.py create mode 100644 src/main_ECG_ver2.py diff --git a/src/lib/compute_hrv_hrf.py b/src/lib/compute_hrv_hrf.py new file mode 100644 index 0000000..15b0422 --- /dev/null +++ b/src/lib/compute_hrv_hrf.py @@ -0,0 +1,155 @@ +import numpy as np +from scipy.signal import lombscargle + +def compute_HRV_HRF(NN, SF): + """ + NN: array of NN intervals in seconds + SF: sampling frequency (Hz) + """ + + NN = np.asarray(NN).flatten() + + # =============================== + # ΔNN + # =============================== + dNN = np.diff(NN) + + n = 1 + thr = n / SF + + # Classification + acc = dNN <= -thr + dec = dNN >= thr + noch = (dNN > -thr) & (dNN < thr) + + # Sign representation + sign_dNN = np.zeros_like(dNN) + sign_dNN[acc] = -1 + sign_dNN[dec] = 1 + + N = len(dNN) + + # =============================== + # PIP (Inflection Points) + # =============================== + inflection = 0 + + for i in range(N - 1): + if (dNN[i+1] * dNN[i] <= 0) and (dNN[i+1] != dNN[i]): + inflection += 1 + + PIP = (inflection / (N - 1)) * 100 if N > 1 else np.nan + + # =============================== + # Segment Detection + # =============================== + segments = [] + if N > 0: + current_seg = sign_dNN[0] + length_seg = 1 + + for i in range(1, N): + if sign_dNN[i] == current_seg and sign_dNN[i] != 0: + length_seg += 1 + else: + if current_seg != 0: + segments.append(length_seg) + current_seg = sign_dNN[i] + length_seg = 1 + + # Add last segment + if current_seg != 0: + segments.append(length_seg) + + segments = np.array(segments) + + # =============================== + # PNNLS & PNNSS + # =============================== + if len(segments) > 0: + long_segments = segments[segments >= 3] + short_segments = segments[segments < 3] + + PNNLS = np.sum(long_segments) / N * 100 + PNNSS = np.sum(short_segments) / np.sum(segments) * 100 + else: + PNNLS = np.nan + PNNSS = np.nan + + # =============================== + # Time-domain HRV + # =============================== + win_length = 300 # seconds + + time = np.cumsum(NN) + + AVNN_all = [] + SDNN_all = [] + RMSSD_all = [] + + i = 0 + while i < len(NN): + t_start = time[i] + t_end = t_start + win_length + + idx = np.where((time >= t_start) & (time < t_end))[0] + + if len(idx) >= 150: + NN_win = NN[idx] + + AVNN_all.append(np.nanmean(NN_win)) + SDNN_all.append(np.nanstd(NN_win, ddof=1)) + + diffNN = np.diff(NN_win) + RMSSD_all.append(np.sqrt(np.nanmean(diffNN**2))) + + next_i = np.where(time >= t_end)[0] + if len(next_i) == 0: + break + i = next_i[0] + + AVNN = np.nanmean(AVNN_all) if len(AVNN_all) > 0 else np.nan + SDNN = np.nanmean(SDNN_all) if len(SDNN_all) > 0 else np.nan + RMSSD = np.nanmean(RMSSD_all) if len(RMSSD_all) > 0 else np.nan + + # =============================== + # Frequency-domain (HF) + # =============================== + HF_all = [] + + for _ in range(len(AVNN_all)): + # NOTE: simplified like MATLAB version + NN_win = NN.copy() + t_win = np.cumsum(NN_win) + + # Convert to angular frequency + f = np.linspace(0.01, 0.5, 1000) + angular_f = 2 * np.pi * f + + # Remove mean (important for Lomb) + NN_detrended = NN_win - np.mean(NN_win) + + Pxx = lombscargle(t_win, NN_detrended, angular_f, normalize=True) + + HF_band = (f >= 0.15) & (f <= 0.4) + + HF_power = np.trapezoid(Pxx[HF_band], f[HF_band]) + + HF_all.append(HF_power) + + HF = np.nanmean(HF_all) if len(HF_all) > 0 else np.nan + + # =============================== + # OUTPUT + # =============================== + results = { + "PIP": PIP, + "PNNLS": PNNLS, + "PNNSS": PNNSS, + "AVNN": AVNN, + "SDNN": SDNN, + "RMSSD": RMSSD, + "HF": HF + } + + return results \ No newline at end of file diff --git a/src/lib/interpolate_NN.py b/src/lib/interpolate_NN.py new file mode 100644 index 0000000..987019f --- /dev/null +++ b/src/lib/interpolate_NN.py @@ -0,0 +1,40 @@ +import numpy as np +from scipy.interpolate import PchipInterpolator + +def interpolate_NN_pchip(NN, maxGap): + """ + NN: array of NN intervals (seconds) + maxGap: max number of consecutive NaNs allowed for interpolation + """ + + NN = np.asarray(NN).flatten() + NN_interp = NN.copy() + + nan_idx = np.isnan(NN) + + # Find NaN segments + d = np.diff(np.concatenate(([0], nan_idx.astype(int), [0]))) + start_idx = np.where(d == 1)[0] + end_idx = np.where(d == -1)[0] - 1 + + for k in range(len(start_idx)): + seg_len = end_idx[k] - start_idx[k] + 1 + + if seg_len <= maxGap: + left = start_idx[k] - 1 + right = end_idx[k] + 1 + + # Check bounds + if (left >= 0 and right < len(NN) and + not np.isnan(NN[left]) and not np.isnan(NN[right])): + + x = np.array([left, right]) + y = np.array([NN[left], NN[right]]) + + xi = np.arange(start_idx[k], end_idx[k] + 1) + + # PCHIP interpolation + interpolator = PchipInterpolator(x, y) + NN_interp[xi] = interpolator(xi) + + return NN_interp \ No newline at end of file diff --git a/src/lib/openECGfunction.py b/src/lib/openECGfunction.py new file mode 100644 index 0000000..5937f76 --- /dev/null +++ b/src/lib/openECGfunction.py @@ -0,0 +1,39 @@ +import pyedflib +from main_ECG_ver2 import ECGprocessing +import pandas as pd +def openECG(physiological_data_file, patient_id): + + f = pyedflib.EdfReader(physiological_data_file) + + signal_labels = f.getSignalLabels() + print(signal_labels) + + ecg_keywords = ['ecg', 'ekg'] + + idx = None + for i, label in enumerate(signal_labels): + label_clean = label.lower().strip() + + # Check if any ECG keyword is inside the label + if any(keyword in label_clean for keyword in ecg_keywords): + idx = i + break # ✅ first ECG channel only + + if idx is None: + raise ValueError("No ECG channel found") + + print("ECG channel:", signal_labels[idx]) + + ecg_signal = f.readSignal(idx) + fs = f.getSampleFrequency(idx) + + f.close() + + all_results = ECGprocessing(ecg_signal, fs, patient_id) + + if all_results is not None: + all_patients_ECGresults = pd.concat( + [all_patients_ECGresults, all_results], + ignore_index=True + ) + return all_patients_ECGresults \ No newline at end of file diff --git a/src/lib/pan_tompkins.py b/src/lib/pan_tompkins.py new file mode 100644 index 0000000..3f37214 --- /dev/null +++ b/src/lib/pan_tompkins.py @@ -0,0 +1,184 @@ +import numpy as np +from scipy.signal import butter, filtfilt, find_peaks + + +def pan_tompkin(ecg, fs, gr=0): + + ecg = np.asarray(ecg).flatten() + delay = 0 + + skip = 0 + m_selected_RR = 0 + mean_RR = 0 + ser_back = 0 + + # ===================== FILTERING ===================== # + ecg = ecg - np.mean(ecg) + + if fs == 200: + # Low-pass + b, a = butter(3, 12*2/fs, btype='low') + ecg_l = filtfilt(b, a, ecg) + ecg_l = ecg_l / np.max(np.abs(ecg_l)) + + # High-pass + b, a = butter(3, 5*2/fs, btype='high') + ecg_h = filtfilt(b, a, ecg_l) + ecg_h = ecg_h / np.max(np.abs(ecg_h)) + else: + b, a = butter(3, [5*2/fs, 15*2/fs], btype='band') + ecg_h = filtfilt(b, a, ecg) + ecg_h = ecg_h / np.max(np.abs(ecg_h)) + + # ===================== DERIVATIVE ===================== # + if fs != 200: + int_c = int((5 - 1) / (fs * (1/40))) + base = np.array([1, 2, 0, -2, -1]) * (1/8) * fs + x_old = np.linspace(1, 5, 5) + x_new = np.linspace(1, 5, int_c) + b = np.interp(x_new, x_old, base) + else: + b = np.array([1, 2, 0, -2, -1]) * (1/8) * fs + + ecg_d = filtfilt(b, [1], ecg_h) + ecg_d = ecg_d / np.max(np.abs(ecg_d)) + + # ===================== SQUARING ===================== # + ecg_s = ecg_d ** 2 + + # ===================== MOVING WINDOW ===================== # + win = int(round(0.150 * fs)) + ecg_m = np.convolve(ecg_s, np.ones(win)/win, mode='same') + delay += win // 2 + + # ===================== PEAK DETECTION ===================== # + locs, _ = find_peaks(ecg_m, distance=int(0.2 * fs)) + pks = ecg_m[locs] + + LLp = len(pks) + + qrs_i = [] + qrs_c = [] + qrs_i_raw = [] + qrs_amp_raw = [] + + nois_i = [] + nois_c = [] + + # Threshold initialization + THR_SIG = np.max(ecg_m[:2*fs]) / 3 + THR_NOISE = np.mean(ecg_m[:2*fs]) / 2 + SIG_LEV = THR_SIG + NOISE_LEV = THR_NOISE + + THR_SIG1 = np.max(ecg_h[:2*fs]) / 3 + THR_NOISE1 = np.mean(ecg_h[:2*fs]) / 2 + SIG_LEV1 = THR_SIG1 + NOISE_LEV1 = THR_NOISE1 + + Beat_C = 0 + Beat_C1 = 0 + + for i in range(LLp): + + loc = locs[i] + + # Find peak in filtered signal + left = max(0, loc - int(0.150 * fs)) + right = loc + + if right < len(ecg_h): + segment = ecg_h[left:right+1] + if len(segment) > 0: + y_i = np.max(segment) + x_i = np.argmax(segment) + else: + continue + else: + continue + + # RR interval update + if len(qrs_i) >= 9: + diffRR = np.diff(qrs_i[-8:]) + mean_RR = np.mean(diffRR) + comp = qrs_i[-1] - qrs_i[-2] + + if comp <= 0.92 * mean_RR or comp >= 1.16 * mean_RR: + THR_SIG *= 0.5 + THR_SIG1 *= 0.5 + else: + m_selected_RR = mean_RR + + test_m = m_selected_RR if m_selected_RR else mean_RR + + # ===================== SEARCH BACK ===================== # + if test_m and len(qrs_i) > 0: + if (loc - qrs_i[-1]) >= int(1.66 * test_m): + + sb_left = qrs_i[-1] + int(0.2 * fs) + sb_right = loc - int(0.2 * fs) + + if sb_right > sb_left: + segment = ecg_m[sb_left:sb_right] + if len(segment) > 0: + pks_temp = np.max(segment) + locs_temp = sb_left + np.argmax(segment) + + if pks_temp > THR_NOISE: + qrs_c.append(pks_temp) + qrs_i.append(locs_temp) + + seg = ecg_h[max(0, locs_temp-int(0.150*fs)):locs_temp] + if len(seg) > 0: + y_i_t = np.max(seg) + x_i_t = np.argmax(seg) + + if y_i_t > THR_NOISE1: + qrs_i_raw.append(locs_temp - int(0.150*fs) + x_i_t) + qrs_amp_raw.append(y_i_t) + SIG_LEV1 = 0.25*y_i_t + 0.75*SIG_LEV1 + + SIG_LEV = 0.25*pks_temp + 0.75*SIG_LEV + + # ===================== CLASSIFICATION ===================== # + if pks[i] >= THR_SIG: + + # T-wave rejection + if len(qrs_i) >= 3: + if (loc - qrs_i[-1]) <= int(0.36 * fs): + + slope1 = np.mean(np.diff(ecg_m[max(0, loc-int(0.075*fs)):loc])) + slope2 = np.mean(np.diff(ecg_m[max(0, qrs_i[-1]-int(0.075*fs)):qrs_i[-1]])) + + if abs(slope1) <= 0.5 * abs(slope2): + NOISE_LEV1 = 0.125*y_i + 0.875*NOISE_LEV1 + NOISE_LEV = 0.125*pks[i] + 0.875*NOISE_LEV + continue + + # Accept QRS + qrs_c.append(pks[i]) + qrs_i.append(loc) + + if y_i >= THR_SIG1: + qrs_i_raw.append(loc - int(0.150*fs) + x_i) + qrs_amp_raw.append(y_i) + SIG_LEV1 = 0.125*y_i + 0.875*SIG_LEV1 + + SIG_LEV = 0.125*pks[i] + 0.875*SIG_LEV + + elif THR_NOISE <= pks[i] < THR_SIG: + NOISE_LEV1 = 0.125*y_i + 0.875*NOISE_LEV1 + NOISE_LEV = 0.125*pks[i] + 0.875*NOISE_LEV + + else: + NOISE_LEV1 = 0.125*y_i + 0.875*NOISE_LEV1 + NOISE_LEV = 0.125*pks[i] + 0.875*NOISE_LEV + + # Update thresholds + THR_SIG = NOISE_LEV + 0.25 * abs(SIG_LEV - NOISE_LEV) + THR_NOISE = 0.5 * THR_SIG + + THR_SIG1 = NOISE_LEV1 + 0.25 * abs(SIG_LEV1 - NOISE_LEV1) + THR_NOISE1 = 0.5 * THR_SIG1 + + return np.array(qrs_amp_raw), np.array(qrs_i_raw), delay \ No newline at end of file diff --git a/src/lib/remove_ectopic_beat.py b/src/lib/remove_ectopic_beat.py new file mode 100644 index 0000000..de60caa --- /dev/null +++ b/src/lib/remove_ectopic_beat.py @@ -0,0 +1,41 @@ +import numpy as np + +def remove_ectopic_beats(NN, window_size, threshold): + NN = np.asarray(NN).flatten() + NN_corrected = NN.copy() + + half_win = window_size // 2 + ectopic_count = 0 + valid_count = 0 + + for i in range(len(NN)): + + if np.isnan(NN[i]): + continue + + valid_count += 1 + + # Define local window + left = max(0, i - half_win) + right = min(len(NN), i + half_win + 1) # Python slice is exclusive + + local_segment = NN[left:right] + local_segment = local_segment[~np.isnan(local_segment)] + + if local_segment.size == 0: + continue + + med_val = np.median(local_segment) + + # Detect ectopic + if abs(NN[i] - med_val) > threshold * med_val: + NN_corrected[i] = med_val + ectopic_count += 1 + + # Percentage over valid NN + if valid_count > 0: + ectopic_perc = (ectopic_count / valid_count) * 100 + else: + ectopic_perc = np.nan + + return NN_corrected, ectopic_perc \ No newline at end of file diff --git a/src/main_ECG_ver2.py b/src/main_ECG_ver2.py new file mode 100644 index 0000000..03aaa63 --- /dev/null +++ b/src/main_ECG_ver2.py @@ -0,0 +1,152 @@ +import numpy as np +import pandas as pd +from scipy.signal import butter, filtfilt, resample +from lib.pan_tompkins import pan_tompkin +from lib.compute_hrv_hrf import compute_HRV_HRF +from lib.interpolate_NN import interpolate_NN_pchip +from lib.remove_ectopic_beat import remove_ectopic_beats +def ECGprocessing(ecg_signal, fs, patient_id): + + all_results = pd.DataFrame() + + ecg_signal = ecg_signal - np.mean(ecg_signal) + + # =============================== + # RESAMPLE TO 200 Hz IF NEEDED + # =============================== + target_fs = 200 + + if fs != target_fs: + num_samples = int(len(ecg_signal) * target_fs / fs) + ecg_signal = resample(ecg_signal, num_samples) + fs = target_fs + + # =============================== + # SEGMENT INTO 5-MIN WINDOWS + # =============================== + win_sec = 300 + win_samples = int(win_sec * fs) + + N = len(ecg_signal) + n_windows = N // win_samples + + if n_windows == 0: + print("Signal too short.") + return None + + # =============================== + # FIND VALID WINDOWS + # =============================== + valid_windows = [] + + for w in range(n_windows): + + idx_start = w * win_samples + idx_end = (w + 1) * win_samples + + ecg_win = ecg_signal[idx_start:idx_end] + + # Quality check + if np.sum(np.isnan(ecg_win)) != 0 or np.sum(ecg_win == 0) > 0.2 * len(ecg_win): + continue + + valid_windows.append(w) + + # =============================== + # PROCESS WINDOWS + # =============================== + HRV_all = [] + + for w in valid_windows: + + idx_start = w * win_samples + idx_end = (w + 1) * win_samples + + ecg_win = ecg_signal[idx_start:idx_end] + + ecg_win = ecg_win - np.mean(ecg_win) + + # --- Filtering --- + # Notch + b, a = butter(3, [59.5/(fs/2), 60.5/(fs/2)], btype='bandstop') + ecg_win = filtfilt(b, a, ecg_win) + + # High-pass + b, a = butter(3, 0.5/(fs/2), btype='high') + ecg_win = filtfilt(b, a, ecg_win) + + # Low-pass + b, a = butter(3, 45/(fs/2), btype='low') + ecg_win = filtfilt(b, a, ecg_win) + + # =============================== + # QRS DETECTION + # =============================== + qrs_amp_raw, R_locs, delay = pan_tompkin(ecg_win, fs, 0) + + if len(R_locs) < 150: + continue + + # =============================== + # NN INTERVALS + # =============================== + NN = np.diff(R_locs) / fs + + # =============================== + # HRV PREPROCESSING + # =============================== + NN, ectopic_perc = remove_ectopic_beats(NN, 40, 0.10) + + NN = interpolate_NN_pchip(NN, 2) + + valid_ratio = np.sum(~np.isnan(NN)) / len(NN) + NN = NN[~np.isnan(NN)] + + if valid_ratio < 0.75: + continue + + # =============================== + # HRV + HRF METRICS + # =============================== + res = compute_HRV_HRF(NN, fs) + + meanNN = np.mean(NN) + + HRV_all.append([ + meanNN, + res["PIP"], res["PNNLS"], res["PNNSS"], + res["AVNN"], res["SDNN"], res["RMSSD"], res["HF"], + ectopic_perc + ]) + + # =============================== + # SUBJECT-LEVEL METRICS + # =============================== + if len(HRV_all) == 0: + print("No valid windows.") + return None + + HRV_all = np.array(HRV_all) + + median_vals = np.nanmedian(HRV_all, axis=0) + std_vals = np.nanstd(HRV_all, axis=0) + + # =============================== + # SAVE RESULTS (DataFrame row) + # =============================== + row = pd.DataFrame([{ + "ID": patient_id, + "mNNmed": median_vals[0], "mNNstd": std_vals[0], + "PIP_med": median_vals[1], "PIP_std": std_vals[1], + "PNNLS_med": median_vals[2], "PNNLS_std": std_vals[2], + "PNNSS_med": median_vals[3], "PNNSS_std": std_vals[3], + "AVNN_med": median_vals[4], "AVNN_std": std_vals[4], + "SDNN_med": median_vals[5], "SDNN_std": std_vals[5], + "RMSSD_med": median_vals[6], "RMSSD_std": std_vals[6], + "HF_med": median_vals[7], "HF_std": std_vals[7], + "ECTOPIC_med": median_vals[8], "ECTOPIC_std": std_vals[8], + }]) + + all_results = pd.concat([all_results, row], ignore_index=True) + + return all_results \ No newline at end of file From 14e0672a7d70a8aac0e7d7c16b79846125828eb9 Mon Sep 17 00:00:00 2001 From: dcajal Date: Fri, 27 Mar 2026 16:02:46 +0100 Subject: [PATCH 27/93] Fix import errors --- src/__init__.py | 1 + src/eeg_processing.py | 2 +- src/lib/EEG_functions.py | 20 +++++++++++++++++--- src/lib/Resp_features.py | 23 +++++++++++++++++++---- src/lib/__init__.py | 1 + src/lib/openECGfunction.py | 2 +- src/lib/peakedness.py | 19 ++++++++++++++++--- src/main_ECG_ver2.py | 8 ++++---- src/resp_processing.py | 2 +- src/results_analysis.py | 2 +- 10 files changed, 62 insertions(+), 18 deletions(-) create mode 100644 src/__init__.py create mode 100644 src/lib/__init__.py diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..6319343 --- /dev/null +++ b/src/__init__.py @@ -0,0 +1 @@ +"""Project source package.""" \ No newline at end of file diff --git a/src/eeg_processing.py b/src/eeg_processing.py index df1303f..04147c6 100644 --- a/src/eeg_processing.py +++ b/src/eeg_processing.py @@ -34,7 +34,7 @@ import pandas as pd import numpy as np import helper_code as helper_code -import lib.EEG_functions as EEG_functions +from .lib import EEG_functions sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) diff --git a/src/lib/EEG_functions.py b/src/lib/EEG_functions.py index 5427f60..83e8ef6 100644 --- a/src/lib/EEG_functions.py +++ b/src/lib/EEG_functions.py @@ -1,12 +1,21 @@ from scipy.signal import butter, filtfilt import numpy as np -import plotly.graph_objects as go -from plotly.subplots import make_subplots from scipy.signal import welch import pandas as pd from scipy import signal from scipy.stats import kurtosis, entropy -import matplotlib.pyplot as plt + +try: + import plotly.graph_objects as go + from plotly.subplots import make_subplots +except ModuleNotFoundError: + go = None + make_subplots = None + +try: + import matplotlib.pyplot as plt +except ModuleNotFoundError: + plt = None def butter_bandpass_filter(data, lowcut, highcut, fs, order=4): nyq = 0.5 * fs # Frecuencia de Nyquist @@ -32,6 +41,8 @@ def butter_bandpass_filter(data, lowcut, highcut, fs, order=4): return y def plot_EEG(df, columns, fs = 200): + if go is None or make_subplots is None: + raise ModuleNotFoundError("plotly is required for plot_EEG") fig = make_subplots(rows=len(columns), cols=1, shared_xaxes=True, @@ -55,6 +66,9 @@ def plot_EEG(df, columns, fs = 200): fig.show() def plot_EEG_sel(sel, name = "EEG_plot_raw.html"): + if go is None or make_subplots is None: + raise ModuleNotFoundError("plotly is required for plot_EEG_sel") + fig = make_subplots(rows=len(sel), cols=1, shared_xaxes=True, vertical_spacing=0.02, diff --git a/src/lib/Resp_features.py b/src/lib/Resp_features.py index 553c9fa..216b3ab 100644 --- a/src/lib/Resp_features.py +++ b/src/lib/Resp_features.py @@ -1,18 +1,29 @@ import pandas as pd import numpy as np -import plotly.graph_objs as go -from lib.peakedness import peakednessCost +from .peakedness import peakednessCost from scipy.interpolate import interp1d -import matplotlib.pyplot as plt from scipy.stats import kruskal from scipy.signal import resample, detrend import scipy.fft as fft from scipy.signal import butter, filtfilt +try: + import plotly.graph_objs as go +except ModuleNotFoundError: + go = None + +try: + import matplotlib.pyplot as plt +except ModuleNotFoundError: + plt = None + def plot_resp(Data, subjet = 1, DownPrinting = 2): """ Plot resp data using Plotly. """ + if go is None: + raise ModuleNotFoundError("plotly is required for plot_resp") + if type(Data) == dict: Data = pd.DataFrame(Data[str(subjet)]) Data = Data.iloc[::DownPrinting, :] @@ -119,7 +130,8 @@ def ODI_application(data, fs, plotflag=True, subjet=1): odi_deepness = np.mean(magnitudes) if magnitudes else 0.0 if plotflag: - import matplotlib.pyplot as plt + if plt is None: + raise ModuleNotFoundError("matplotlib is required when plotflag=True") times = np.arange(len(sp)) / fs / 60.0 # minutos plt.figure(figsize=(10, 4)) plt.plot(times, sp.values, label='SpO2') @@ -292,6 +304,9 @@ def Significance_tests(RespData): results = results.reset_index() results.to_excel('./Graphs/kruskal_results.xlsx', index=False) + if plt is None: + raise ModuleNotFoundError("matplotlib is required for Significance_tests plotting") + plt.plot(results['index'], results['p_value']) plt.axhline(y=0.05, color='r', linestyle='--') plt.xlabel('Métrica') diff --git a/src/lib/__init__.py b/src/lib/__init__.py new file mode 100644 index 0000000..665f43f --- /dev/null +++ b/src/lib/__init__.py @@ -0,0 +1 @@ +"""Signal-processing helper package.""" \ No newline at end of file diff --git a/src/lib/openECGfunction.py b/src/lib/openECGfunction.py index 5937f76..745b1ac 100644 --- a/src/lib/openECGfunction.py +++ b/src/lib/openECGfunction.py @@ -1,5 +1,5 @@ import pyedflib -from main_ECG_ver2 import ECGprocessing +from ..main_ECG_ver2 import ECGprocessing import pandas as pd def openECG(physiological_data_file, patient_id): diff --git a/src/lib/peakedness.py b/src/lib/peakedness.py index 6323aca..9c47f96 100644 --- a/src/lib/peakedness.py +++ b/src/lib/peakedness.py @@ -3,12 +3,21 @@ from numpy.fft import fftshift from numpy.fft import fft from scipy.signal import detrend, find_peaks -import matplotlib.pyplot as plt -import plotly.graph_objs as go -from plotly import subplots from time import time import os +try: + import matplotlib.pyplot as plt +except ModuleNotFoundError: + plt = None + +try: + import plotly.graph_objs as go + from plotly import subplots +except ModuleNotFoundError: + go = None + subplots = None + def setParamFr(Setup): if 'DT' not in Setup.keys(): Setup["DT"] = 5 @@ -265,6 +274,8 @@ def init_module(kk,vars,param, plotflag): # vars["bar_fr"][kk] = f[fj] if plotflag: + if plt is None: + raise ModuleNotFoundError("matplotlib is required when plotflag=True") plt.plot(f, averS) plt.plot(f[fj], averS[fj], '-') plt.title('Initialization - Averaged Spectrum') @@ -605,6 +616,8 @@ def peakednessCost(signals, ts, fs, Setup = {}, title = "", storeGraph = False, t_fin = time() if plotflag: + if go is None or subplots is None: + raise ModuleNotFoundError("plotly is required when plotflag=True") fig = subplots.make_subplots(rows=2,shared_xaxes=True, subplot_titles=('Peak-condition averaged EDR Spectra in '+title,"EDR/RESP signals"), row_heights=[0.7, 0.3]) diff --git a/src/main_ECG_ver2.py b/src/main_ECG_ver2.py index 03aaa63..6206e23 100644 --- a/src/main_ECG_ver2.py +++ b/src/main_ECG_ver2.py @@ -1,10 +1,10 @@ import numpy as np import pandas as pd from scipy.signal import butter, filtfilt, resample -from lib.pan_tompkins import pan_tompkin -from lib.compute_hrv_hrf import compute_HRV_HRF -from lib.interpolate_NN import interpolate_NN_pchip -from lib.remove_ectopic_beat import remove_ectopic_beats +from .lib.pan_tompkins import pan_tompkin +from .lib.compute_hrv_hrf import compute_HRV_HRF +from .lib.interpolate_NN import interpolate_NN_pchip +from .lib.remove_ectopic_beat import remove_ectopic_beats def ECGprocessing(ecg_signal, fs, patient_id): all_results = pd.DataFrame() diff --git a/src/resp_processing.py b/src/resp_processing.py index 8bc284c..c158d2b 100644 --- a/src/resp_processing.py +++ b/src/resp_processing.py @@ -1,4 +1,4 @@ -import lib.Resp_features as Resp_features +from .lib import Resp_features import sys import os import pandas as pd diff --git a/src/results_analysis.py b/src/results_analysis.py index c2fef8b..b415a4b 100644 --- a/src/results_analysis.py +++ b/src/results_analysis.py @@ -5,7 +5,7 @@ import plotly.express as px sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -import lib.EEG_functions as EEG_functions +from src.lib import EEG_functions import seaborn as sns import matplotlib.pyplot as plt import plotly.express as px From 6c4f780e55f3221b9f02856c084abac67534fe1d Mon Sep 17 00:00:00 2001 From: rolopu1 <718694@unizar.es> Date: Fri, 27 Mar 2026 17:33:41 +0100 Subject: [PATCH 28/93] Bypass CPAP --- src/resp_processing.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/resp_processing.py b/src/resp_processing.py index c158d2b..245144a 100644 --- a/src/resp_processing.py +++ b/src/resp_processing.py @@ -61,6 +61,7 @@ def processResp(physiological_data, physiological_fs, csv_path): print(f"Warning: All values in the signal data for {label} are zero. Skipping feature extraction for this channel.") else: name = "" + name = "" #Bypass CPAP elif label.lower() in selectResp['Channel_Names'][33].lower(): # CEPAP name = "" From 4fe3992d51a3096e15e112488457a73674221052 Mon Sep 17 00:00:00 2001 From: rolopu1 <718694@unizar.es> Date: Fri, 27 Mar 2026 17:51:48 +0100 Subject: [PATCH 29/93] Correccion EEG --- src/eeg_processing.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/eeg_processing.py b/src/eeg_processing.py index 04147c6..eca4b44 100644 --- a/src/eeg_processing.py +++ b/src/eeg_processing.py @@ -93,8 +93,9 @@ def processEEG(physiological_data, physiological_fs, csv_path): labels.append(col) # columns = Bipolar.columns.tolist() else: + labels2 = labels labels = [] - for l in labels: + for l in labels2: # print(f"Archivo: {file}, Canal: {sig.label}, Frecuencia de muestreo: {sig.sampling_frequency} Hz, Duración: {len(sig.data)/sig.sampling_frequency:.2f} segundos") fs = physiological_fs[l] fil = EEG_functions.butter_bandpass_filter(physiological_data[l], lowcut=0.3, highcut=35, fs=fs, order=4) From 7bf25001fa0ca8bdbd88348abcb00805800958d3 Mon Sep 17 00:00:00 2001 From: dcajal Date: Fri, 27 Mar 2026 22:41:30 +0100 Subject: [PATCH 30/93] Update smoke dataset configuration to use 10 subjects by default and enhance demographic filtering process --- docs/03_smoke_dataset.md | 5 +-- scripts/create_smoke.ps1 | 69 ++++++++++++++++++++++++++++++++++++---- scripts/create_smoke.sh | 66 ++++++++++++++++++++++++++++++++------ 3 files changed, 122 insertions(+), 18 deletions(-) diff --git a/docs/03_smoke_dataset.md b/docs/03_smoke_dataset.md index 5ba827b..7c0e3d7 100644 --- a/docs/03_smoke_dataset.md +++ b/docs/03_smoke_dataset.md @@ -2,7 +2,7 @@ Entrenar con el dataset completo tarda aproximadamente 30–40 minutos con el modelo de ejemplo. -Para desarrollo utilizamos un dataset reducido (5 sujetos). +Para desarrollo utilizamos un dataset reducido (10 sujetos por defecto). Este documento describe cuándo y por qué usar smoke. Los comandos de ejecución están centralizados en `docs/04_run_script.md`. @@ -11,9 +11,10 @@ Los comandos de ejecución están centralizados en `docs/04_run_script.md`. ## Qué incluye -- Muestra reducida del dataset (5 sujetos) +- Muestra reducida del dataset (10 sujetos por defecto) - Estructura compatible con el flujo oficial del proyecto - Directorio de salida en `data/training_smoke/` +- `demographics.csv` filtrado para que solo incluya los registros copiados al smoke ## Para qué se usa diff --git a/scripts/create_smoke.ps1 b/scripts/create_smoke.ps1 index bd39bcd..d66aaff 100644 --- a/scripts/create_smoke.ps1 +++ b/scripts/create_smoke.ps1 @@ -10,7 +10,7 @@ $FULL_DATA_PATH = "data/training_set" # <-- CHANGE THIS IF NEEDED $SMOKE_PATH = "data/training_smoke" -$N_RECORDS = 5 +$N_RECORDS = 10 Write-Host "Creating smoke dataset..." Write-Host "Source: $FULL_DATA_PATH" @@ -19,8 +19,7 @@ Write-Host "Destination: $SMOKE_PATH" Remove-Item -Recurse -Force $SMOKE_PATH -ErrorAction SilentlyContinue New-Item -ItemType Directory -Force -Path $SMOKE_PATH | Out-Null -# Copy demographics -Copy-Item "$FULL_DATA_PATH/demographics.csv" "$SMOKE_PATH/demographics.csv" +$selectedRecords = New-Object System.Collections.Generic.List[object] # Select first N EDF files $edfs = Get-ChildItem "$FULL_DATA_PATH/physiological_data" -Recurse -Filter *.edf | @@ -32,10 +31,68 @@ foreach ($f in $edfs) { $target = Join-Path $SMOKE_PATH $rel New-Item -ItemType Directory -Force -Path (Split-Path $target) | Out-Null Copy-Item $f.FullName $target + + $stem = [System.IO.Path]::GetFileNameWithoutExtension($f.Name) + $parts = $stem -split '_ses-' + $selectedRecords.Add([pscustomobject]@{ + SiteID = $f.Directory.Name + Patient = $parts[0] + Session = $parts[1] + }) | Out-Null } -# Copy full annotation folders (simpler and robust) -Copy-Item "$FULL_DATA_PATH/algorithmic_annotations" "$SMOKE_PATH/algorithmic_annotations" -Recurse -ErrorAction SilentlyContinue -Copy-Item "$FULL_DATA_PATH/human_annotations" "$SMOKE_PATH/human_annotations" -Recurse -ErrorAction SilentlyContinue +# Copy only annotation EDFs for the selected smoke records. +foreach ($record in $selectedRecords) { + $algoSource = Join-Path $FULL_DATA_PATH "algorithmic_annotations/$($record.SiteID)/$($record.Patient)_ses-$($record.Session)_caisr_annotations.edf" + $algoTarget = Join-Path $SMOKE_PATH "algorithmic_annotations/$($record.SiteID)/$($record.Patient)_ses-$($record.Session)_caisr_annotations.edf" + if (Test-Path $algoSource) { + New-Item -ItemType Directory -Force -Path (Split-Path $algoTarget) | Out-Null + Copy-Item $algoSource $algoTarget + } + + $humanSource = Join-Path $FULL_DATA_PATH "human_annotations/$($record.SiteID)/$($record.Patient)_ses-$($record.Session)_expert_annotations.edf" + $humanTarget = Join-Path $SMOKE_PATH "human_annotations/$($record.SiteID)/$($record.Patient)_ses-$($record.Session)_expert_annotations.edf" + if (Test-Path $humanSource) { + New-Item -ItemType Directory -Force -Path (Split-Path $humanTarget) | Out-Null + Copy-Item $humanSource $humanTarget + } +} + +# Filter demographics to the copied smoke records. +$env:SMOKE_FULL_DATA_PATH = (Resolve-Path $FULL_DATA_PATH).Path +$env:SMOKE_PATH = (Resolve-Path $SMOKE_PATH).Path +python -c @" +import csv +import os +from pathlib import Path + +full_data = Path(os.environ['SMOKE_FULL_DATA_PATH']) +smoke_path = Path(os.environ['SMOKE_PATH']) + +source_csv = full_data / 'demographics.csv' +target_csv = smoke_path / 'demographics.csv' +phys_root = smoke_path / 'physiological_data' + +selected_records = set() +for edf_path in phys_root.rglob('*.edf'): + site_id = edf_path.parent.name + patient_part, session_part = edf_path.stem.rsplit('_ses-', 1) + selected_records.add((site_id, patient_part, session_part)) + +with source_csv.open('r', newline='', encoding='utf-8') as source_file: + reader = csv.DictReader(source_file) + rows = [ + row for row in reader + if (row['SiteID'], row['BidsFolder'], str(row['SessionID'])) in selected_records + ] + fieldnames = reader.fieldnames + +with target_csv.open('w', newline='', encoding='utf-8') as target_file: + writer = csv.DictWriter(target_file, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(rows) +"@ +Remove-Item Env:SMOKE_FULL_DATA_PATH -ErrorAction SilentlyContinue +Remove-Item Env:SMOKE_PATH -ErrorAction SilentlyContinue Write-Host "Smoke dataset created successfully." \ No newline at end of file diff --git a/scripts/create_smoke.sh b/scripts/create_smoke.sh index 34bd020..9562438 100644 --- a/scripts/create_smoke.sh +++ b/scripts/create_smoke.sh @@ -13,7 +13,7 @@ set -euo pipefail FULL_DATA_PATH="${FULL_DATA_PATH:-data/training_set}" # Override with env var if needed SMOKE_PATH="data/training_smoke" -N_RECORDS="${N_RECORDS:-5}" +N_RECORDS="${N_RECORDS:-10}" echo "Creating smoke dataset..." echo "Source: ${FULL_DATA_PATH}" @@ -22,8 +22,8 @@ echo "Destination: ${SMOKE_PATH}" rm -rf "${SMOKE_PATH}" mkdir -p "${SMOKE_PATH}" -# Copy demographics -cp "${FULL_DATA_PATH}/demographics.csv" "${SMOKE_PATH}/demographics.csv" +selected_records_file="$(mktemp)" +trap 'rm -f "${selected_records_file}"' EXIT # Select first N EDF files while IFS= read -r file_path; do @@ -31,17 +31,63 @@ while IFS= read -r file_path; do target_path="${SMOKE_PATH}/${rel_path}" mkdir -p "$(dirname "${target_path}")" cp "${file_path}" "${target_path}" + stem="$(basename "${file_path}" .edf)" + patient_part="${stem%_ses-*}" + session_part="${stem##*_ses-}" + site_id="$(basename "$(dirname "${file_path}")")" + printf '%s,%s,%s\n' "${site_id}" "${patient_part}" "${session_part}" >> "${selected_records_file}" done < <( find "${FULL_DATA_PATH}/physiological_data" -type f -name "*.edf" | sort | head -n "${N_RECORDS}" ) -# Copy full annotation folders (simpler and robust) -if [[ -d "${FULL_DATA_PATH}/algorithmic_annotations" ]]; then - cp -R "${FULL_DATA_PATH}/algorithmic_annotations" "${SMOKE_PATH}/algorithmic_annotations" -fi +# Copy only annotation EDFs for the selected smoke records. +while IFS=',' read -r site_id patient_part session_part; do + algo_source="${FULL_DATA_PATH}/algorithmic_annotations/${site_id}/${patient_part}_ses-${session_part}_caisr_annotations.edf" + algo_target="${SMOKE_PATH}/algorithmic_annotations/${site_id}/${patient_part}_ses-${session_part}_caisr_annotations.edf" + if [[ -f "${algo_source}" ]]; then + mkdir -p "$(dirname "${algo_target}")" + cp "${algo_source}" "${algo_target}" + fi -if [[ -d "${FULL_DATA_PATH}/human_annotations" ]]; then - cp -R "${FULL_DATA_PATH}/human_annotations" "${SMOKE_PATH}/human_annotations" -fi + human_source="${FULL_DATA_PATH}/human_annotations/${site_id}/${patient_part}_ses-${session_part}_expert_annotations.edf" + human_target="${SMOKE_PATH}/human_annotations/${site_id}/${patient_part}_ses-${session_part}_expert_annotations.edf" + if [[ -f "${human_source}" ]]; then + mkdir -p "$(dirname "${human_target}")" + cp "${human_source}" "${human_target}" + fi +done < "${selected_records_file}" + +# Filter demographics to the copied smoke records. +python - <<'PY' +import csv +from pathlib import Path + +full_data = Path("data/training_set") +smoke_path = Path("data/training_smoke") + +source_csv = full_data / "demographics.csv" +target_csv = smoke_path / "demographics.csv" +phys_root = smoke_path / "physiological_data" + +selected_records = set() +for edf_path in phys_root.rglob("*.edf"): + site_id = edf_path.parent.name + stem = edf_path.stem + patient_part, session_part = stem.rsplit("_ses-", 1) + selected_records.add((site_id, patient_part, session_part)) + +with source_csv.open("r", newline="", encoding="utf-8") as source_file: + reader = csv.DictReader(source_file) + rows = [ + row for row in reader + if (row["SiteID"], row["BidsFolder"], str(row["SessionID"])) in selected_records + ] + fieldnames = reader.fieldnames + +with target_csv.open("w", newline="", encoding="utf-8") as target_file: + writer = csv.DictWriter(target_file, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(rows) +PY echo "Smoke dataset created successfully." From a550325c5fe6af6a786dda7766bbd153252cf2ed Mon Sep 17 00:00:00 2001 From: dcajal Date: Fri, 27 Mar 2026 22:52:10 +0100 Subject: [PATCH 31/93] Update smoke dataset configuration to use 5 subjects by default and enhance feature extraction process --- docs/03_smoke_dataset.md | 4 +-- scripts/create_smoke.ps1 | 2 +- team_code.py | 78 +++++++++++++++++++++++++++++----------- 3 files changed, 61 insertions(+), 23 deletions(-) diff --git a/docs/03_smoke_dataset.md b/docs/03_smoke_dataset.md index 7c0e3d7..ae4ee23 100644 --- a/docs/03_smoke_dataset.md +++ b/docs/03_smoke_dataset.md @@ -2,7 +2,7 @@ Entrenar con el dataset completo tarda aproximadamente 30–40 minutos con el modelo de ejemplo. -Para desarrollo utilizamos un dataset reducido (10 sujetos por defecto). +Para desarrollo utilizamos un dataset reducido (5 sujetos por defecto). Este documento describe cuándo y por qué usar smoke. Los comandos de ejecución están centralizados en `docs/04_run_script.md`. @@ -11,7 +11,7 @@ Los comandos de ejecución están centralizados en `docs/04_run_script.md`. ## Qué incluye -- Muestra reducida del dataset (10 sujetos por defecto) +- Muestra reducida del dataset (5 sujetos por defecto) - Estructura compatible con el flujo oficial del proyecto - Directorio de salida en `data/training_smoke/` - `demographics.csv` filtrado para que solo incluya los registros copiados al smoke diff --git a/scripts/create_smoke.ps1 b/scripts/create_smoke.ps1 index d66aaff..c8f45dd 100644 --- a/scripts/create_smoke.ps1 +++ b/scripts/create_smoke.ps1 @@ -10,7 +10,7 @@ $FULL_DATA_PATH = "data/training_set" # <-- CHANGE THIS IF NEEDED $SMOKE_PATH = "data/training_smoke" -$N_RECORDS = 10 +$N_RECORDS = 5 Write-Host "Creating smoke dataset..." Write-Host "Source: $FULL_DATA_PATH" diff --git a/team_code.py b/team_code.py index 7296cc1..c23df14 100644 --- a/team_code.py +++ b/team_code.py @@ -22,8 +22,8 @@ from tqdm import tqdm from helper_code import * -from src.resp_processing import processResp -from src.eeg_processing import processEEG +from src.resp_processing import RESP_FEATURE_LENGTH, processResp +from src.eeg_processing import EEG_FEATURE_LENGTH, processEEG ################################################################################ # Path & Constant Configuration (Added for Robustness) ################################################################################ @@ -67,6 +67,50 @@ def get_rename_rules(csv_path): return rename_rules +def _coerce_feature_vector(features): + vector = np.asarray(features, dtype=np.float32).reshape(-1) + return np.nan_to_num(vector, nan=0.0, posinf=0.0, neginf=0.0) + + +def _extract_optional_features(extractor, expected_length, *args, **kwargs): + vector = _coerce_feature_vector(extractor(*args, **kwargs)) + if vector.size != expected_length: + raise ValueError( + f"{extractor.__name__} returned {vector.size} features; expected {expected_length}." + ) + return vector + + +def extract_extended_physiological_features(physiological_data, physiological_fs, csv_path=DEFAULT_CSV_PATH): + base_features = _coerce_feature_vector( + extract_physiological_features(physiological_data, physiological_fs, csv_path=csv_path) + ) + + try: + resp_features = _extract_optional_features( + processResp, + RESP_FEATURE_LENGTH, + physiological_data, + physiological_fs, + csv_path=csv_path, + ) + except Exception: + resp_features = np.zeros(RESP_FEATURE_LENGTH, dtype=np.float32) + + try: + eeg_features = _extract_optional_features( + processEEG, + EEG_FEATURE_LENGTH, + physiological_data, + physiological_fs, + csv_path=csv_path, + ) + except Exception: + eeg_features = np.zeros(EEG_FEATURE_LENGTH, dtype=np.float32) + + return np.hstack([base_features, resp_features, eeg_features]).astype(np.float32) + + def process_training_record(record, data_folder, demographics_cache, diagnosis_cache, csv_path): patient_id = record[HEADERS['bids_folder']] site_id = record[HEADERS['site_id']] @@ -86,30 +130,22 @@ def process_training_record(record, data_folder, demographics_cache, diagnosis_c return patient_id, None, None, f"Missing physiological data for {patient_id}. Skipping..." physiological_data, physiological_fs = load_signal_data(physiological_data_file) - physiological_features = extract_physiological_features( - physiological_data, - physiological_fs, - csv_path=csv_path - ) - resp_features = processResp( + physiological_features = extract_extended_physiological_features( physiological_data, physiological_fs, csv_path=csv_path ) - eeg_features = processEEG( - physiological_data, - physiological_fs, - csv_path=csv_path - ) - physiological_features = np.concatenate([physiological_features, resp_features, eeg_features], axis = 1) algorithmic_annotations_file = os.path.join( data_folder, ALGORITHMIC_ANNOTATIONS_SUBFOLDER, site_id, f"{patient_id}_ses-{session_id}_caisr_annotations.edf" ) - algorithmic_annotations, algorithmic_fs = load_signal_data(algorithmic_annotations_file) - algorithmic_features = extract_algorithmic_annotations_features(algorithmic_annotations) + if os.path.exists(algorithmic_annotations_file): + algorithmic_annotations, _ = load_signal_data(algorithmic_annotations_file) + algorithmic_features = extract_algorithmic_annotations_features(algorithmic_annotations) + else: + algorithmic_features = np.zeros(12, dtype=np.float32) label = diagnosis_cache.get(patient_id) @@ -215,6 +251,9 @@ def train_model(data_folder, model_folder, verbose, csv_path=DEFAULT_CSV_PATH): features = np.asarray(features, dtype=np.float32) labels = np.asarray(labels, dtype=bool) + if features.size == 0 or features.ndim != 2 or features.shape[0] == 0: + raise ValueError('No valid training samples were extracted. Review feature extraction logs for the skipped records.') + # Train the models on the features. if verbose: print('Training the model on the data...') @@ -294,10 +333,9 @@ def run_model(model, record, data_folder, verbose): if os.path.exists(phys_file): phys_data, phys_fs = load_signal_data(phys_file) # Ensure csv_path is accessible or defined - physiological_features = extract_physiological_features(phys_data, phys_fs) + physiological_features = extract_extended_physiological_features(phys_data, phys_fs) else: - # Fallback to zeros if file is missing (length 49) - physiological_features = np.zeros(49) + physiological_features = np.zeros(49 + RESP_FEATURE_LENGTH + EEG_FEATURE_LENGTH, dtype=np.float32) # Load Algorithmic Annotations algo_file = os.path.join(data_folder, ALGORITHMIC_ANNOTATIONS_SUBFOLDER, site_id, f"{patient_id}_ses-{session_id}_caisr_annotations.edf") @@ -306,7 +344,7 @@ def run_model(model, record, data_folder, verbose): algorithmic_features = extract_algorithmic_annotations_features(algo_data) else: # Fallback to zeros (length 12) - algorithmic_features = np.zeros(12) + algorithmic_features = np.zeros(12, dtype=np.float32) features = np.hstack([demographic_features, physiological_features, algorithmic_features]).reshape(1, -1) From ca1f9e6c42c2bda54e1284c0dc603099ee2dab53 Mon Sep 17 00:00:00 2001 From: dcajal Date: Fri, 27 Mar 2026 22:55:20 +0100 Subject: [PATCH 32/93] Update smoke dataset configuration to use 5 subjects by default --- scripts/create_smoke.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/create_smoke.sh b/scripts/create_smoke.sh index 9562438..88765ec 100644 --- a/scripts/create_smoke.sh +++ b/scripts/create_smoke.sh @@ -13,7 +13,7 @@ set -euo pipefail FULL_DATA_PATH="${FULL_DATA_PATH:-data/training_set}" # Override with env var if needed SMOKE_PATH="data/training_smoke" -N_RECORDS="${N_RECORDS:-10}" +N_RECORDS="${N_RECORDS:-5}" echo "Creating smoke dataset..." echo "Source: ${FULL_DATA_PATH}" From e1cc32e9f1e80b70681ec668913c9b9f9f1d3373 Mon Sep 17 00:00:00 2001 From: dcajal Date: Fri, 27 Mar 2026 22:58:24 +0100 Subject: [PATCH 33/93] Refactor EEG and respiratory signal processing functions to enhance feature extraction and improve data handling --- src/eeg_processing.py | 209 +++++++++++++++------------- src/lib/Resp_features.py | 2 +- src/lib/peakedness.py | 103 +++++++------- src/resp_processing.py | 291 ++++++++++++++++++++++----------------- 4 files changed, 327 insertions(+), 278 deletions(-) diff --git a/src/eeg_processing.py b/src/eeg_processing.py index 04147c6..cf95237 100644 --- a/src/eeg_processing.py +++ b/src/eeg_processing.py @@ -29,7 +29,7 @@ El módulo depende de `numpy`, `pandas`, `matplotlib`, `plotly` y de las utilidades definidas en `lib/helper_code` y `lib/EEG_functions`. """ -import sys +import sys import os import pandas as pd import numpy as np @@ -39,103 +39,114 @@ sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -def processEEG(physiological_data, physiological_fs, csv_path): +EEG_FEATURE_NAMES = [ + 'EEG_Channel_Count', + 'EEG_Rel_Delta', + 'EEG_Rel_Theta', + 'EEG_Rel_Alpha', + 'EEG_Rel_Sigma', + 'EEG_Rel_Beta', + 'EEG_Theta_Alpha_Ratio', + 'EEG_Hjorth_Complexity', +] +EEG_FEATURE_LENGTH = len(EEG_FEATURE_NAMES) + + +def _normalize_label(text): + normalized = ''.join(ch if ch.isalnum() else ' ' for ch in str(text).lower()) + return ' '.join(normalized.split()) + + +def _split_aliases(raw_aliases): + return {_normalize_label(alias) for alias in str(raw_aliases).split(';') if alias} + + +def _build_eeg_aliases(channels): + eeg_rows = channels[channels['Category'].eq('eeg')] + aliases = set() + for _, row in eeg_rows.iterrows(): + aliases.update(_split_aliases(row['Channel_Names'])) + return aliases + + +def _resample_signal(signal, fs, target_fs): + signal = np.asarray(signal, dtype=float) + if signal.size == 0: + return signal, target_fs + if fs == target_fs: + return signal, target_fs + + duration = signal.size / fs + target_samples = max(1, int(round(duration * target_fs))) + time_original = np.linspace(0, duration, signal.size) + time_target = np.linspace(0, duration, target_samples) + return np.interp(time_target, time_original, signal), target_fs + + +def _extract_channel_metrics(signal, fs): + signal = np.nan_to_num(np.asarray(signal, dtype=float), nan=0.0, posinf=0.0, neginf=0.0) + if signal.size < max(int(fs * 30), 2): + return None + + if fs != 200: + signal, fs = _resample_signal(signal, fs, 200) + + filtered = EEG_functions.butter_bandpass_filter(signal, lowcut=0.3, highcut=35, fs=fs, order=4) + signal_std = np.std(filtered) + if signal_std == 0 or not np.isfinite(signal_std): + return None + + normalized = (filtered - np.mean(filtered)) / signal_std + epochs = EEG_functions.create_epochs(normalized, fs, epoch_duration=30) + if epochs.size == 0: + return None + + band_powers, complexities = EEG_functions.extract_band_powers(epochs, fs, win_len=15) + if len(band_powers) > 60: + band_powers = band_powers.iloc[60:] + complexities = complexities.iloc[60:] + if band_powers.empty: + return None + + total_power = band_powers.sum(axis=1).replace(0, np.nan) + relative_powers = band_powers.div(total_power, axis=0).replace([np.inf, -np.inf], np.nan).fillna(0.0).mean() + alpha_power = float(relative_powers.get('Alpha', 0.0)) + theta_power = float(relative_powers.get('Theta', 0.0)) + theta_alpha_ratio = theta_power / alpha_power if alpha_power > 0 else 0.0 + + complexity_mean = float( + complexities['Hjorth_Complexity'].replace([np.inf, -np.inf], np.nan).fillna(0.0).mean() + ) if 'Hjorth_Complexity' in complexities else 0.0 + + return np.array([ + float(relative_powers.get('Delta', 0.0)), + theta_power, + alpha_power, + float(relative_powers.get('Sigma', 0.0)), + float(relative_powers.get('Beta', 0.0)), + float(theta_alpha_ratio), + complexity_mean, + ], dtype=np.float32) + +def processEEG(physiological_data, physiological_fs, csv_path): channels = pd.read_csv(csv_path) - selectEEG = channels[channels['Category'].isin(['eeg'])] - - for label in original_labels: - fs = physiological_fs[label] - - data = [] - original_labels = list(physiological_data.keys()) - - # Listar canales para identificar los de interés (ej: C3-M2, O1-M2) - HayEEG = False - for i, label in enumerate(original_labels): - for index in selectEEG.index: - if label.lower() in selectEEG['Channel_Names'][index].lower(): - print(f"Canal seleccionado: {label}") - labels.append(label) - HayEEG = True - break - - results = [] - labels2 = [] - if HayEEG: - Bipolar = pd.DataFrame() - if all(label in labels for label in ["F3", "F4", "M1", "M2"]): - Bipolar['F3-M2'] = physiological_data["F3"] - physiological_data["M2"] - Bipolar['F4-M1'] = physiological_data["F4"] - physiological_data["M1"] - labels2.append('F3-M2') - labels2.append('F4-M1') - if all(label in labels for label in ["C3", "C4", "M1", "M2"]): - Bipolar['C3-M2'] = physiological_data["C3"] - physiological_data["M2"] - Bipolar['C4-M1'] = physiological_data["C4"] - physiological_data["M1"] - labels2.append('C3-M2') - labels2.append('C4-M1') - if all(label in labels for label in ["O2", "O1", "M1", "M2"]): - Bipolar['O2-M2'] = physiological_data["O1"] - physiological_data["M2"] - Bipolar['O1-M1'] = physiological_data["O2"] - physiological_data["M1"] - labels2.append('O1-M1') - labels2.append('O2-M2') - # print(f"Archivo {file} tiene ECG, RESP y EEG. Se procesará con canales bipolares.") - - if not Bipolar.empty: - labels = [] - for col in Bipolar.columns: - # print(f"Archivo: {file}, Canal: {col}, Frecuencia de muestreo: {sig.sampling_frequency} Hz, Duración: {len(Bipolar[col])/sig.sampling_frequency:.2f} segundos") - fs = physiological_data["M2"].sampling_frequency # Asumimos que todos los canales tienen la misma frecuencia de muestreo - fil = EEG_functions.butter_bandpass_filter(Bipolar[col], lowcut=0.3, highcut=35, fs=fs, order=4) - norm = (fil-np.mean(fil))/np.std(fil) - - data.append(norm) # Restar la media para centrar la señal - labels.append(col) - # columns = Bipolar.columns.tolist() - else: - labels = [] - for l in labels: - # print(f"Archivo: {file}, Canal: {sig.label}, Frecuencia de muestreo: {sig.sampling_frequency} Hz, Duración: {len(sig.data)/sig.sampling_frequency:.2f} segundos") - fs = physiological_fs[l] - fil = EEG_functions.butter_bandpass_filter(physiological_data[l], lowcut=0.3, highcut=35, fs=fs, order=4) - norm = (fil-np.mean(fil))/np.std(fil) - labels.append(l) - data.append(norm) # Restar la media para centrar la señal - - # columns = [selEEG[i][1].label for i in range(len(selEEG))] - - - for i, elec in enumerate(labels): - epoch_length = 30 # Duración de cada época en segundos - if Bipolar.empty: - fs = physiological_fs[l] - else: - fs = physiological_fs['M1'] - - if fs != 200: - # print(f"Warning: Sampling frequency for channel {elec} in file {file} is {fs} Hz, expected 200 Hz. Check the data.") - duration = len(data[i]) / fs - time_original = np.linspace(0, duration, len(data[i])) - - num_samples_target = int(duration * 200 ) - time_target = np.linspace(0, duration, num_samples_target) - data[i] = np.interp(time_target, time_original, data[i]) - fs = 200 # Update fs to the target sampling frequency after resampling - - epochs = EEG_functions.create_epochs(data[i], fs, epoch_duration=epoch_length) - - band_powers, complexities = EEG_functions.extract_band_powers(epochs, fs, win_len=15) - band_powers = band_powers.iloc[60:] # Eliminar las primeras 60 épocas (30 min) para evitar el tiempo despierto al inicio de la grabación - - - # Ejecución - patient_summar = EEG_functions.get_patient_profile(band_powers) - - d = complexities.iloc[:].std().to_dict() - results.append({ - 'Channel': elec, - **d, - **patient_summar - }) - df_results = np.array(results) - return df_results \ No newline at end of file + eeg_aliases = _build_eeg_aliases(channels) + channel_metrics = [] + + for label, signal in physiological_data.items(): + if label not in physiological_fs: + continue + if _normalize_label(label) not in eeg_aliases: + continue + + metrics = _extract_channel_metrics(signal, physiological_fs[label]) + if metrics is not None: + channel_metrics.append(metrics) + + if not channel_metrics: + return np.zeros(EEG_FEATURE_LENGTH, dtype=np.float32) + + stacked = np.vstack(channel_metrics) + aggregated = np.mean(stacked, axis=0) + return np.hstack([np.array([len(channel_metrics)], dtype=np.float32), aggregated]).astype(np.float32) \ No newline at end of file diff --git a/src/lib/Resp_features.py b/src/lib/Resp_features.py index 216b3ab..60ff3bf 100644 --- a/src/lib/Resp_features.py +++ b/src/lib/Resp_features.py @@ -176,7 +176,7 @@ def Metrics_per_segment(Data): section = sel_sujeto[secc].values section = section[~np.isnan(section)] - hat_Br, Sk_Br, t_aver = peakedness_application(section, stage=secc, plotflag = False, subjet= subjet) + hat_Br, Sk_Br, t_aver, _ = peakedness_application(section, stage=secc, plotflag = False, subjet= subjet) # print(f"Subjet: {subjet}, section: {secc} hat_Br: {hat_Br}, Sk_Br: {Sk_Br}") # Ajuste lineal diff --git a/src/lib/peakedness.py b/src/lib/peakedness.py index 9c47f96..1bda09c 100644 --- a/src/lib/peakedness.py +++ b/src/lib/peakedness.py @@ -6,17 +6,13 @@ from time import time import os -try: - import matplotlib.pyplot as plt -except ModuleNotFoundError: - plt = None - -try: - import plotly.graph_objs as go - from plotly import subplots -except ModuleNotFoundError: - go = None - subplots = None +def _safe_ratio(numerator, denominator, default=0.0): + if denominator is None or not np.isfinite(denominator) or denominator == 0: + return default + value = numerator / denominator + if np.isfinite(value): + return value + return default def setParamFr(Setup): if 'DT' not in Setup.keys(): @@ -207,7 +203,9 @@ def init_module(kk,vars,param, plotflag): # Peakedness # print(S[Omega]) - Pkl = 100*sum(S[Omega_p])/sum(S[Omega]) + band_power = np.sum(S[Omega]) + peaky_power = np.sum(S[Omega_p]) + Pkl = 100*_safe_ratio(peaky_power, band_power) if Pkl >= ksi_p: Xkl[k,l] = 1 @@ -258,7 +256,8 @@ def init_module(kk,vars,param, plotflag): j_pk = j_pk[~j_del] # Cost function for deviation from previous fr and maximum power - C_a = 1-np.transpose(pk)/np.max(S) + max_s = np.max(S) + C_a = 1-_safe_ratio(np.transpose(pk), max_s, default=np.zeros_like(pk, dtype=float)) fr_prev = vars["bar_fr"][np.max(kk,0)] C_f = abs(f[j_pk[:]]-fr_prev)/(2*d) # C_f = abs(f(i_pk(:))-fr_prev)/(Omega_r(2)-Omega(1)); @@ -273,13 +272,13 @@ def init_module(kk,vars,param, plotflag): # Save in vars # vars["bar_fr"][kk] = f[fj] - if plotflag: - if plt is None: - raise ModuleNotFoundError("matplotlib is required when plotflag=True") - plt.plot(f, averS) - plt.plot(f[fj], averS[fj], '-') - plt.title('Initialization - Averaged Spectrum') - plt.show() + # if plotflag: + # if plt is None: + # raise ModuleNotFoundError("matplotlib is required when plotflag=True") + # plt.plot(f, averS) + # plt.plot(f[fj], averS[fj], '-') + # plt.title('Initialization - Averaged Spectrum') + # plt.show() return vars # # No spectra fulfill the initialization @@ -323,11 +322,15 @@ def compute_Xkl( Skl, f, bar_fr, O, ksi_p, ksi_a, d): S = Skl[:, O[k], l] # % Define peakedness based on the power concentration - Pkl = 100*sum(S[Omega_p])/sum(S[Omega]) + band_power = np.sum(S[Omega]) + peaky_power = np.sum(S[Omega_p]) + Pkl = 100*_safe_ratio(peaky_power, band_power) # % Define peakedness based on the absolute maximum # print(max(S)) - Akl = 100*max(S[Omega])/max(S) + max_s = np.max(S) + max_band = np.max(S[Omega]) if np.any(Omega) else 0.0 + Akl = 100*_safe_ratio(max_band, max_s) # % If the spectrum is concidered peaky by both conditions, mark as # % peaky if np.bitwise_and(Pkl >= ksi_p, Akl >= ksi_a): @@ -474,7 +477,9 @@ def peakednessCost(signals, ts, fs, Setup = {}, title = "", storeGraph = False, vars["t_aver"] = vars["t_orig"][N:-N] if vars["t_aver"].shape[0] == 0: print("No hay tiempo para promediar") - return np.nan, np.nan, np.nan + empty_spectra = np.empty((vars["f"].shape[0], 0)) + empty_used = np.empty((0, vars["L"])) + return np.array([]), empty_spectra, np.array([]), empty_used vars["Sk"] = np.empty((vars["f"].shape[0], vars["t_aver"].shape[0])) vars["Sk"][:] = np.nan vars["bar_fr"] = np.empty(( vars["t_aver"].shape[0])) @@ -615,33 +620,33 @@ def peakednessCost(signals, ts, fs, Setup = {}, title = "", storeGraph = False, vars["t_orig"] = vars["t_orig"] + ts1 t_fin = time() - if plotflag: - if go is None or subplots is None: - raise ModuleNotFoundError("plotly is required when plotflag=True") + # if plotflag: + # if go is None or subplots is None: + # raise ModuleNotFoundError("plotly is required when plotflag=True") - fig = subplots.make_subplots(rows=2,shared_xaxes=True, subplot_titles=('Peak-condition averaged EDR Spectra in '+title,"EDR/RESP signals"), row_heights=[0.7, 0.3]) + # fig = subplots.make_subplots(rows=2,shared_xaxes=True, subplot_titles=('Peak-condition averaged EDR Spectra in '+title,"EDR/RESP signals"), row_heights=[0.7, 0.3]) - fig.add_heatmap(x=vars["t_aver"], y=vars["f"], z=vars["Sk"]/np.max(vars["Sk"]),colorscale='jet',colorbar=dict(orientation='h')) - fig.update_layout(coloraxis_showscale=False) - fig.add_trace(go.Line(x=vars["t_aver"], y=vars["hat_fr"],name = 'f\u0302_r(k)'), row = 1, col=1) - fig.add_trace(go.Line(x=vars["t_aver"],y=vars["bar_fr"],name= 'f\u0304_r(k)'), row = 1, col=1) + # fig.add_heatmap(x=vars["t_aver"], y=vars["f"], z=vars["Sk"]/np.max(vars["Sk"]),colorscale='jet',colorbar=dict(orientation='h')) + # fig.update_layout(coloraxis_showscale=False) + # fig.add_trace(go.Line(x=vars["t_aver"], y=vars["hat_fr"],name = 'f\u0302_r(k)'), row = 1, col=1) + # fig.add_trace(go.Line(x=vars["t_aver"],y=vars["bar_fr"],name= 'f\u0304_r(k)'), row = 1, col=1) - fig.add_trace(go.Line(x=vars["t_aver"],y=vars["used"]), row = 1, col=1) - # fig.axis([vars.t_aver(1), vars.t_aver(end), vars.f(1), vars.f(end)]) - for i in range(signals.shape[1]): - fig.add_trace(go.Line(x=ts+ts1,y=signals[:,i],name = 'Signal '+str(i)), row = 2, col=1) - - fig.update_layout(coloraxis_showscale=False) - fig.update_yaxes(title_text="f (Hz)", row=1, col=1) - fig.update_yaxes(title_text="(n.u.)", row=2, col=1) - fig.update_xaxes(title_text="time (s)", row=2, col=1) - if storeGraph: - os.makedirs("Graphs/Peakedness/"+str(subjet), exist_ok=True) - # fig.write_image(os.path.join("Graphs", "Peakedness",str(subjet),title+".png")) - fig.write_html(os.path.join("Graphs", "Peakedness",str(subjet),title+".html")) - # fig.write_image() - else: - fig.show() - - return vars["hat_fr"], vars["Sk"], vars["t_aver"] + # fig.add_trace(go.Line(x=vars["t_aver"],y=vars["used"]), row = 1, col=1) + # # fig.axis([vars.t_aver(1), vars.t_aver(end), vars.f(1), vars.f(end)]) + # for i in range(signals.shape[1]): + # fig.add_trace(go.Line(x=ts+ts1,y=signals[:,i],name = 'Signal '+str(i)), row = 2, col=1) + + # fig.update_layout(coloraxis_showscale=False) + # fig.update_yaxes(title_text="f (Hz)", row=1, col=1) + # fig.update_yaxes(title_text="(n.u.)", row=2, col=1) + # fig.update_xaxes(title_text="time (s)", row=2, col=1) + # if storeGraph: + # os.makedirs("Graphs/Peakedness/"+str(subjet), exist_ok=True) + # # fig.write_image(os.path.join("Graphs", "Peakedness",str(subjet),title+".png")) + # fig.write_html(os.path.join("Graphs", "Peakedness",str(subjet),title+".html")) + # # fig.write_image() + # else: + # fig.show() + + return vars["hat_fr"], vars["Sk"], vars["t_aver"], vars["used"] # return vars["hat_fr"], vars["Sk"], vars["bar_fr"],vars["t_aver"], vars["f"], vars["used"] \ No newline at end of file diff --git a/src/resp_processing.py b/src/resp_processing.py index c158d2b..c1e7845 100644 --- a/src/resp_processing.py +++ b/src/resp_processing.py @@ -1,138 +1,171 @@ from .lib import Resp_features -import sys +import sys import os import pandas as pd import numpy as np sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -def processResp(physiological_data, physiological_fs, csv_path): +RESP_CHANNEL_GROUPS = ("Abdomen", "Chest", "Nasal", "Flow") +RESP_FEATURE_NAMES = [ + f"{group}_Peakedness_{metric}" + for group in RESP_CHANNEL_GROUPS + for metric in ("Max", "Min", "Mean", "Median", "Std") +] + [ + "SpO2_Max", + "SpO2_Min", + "SpO2_Mean", + "SpO2_Std", + "CET90", + "ODI_Mean", + "ODI_deepness", +] +RESP_FEATURE_LENGTH = len(RESP_FEATURE_NAMES) + + +def _normalize_label(text): + normalized = ''.join(ch if ch.isalnum() else ' ' for ch in str(text).lower()) + return ' '.join(normalized.split()) + + +def _split_aliases(raw_aliases): + return {_normalize_label(alias) for alias in str(raw_aliases).split(';') if alias} + + +def _build_resp_alias_groups(channels): + resp_rows = channels[channels['Category'].eq('resp')].reset_index(drop=True) + if len(resp_rows) < 7: + return {} + return { + 'Abdomen': _split_aliases(resp_rows.iloc[0]['Channel_Names']), + 'Chest': _split_aliases(resp_rows.iloc[1]['Channel_Names']), + 'Nasal': _split_aliases(resp_rows.iloc[2]['Channel_Names']), + 'Flow': _split_aliases(resp_rows.iloc[3]['Channel_Names']), + 'SpO2': _split_aliases(resp_rows.iloc[6]['Channel_Names']), + } + + +def _find_resp_group(label, alias_groups): + normalized = _normalize_label(label) + for group_name, aliases in alias_groups.items(): + if normalized in aliases: + return group_name + return None + + +def _resample_signal(signal, fs, target_fs): + signal = np.asarray(signal, dtype=float) + if signal.size == 0: + return signal, target_fs + if fs == target_fs: + return signal, target_fs + + duration = signal.size / fs + target_samples = max(1, int(round(duration * target_fs))) + time_original = np.linspace(0, duration, signal.size) + time_target = np.linspace(0, duration, target_samples) + return np.interp(time_target, time_original, signal), target_fs + + +def _compute_resp_quality(used, hat_br): + used_array = np.asarray(used, dtype=float) + if used_array.size: + quality = float(np.nanmean(used_array)) + if np.isfinite(quality): + return quality + hat_br = np.asarray(hat_br, dtype=float) + if hat_br.size == 0: + return 0.0 + return float(np.mean(np.isfinite(hat_br))) + + +def _summarize_peakedness(hat_br): + finite_values = np.asarray(hat_br, dtype=float) + finite_values = finite_values[np.isfinite(finite_values)] + if finite_values.size == 0: + return None + return { + 'Max': float(np.max(finite_values)), + 'Min': float(np.min(finite_values)), + 'Mean': float(np.mean(finite_values)), + 'Median': float(np.median(finite_values)), + 'Std': float(np.std(finite_values)), + } + +def _summarize_spo2(data, fs): + if data.size == 0: + return {} + working = np.asarray(data, dtype=float).copy() + if np.nanmax(working) < 2: + working = np.round((working / 1.055) * 100) + + desaturation_mask = working.copy() + threshold = 0.7 + for index, value in enumerate(working): + if value < threshold: + start = int(max(0, index - fs * 2)) + end = int(min(working.size, index + fs * 2)) + desaturation_mask[start:end] = np.nan + + cet90 = float(np.count_nonzero(desaturation_mask < 90) / max(working.size, 1)) + valid = desaturation_mask[np.isfinite(desaturation_mask)] + if valid.size == 0: + return {'CET90': cet90} + + odi_mean, odi_deepness = Resp_features.ODI_application(desaturation_mask, fs, plotflag=False, subjet=1) + return { + 'SpO2_Max': float(np.max(valid)), + 'SpO2_Min': float(np.min(valid)), + 'SpO2_Mean': float(np.mean(valid)), + 'SpO2_Std': float(np.std(valid)), + 'CET90': cet90, + 'ODI_Mean': float(odi_mean), + 'ODI_deepness': float(odi_deepness), + } + + +def processResp(physiological_data, physiological_fs, csv_path): channels = pd.read_csv(csv_path) - selectResp = channels[channels['Category'].isin(['resp'])] - - resultados = {} - UsedFlow = 0 - UsedChest = 0 - UsedAbdomen = 0 - UsedSpO2 = 0 - UsedNasal = 0 - UsedCepap = 0 - - data = [] - original_labels = list(physiological_data.keys()) - - for label in original_labels: - fs = physiological_fs[label] - sig = physiological_data[label] - if fs != 25: - duration = len(sig) / fs - time_original = np.linspace(0, duration, len(sig)) - num_samples_target = int(duration * 25 ) - time_target = np.linspace(0, duration, num_samples_target) - data = np.interp(time_target, time_original, sig) - fs = 25 # Update fs to the target sampling frequency after resampling - else: - data = sig - - # Check nan in sig.data - if np.isnan(sig).any(): - print(f"Warning: NaN values found in signal data for {label}. Filling NaNs with zeros.") - data = np.nan_to_num(data) - - name = "" - if label.lower() not in selectResp['Channel_Names'][34].lower(): - d = Resp_features.peakedness_application(data, stage=label, plotflag = False, subjet =label) - if label.lower() in selectResp['Channel_Names'][28].lower(): - name = "Chest" - # EFFORT RESPIRATORY Chest - elif label.lower() in selectResp['Channel_Names'][29].lower(): - # EFFORT RESPIRATORY Abdomen - name = "Abdomen" - elif label.lower() in selectResp['Channel_Names'][30].lower(): - # RESPIRATORY NASAL - name = "Nasal" - elif label.lower() in selectResp['Channel_Names'][31].lower(): - # RESPIRATORY FLOW - name = "Flow" - elif label.lower() in selectResp['Channel_Names'][32].lower(): - # CEPAP - if np.all(data == 0) or np.std(data) < 5: - print(f"Warning: All values in the signal data for {label} are zero. Skipping feature extraction for this channel.") - else: - name = "" - elif label.lower() in selectResp['Channel_Names'][33].lower(): - # CEPAP - name = "" - - if name != "": - DSinNan = d[0][~np.isnan(d[0])] # Eliminar NaN antes de calcular min y max - if len(DSinNan) != 0: - maximo = DSinNan.max() - minimo = DSinNan.min() - media = np.mean(DSinNan) - mediana = np.median(DSinNan) - std = DSinNan.std() - write = False - if name == "Nasal" and UsedNasal< d[-1]: - UsedNasal = d[-1] - write = True - elif name == "Chest" and UsedChest< d[-1]: - UsedChest = d[-1] - write = True - elif name == "Abdomen" and UsedAbdomen< d[-1]: - UsedAbdomen = d[-1] - write = True - elif name == "Flow" and UsedFlow< d[-1]: - UsedFlow = d[-1] - write = True - elif name == "SpO2" and UsedSpO2< d[-1]: - UsedSpO2 = d[-1] - write = True - elif name == "CEPAP" and UsedCepap < d[-1]: - UsedCepap = d[-1] - write = True - if write: - resultados.update({ - name+"_Peakedness_Max": maximo, - name+"_Peakedness_Min": minimo, - name+"_Peakedness_Mean": media, - name+"_Peakedness_Median": mediana, - name+"_Peakedness_Std": std - }) - - elif label.lower() in selectResp['Channel_Names'][34].lower(): - #O2 SATURATION - if np.max(data) < 2: - data = np.round((data/1.055)*100) - - lim = 0.7 - # Quitar los valores por debajo de lim y sus 10 valores anteriores y posteriores para quedarnos solo con los eventos de desaturación - dataReal = data.copy() - for i in range(len(data)): - if data[i] < lim: - start = int(max(0, i-fs*2)) - end = int(min(len(data), i+fs*2)) - dataReal[start:end] = np.nan # Marcar los valores por debajo del límite y sus alrededores como NaN - - CET90 = dataReal[dataReal < 90] - # CET90SinNan = CET90[~np.isnan(CET90)] # Eliminar NaN antes de calcular min y max - CET90 = len(CET90)/len(data) - dataRealSinNan = dataReal[~np.isnan(dataReal)] # Eliminar NaN antes de calcular min y max - if len(dataRealSinNan)>0: - maximo = dataRealSinNan.max() - minimo = dataRealSinNan.min() - std = dataRealSinNan.std() - media = dataRealSinNan.mean() - ODI_mean, ODI_deepness = Resp_features.ODI_application(dataReal, fs, plotflag=False, subjet=1) - - resultados.update({"SpO2_Max": maximo, - "SpO2_Min": minimo, - "SpO2_Mean": media, - "SpO2_Std": std, - "CET90": CET90, - "ODI_Mean": ODI_mean, - "ODI_deepness": ODI_deepness, - }) - - return np.array(resultados) + alias_groups = _build_resp_alias_groups(channels) + results = {feature_name: 0.0 for feature_name in RESP_FEATURE_NAMES} + best_quality = {group_name: -np.inf for group_name in RESP_CHANNEL_GROUPS} + + for label, signal in physiological_data.items(): + if label not in physiological_fs: + continue + + group_name = _find_resp_group(label, alias_groups) + if group_name is None: + continue + + resampled, fs = _resample_signal(signal, physiological_fs[label], 25) + resampled = np.nan_to_num(resampled, nan=0.0, posinf=0.0, neginf=0.0) + + if group_name == 'SpO2': + results.update(_summarize_spo2(resampled, fs)) + continue + + try: + hat_br, _, _, used = Resp_features.peakedness_application( + resampled, + stage=label, + plotflag=False, + subjet=label, + ) + except Exception: + continue + + summary = _summarize_peakedness(hat_br) + if summary is None: + continue + + quality = _compute_resp_quality(used, hat_br) + if quality <= best_quality[group_name]: + continue + + best_quality[group_name] = quality + for metric_name, metric_value in summary.items(): + results[f'{group_name}_Peakedness_{metric_name}'] = metric_value + + return np.array([results[name] for name in RESP_FEATURE_NAMES], dtype=np.float32) From 616a1b80cce6999860afdf0585f6e81b8bef3261 Mon Sep 17 00:00:00 2001 From: sromagnoli-10 Date: Sat, 28 Mar 2026 09:34:18 +0100 Subject: [PATCH 34/93] Update openECGfunction.py --- src/lib/openECGfunction.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/openECGfunction.py b/src/lib/openECGfunction.py index 745b1ac..2710531 100644 --- a/src/lib/openECGfunction.py +++ b/src/lib/openECGfunction.py @@ -17,7 +17,7 @@ def openECG(physiological_data_file, patient_id): # Check if any ECG keyword is inside the label if any(keyword in label_clean for keyword in ecg_keywords): idx = i - break # ✅ first ECG channel only + break #first ECG channel only if idx is None: raise ValueError("No ECG channel found") @@ -36,4 +36,4 @@ def openECG(physiological_data_file, patient_id): [all_patients_ECGresults, all_results], ignore_index=True ) - return all_patients_ECGresults \ No newline at end of file + return all_patients_ECGresults From 30e2369dbde4dac196bcc8fe1bc8c35873175c7e Mon Sep 17 00:00:00 2001 From: sromagnoli-10 Date: Sat, 28 Mar 2026 09:40:13 +0100 Subject: [PATCH 35/93] Update team_code.py --- team_code.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/team_code.py b/team_code.py index 7296cc1..9f9efb7 100644 --- a/team_code.py +++ b/team_code.py @@ -24,6 +24,7 @@ from helper_code import * from src.resp_processing import processResp from src.eeg_processing import processEEG +from lib.openECGfunction import ################################################################################ # Path & Constant Configuration (Added for Robustness) ################################################################################ @@ -668,4 +669,4 @@ def count_discrete_events(key): def save_model(model_folder, model): d = {'model': model} filename = os.path.join(model_folder, 'model.sav') - joblib.dump(d, filename, protocol=0) \ No newline at end of file + joblib.dump(d, filename, protocol=0) From fe8e28bfdb3d134686fc0e332a8b3ed5f456cb37 Mon Sep 17 00:00:00 2001 From: sromagnoli-10 Date: Sat, 28 Mar 2026 09:52:33 +0100 Subject: [PATCH 36/93] Update team_code.py --- team_code.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/team_code.py b/team_code.py index 9f9efb7..740983f 100644 --- a/team_code.py +++ b/team_code.py @@ -24,7 +24,7 @@ from helper_code import * from src.resp_processing import processResp from src.eeg_processing import processEEG -from lib.openECGfunction import +from lib.openECGfunction import openECG ################################################################################ # Path & Constant Configuration (Added for Robustness) ################################################################################ From 112aae078308f9304c7cfa993aaa94433fa1f709 Mon Sep 17 00:00:00 2001 From: dcajal Date: Sat, 28 Mar 2026 10:04:52 +0100 Subject: [PATCH 37/93] Fix invalid value encounteres in sacalar divide --- src/lib/EEG_functions.py | 140 +++++++++++++++++++++------------------ src/lib/peakedness.py | 24 +++---- 2 files changed, 87 insertions(+), 77 deletions(-) diff --git a/src/lib/EEG_functions.py b/src/lib/EEG_functions.py index 83e8ef6..8c7137c 100644 --- a/src/lib/EEG_functions.py +++ b/src/lib/EEG_functions.py @@ -5,17 +5,27 @@ from scipy import signal from scipy.stats import kurtosis, entropy -try: - import plotly.graph_objects as go - from plotly.subplots import make_subplots -except ModuleNotFoundError: - go = None - make_subplots = None +# try: +# import plotly.graph_objects as go +# from plotly.subplots import make_subplots +# except ModuleNotFoundError: +# go = None +# make_subplots = None -try: - import matplotlib.pyplot as plt -except ModuleNotFoundError: - plt = None +# try: +# import matplotlib.pyplot as plt +# except ModuleNotFoundError: +# plt = None + +def _safe_sqrt_variance_ratio(numerator_signal, denominator_signal): + numerator_var = np.var(numerator_signal) + denominator_var = np.var(denominator_signal) + if denominator_var <= 0 or not np.isfinite(denominator_var): + return 0.0 + ratio = numerator_var / denominator_var + if ratio <= 0 or not np.isfinite(ratio): + return 0.0 + return float(np.sqrt(ratio)) def butter_bandpass_filter(data, lowcut, highcut, fs, order=4): nyq = 0.5 * fs # Frecuencia de Nyquist @@ -40,66 +50,66 @@ def butter_bandpass_filter(data, lowcut, highcut, fs, order=4): y = filtfilt(b, a, data) return y -def plot_EEG(df, columns, fs = 200): - if go is None or make_subplots is None: - raise ModuleNotFoundError("plotly is required for plot_EEG") +# def plot_EEG(df, columns, fs = 200): +# if go is None or make_subplots is None: +# raise ModuleNotFoundError("plotly is required for plot_EEG") - fig = make_subplots(rows=len(columns), cols=1, - shared_xaxes=True, - vertical_spacing=0.02, - subplot_titles=columns) - limit = int(3000 * fs) - x = np.arange(df[0].shape[0]) / fs # Asumiendo fs=100Hz, ajusta si es diferente - downsample = 10 # Factor de downsampling para mejorar rendimiento (ajusta según necesidad) - for i, col in enumerate(columns): - fig.add_trace( - go.Scattergl(x=x[:limit:downsample], y=df[i][:limit:downsample], name=col, mode='lines'), - row=i+1, col=1 - ) - fig.update_layout( - height=900, - title_text="Polisomnografía - Canales EEG", - showlegend=False, - template="plotly_white" - ) - fig.update_xaxes(title_text="Tiempo (segundos)", row=len(columns), col=1) - fig.show() +# fig = make_subplots(rows=len(columns), cols=1, +# shared_xaxes=True, +# vertical_spacing=0.02, +# subplot_titles=columns) +# limit = int(3000 * fs) +# x = np.arange(df[0].shape[0]) / fs # Asumiendo fs=100Hz, ajusta si es diferente +# downsample = 10 # Factor de downsampling para mejorar rendimiento (ajusta según necesidad) +# for i, col in enumerate(columns): +# fig.add_trace( +# go.Scattergl(x=x[:limit:downsample], y=df[i][:limit:downsample], name=col, mode='lines'), +# row=i+1, col=1 +# ) +# fig.update_layout( +# height=900, +# title_text="Polisomnografía - Canales EEG", +# showlegend=False, +# template="plotly_white" +# ) +# fig.update_xaxes(title_text="Tiempo (segundos)", row=len(columns), col=1) +# fig.show() -def plot_EEG_sel(sel, name = "EEG_plot_raw.html"): - if go is None or make_subplots is None: - raise ModuleNotFoundError("plotly is required for plot_EEG_sel") +# def plot_EEG_sel(sel, name = "EEG_plot_raw.html"): +# if go is None or make_subplots is None: +# raise ModuleNotFoundError("plotly is required for plot_EEG_sel") - fig = make_subplots(rows=len(sel), cols=1, - shared_xaxes=True, - vertical_spacing=0.02, - subplot_titles=[ch[1].label for ch in sel]) +# fig = make_subplots(rows=len(sel), cols=1, +# shared_xaxes=True, +# vertical_spacing=0.02, +# subplot_titles=[ch[1].label for ch in sel]) - for i, (idx, sig) in enumerate(sel): - # Crear eje de tiempo en segundos - fs = sig.sampling_frequency - time = np.linspace(0, len(sig.data) / fs, len(sig.data)) +# for i, (idx, sig) in enumerate(sel): +# # Crear eje de tiempo en segundos +# fs = sig.sampling_frequency +# time = np.linspace(0, len(sig.data) / fs, len(sig.data)) - # Añadir traza (solo mostramos los primeros 30s por defecto para no saturar el navegador) - # Puedes quitar el slice [:int(30*fs)] para ver todo, pero cuidado con el rendimiento - limit = int(3000 * fs) - # limit = len(sig.data) if limit > len(sig.data) else limit - # limit = len(sig.data) - # downsample = 10 # Factor de downsampling para mejorar rendimiento (ajusta según necesidad) - fig.add_trace( - go.Scattergl(x=time[:limit], y=sig.data[:limit], name=sig.label, mode='lines'), - row=i+1, col=1 - ) +# # Añadir traza (solo mostramos los primeros 30s por defecto para no saturar el navegador) +# # Puedes quitar el slice [:int(30*fs)] para ver todo, pero cuidado con el rendimiento +# limit = int(3000 * fs) +# # limit = len(sig.data) if limit > len(sig.data) else limit +# # limit = len(sig.data) +# # downsample = 10 # Factor de downsampling para mejorar rendimiento (ajusta según necesidad) +# fig.add_trace( +# go.Scattergl(x=time[:limit], y=sig.data[:limit], name=sig.label, mode='lines'), +# row=i+1, col=1 +# ) - fig.update_layout( - height=900, - title_text="Polisomnografía - Canales EEG", - showlegend=False, - template="plotly_white" - ) +# fig.update_layout( +# height=900, +# title_text="Polisomnografía - Canales EEG", +# showlegend=False, +# template="plotly_white" +# ) - fig.update_xaxes(title_text="Tiempo (segundos)", row=len(sel), col=1) - fig.write_html(f"graphs/{name}.html") # Guardar como HTML para visualización interactiva - # fig.show() +# fig.update_xaxes(title_text="Tiempo (segundos)", row=len(sel), col=1) +# fig.write_html(f"graphs/{name}.html") # Guardar como HTML para visualización interactiva +# # fig.show() def filtering_and_normalization(sig, sig_fs): b, a = signal.butter(4, 0.3, btype='highpass', fs=sig_fs) @@ -249,10 +259,10 @@ def extract_band_powers(epochs, fs, win_len = 2): features.append(epoch_features) diff = np.diff(epoch) - mobility = np.sqrt(np.var(diff) / np.var(epoch)) + mobility = _safe_sqrt_variance_ratio(diff, epoch) # 2. Complejidad de Hjorth: Qué tan similar es la señal a una onda senoidal diff2 = np.diff(diff) - mobility_diff = np.sqrt(np.var(diff2) / np.var(diff)) + mobility_diff = _safe_sqrt_variance_ratio(diff2, diff) complexity = mobility_diff / mobility if mobility > 0 else 0 complexities.append({'Hjorth_Mobility': mobility, 'Hjorth_Complexity': complexity}) diff --git a/src/lib/peakedness.py b/src/lib/peakedness.py index 1bda09c..eaf224a 100644 --- a/src/lib/peakedness.py +++ b/src/lib/peakedness.py @@ -116,7 +116,7 @@ def extract_interval( x, t, int_ini, int_end ): return [ x_int, t_int ] -def normalizar_PSD( PSD, f = 'default', rango = 'default'): +def normalizar_PSD( PSD, f = None, rango = None): # NORMALIZAR_PSD Normaliza una densidad espectral de potencia en el rango # de frecuencias requerido. # @@ -131,17 +131,17 @@ def normalizar_PSD( PSD, f = 'default', rango = 'default'): # f_PSD_norm = Vector de frecuencias para PSD_norm # factor_norm = Factor de normalizaci�n utilizado - if f == 'default': + if f is None: f = np.arange(0,PSD.shape[0]) / PSD.shape[0] - 1/2 - if rango == 'default': + if rango is None: rango = [f[0], f[-1]] # Seleccionar rango de inter�s: f_PSD_norm = f[(f>=rango[0]) & (f<=rango[1])] PSD = PSD[(f>=rango[0]) & (f<=rango[1])] - if ~f_PSD_norm.any(): # El vector de frecuencias no estaba ordenado + if not np.any(f_PSD_norm): # El vector de frecuencias no estaba ordenado print('El vector de frecuencias debe estar ordenado de forma ascendente'); @@ -257,17 +257,20 @@ def init_module(kk,vars,param, plotflag): # Cost function for deviation from previous fr and maximum power max_s = np.max(S) - C_a = 1-_safe_ratio(np.transpose(pk), max_s, default=np.zeros_like(pk, dtype=float)) + if np.isfinite(max_s) and max_s != 0: + C_a = 1 - (np.transpose(pk) / max_s) + else: + C_a = np.ones_like(pk, dtype=float) fr_prev = vars["bar_fr"][np.max(kk,0)] C_f = abs(f[j_pk[:]]-fr_prev)/(2*d) # C_f = abs(f(i_pk(:))-fr_prev)/(Omega_r(2)-Omega(1)); C = C_a +C_f - try: + if C.size > 0: j_min = C.argmin() fj = j_pk[j_min] vars["bar_fr"][kk] = f[fj] - except: + else: vars["bar_fr"][kk] = 0 # Save in vars # vars["bar_fr"][kk] = f[fj] @@ -592,11 +595,8 @@ def peakednessCost(signals, ts, fs, Setup = {}, title = "", storeGraph = False, if np.isnan(vars["hat_fr"][0]) and int_e.shape[0]>1: int_e = int_e[1:] - try: - if (int_e[0]-int_b[0])[0] < 0: - int_e = int_e[1:] - except: - print("int vacio") + if int_b.size > 0 and int_e.size > 0 and (int_e[0]-int_b[0])[0] < 0: + int_e = int_e[1:] int_small = (int_e-int_b)<=(N_k-1) From 866d466416b21ff2cf10731f59e8980f1041e275 Mon Sep 17 00:00:00 2001 From: sromagnoli-10 Date: Sat, 28 Mar 2026 10:54:18 +0100 Subject: [PATCH 38/93] creating ECG processing branch, changing folder of two functions and rename files --- src/{main_ECG_ver2.py => lib/ECG_processing.py} | 0 src/{lib => }/openECGfunction.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename src/{main_ECG_ver2.py => lib/ECG_processing.py} (100%) rename src/{lib => }/openECGfunction.py (95%) diff --git a/src/main_ECG_ver2.py b/src/lib/ECG_processing.py similarity index 100% rename from src/main_ECG_ver2.py rename to src/lib/ECG_processing.py diff --git a/src/lib/openECGfunction.py b/src/openECGfunction.py similarity index 95% rename from src/lib/openECGfunction.py rename to src/openECGfunction.py index 2710531..64c72e7 100644 --- a/src/lib/openECGfunction.py +++ b/src/openECGfunction.py @@ -1,5 +1,5 @@ import pyedflib -from ..main_ECG_ver2 import ECGprocessing +from .lib.ECG_processing import ECGprocessing import pandas as pd def openECG(physiological_data_file, patient_id): From af8c3927c8e8dd9bee47fdbdafa2d8c2486b56f8 Mon Sep 17 00:00:00 2001 From: rolopu1 <718694@unizar.es> Date: Sun, 29 Mar 2026 19:21:40 +0200 Subject: [PATCH 39/93] add ecg calls --- requirements.txt | 3 ++- src/lib/openECGfunction.py | 15 +++++++++++++++ team_code.py | 15 ++++++++++++++- 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 28b6875..ae49d51 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,4 +4,5 @@ numpy==2.0.2 pandas==2.2.2 scikit-learn==1.6.0 scipy==1.13.1 -tqdm==4.67.1 \ No newline at end of file +tqdm==4.67.1 +pyedflib==0.1.42 \ No newline at end of file diff --git a/src/lib/openECGfunction.py b/src/lib/openECGfunction.py index 2710531..cb54341 100644 --- a/src/lib/openECGfunction.py +++ b/src/lib/openECGfunction.py @@ -1,6 +1,21 @@ import pyedflib from ..main_ECG_ver2 import ECGprocessing import pandas as pd + +ECG_FEATURE_NAMES = [ + "ID", + "mNNmed", "mNNstd", + "PIP_med", "PIP_std", + "PNNLS_med", "PNNLS_std", + "PNNSS_med", "PNNSS_std", + "AVNN_med", "AVNN_std", + "SDNN_med", "SDNN_std", + "RMSSD_med", "RMSSD_std", + "HF_med", "HF_std", + "ECTOPIC_med", "ECTOPIC_std", +] +ECG_FEATURE_LENGTH = len(ECG_FEATURE_NAMES) + def openECG(physiological_data_file, patient_id): f = pyedflib.EdfReader(physiological_data_file) diff --git a/team_code.py b/team_code.py index d5194c9..d5f78e3 100644 --- a/team_code.py +++ b/team_code.py @@ -24,6 +24,8 @@ from helper_code import * from src.resp_processing import RESP_FEATURE_LENGTH, processResp from src.eeg_processing import EEG_FEATURE_LENGTH, processEEG +from src.lib.openECGfunction import ECG_FEATURE_LENGTH,openECG + ################################################################################ # Path & Constant Configuration (Added for Robustness) ################################################################################ @@ -107,8 +109,19 @@ def extract_extended_physiological_features(physiological_data, physiological_fs ) except Exception: eeg_features = np.zeros(EEG_FEATURE_LENGTH, dtype=np.float32) + + try: + ecg_features = _extract_optional_features( + openECG, + ECG_FEATURE_LENGTH, + physiological_data, + physiological_fs, + csv_path=csv_path, + ) + except Exception: + ecg_features = np.zeros(ECG_FEATURE_LENGTH, dtype=np.float32) - return np.hstack([base_features, resp_features, eeg_features]).astype(np.float32) + return np.hstack([base_features, resp_features, eeg_features, ecg_features]).astype(np.float32) def process_training_record(record, data_folder, demographics_cache, diagnosis_cache, csv_path): From 19149a208ece438e8ba5283bcaa21c62eb2b7d27 Mon Sep 17 00:00:00 2001 From: rolopu1 <718694@unizar.es> Date: Sun, 29 Mar 2026 19:57:30 +0200 Subject: [PATCH 40/93] models functions --- src/lib/model_functions.py | 295 +++++++++++++++++++++++++++++++++++++ src/main_ECG_ver2.py | 2 + 2 files changed, 297 insertions(+) create mode 100644 src/lib/model_functions.py diff --git a/src/lib/model_functions.py b/src/lib/model_functions.py new file mode 100644 index 0000000..94b8432 --- /dev/null +++ b/src/lib/model_functions.py @@ -0,0 +1,295 @@ +import numpy as np +import pandas as pd +from xgboost import XGBClassifier +import numpy as np +import pandas as pd +from sklearn.model_selection import train_test_split +from sklearn.compose import ColumnTransformer +from sklearn.pipeline import Pipeline +from sklearn.impute import KNNImputer, SimpleImputer +from sklearn.preprocessing import StandardScaler, OneHotEncoder +from sklearn.metrics import classification_report, roc_auc_score, f1_score + +def best_threshold(proba_final,y_test): + thresholds = np.linspace(0, 1, 100) + best_thr = 0 + best_f1 = 0 + for t in thresholds: + preds = (proba_final >= t).astype(int) + f1 = f1_score(y_test, preds) + if f1 > best_f1: + best_f1 = f1 + best_thr = t + return best_thr + +def evaluar_rendimiento(y_real,y_pred, proba_final, dict_probas, nombre_ensemble="Ensemble"): + """ + Imprime un reporte completo de métricas de clasificación y AUC. + + Args: + y_real: Etiquetas verdaderas (y_test o y_val). + proba_final: Probabilidades del modelo combinado. + dict_probas: Diccionario con formato {'Nombre': lista_probas} para modelos individuales. + nombre_ensemble: Nombre personalizado para el modelo principal. + """ + + # Impresión de métricas principales + print(f"Reporte de clasificacion {nombre_ensemble}") + print(classification_report(y_real, y_pred)) + + # AUC del Ensemble + roc_auc = roc_auc_score(y_real, proba_final) + print(f"AUC {nombre_ensemble}: {roc_auc:.2%}") + print("\n--- Comparativa AUC Modelos Individuales ---") + + # 4. AUC de modelos individuales usando el diccionario + for nombre, probas in dict_probas.items(): + auc_score = roc_auc_score(y_real, probas) + print(f"AUC {nombre:<8}: {auc_score:.2%}") + print("-" * 30) + +def preprocess_multimodal_data(demo, df_model): + + #Llamamos al dataset de entrada que contiene las variables calculadas y confirma que no hay IDs duplicados + df_model = df_model.drop_duplicates(subset='ID') + demo=demo.drop_duplicates(subset='ID') + #Haz el merge con las demograficas del paciente, Age, Sex y Label + # Asegurar tipo string en ID + for d in [demo, df_model]: + d['ID'] = d['ID'].astype(str).str.strip() # .strip() por si hay espacios invisibles + + df_model2= (demo.merge(df_model, on="ID", how="inner")) + + # Extrae las columnas que se usarán + df_model2=df_model2[['ID','label', 'Age','Sex','Nasal_Peakedness_Max', 'Nasal_Peakedness_Min', 'Nasal_Peakedness_Median','Nasal_Peakedness_Std', + 'Chest_Peakedness_Max', 'Chest_Peakedness_Min','Abdomen_Peakedness_Max','Abdomen_Peakedness_Min','Abdomen_Peakedness_Std', + 'Flow_Peakedness_Max','Flow_Peakedness_Min','Flow_Peakedness_Median','Flow_Peakedness_Std','SpO2_Max','SpO2_Min','SpO2_Std', + 'CET90','ODI_Mean','ODI_deepness','C3-M2_Hjorth_Complexity','C4-M1_Hjorth_Complexity','F3-M2_Hjorth_Complexity', + 'F4-M1_Hjorth_Complexity','C3-M2_Hjorth_Mobility','F3-M2_Hjorth_Mobility','F4-M1_Hjorth_Mobility','C3-M2_Ratio_Slow_Fast', + 'C4-M1_Ratio_Slow_Fast','F3-M2_Ratio_Slow_Fast','F4-M1_Ratio_Slow_Fast','C3-M2_Rel_Beta','F3-M2_Rel_Beta','F4-M1_Rel_Beta', + 'C4-M1_Rel_Sigma','F3-M2_Rel_Sigma','C3-M2_Relative_Delta_Power','C4-M1_Relative_Delta_Power','F3-M2_Relative_Delta_Power', + 'F4-M1_Relative_Delta_Power','C3-M2_Theta_Alpha_Ratio','C4-M1_Theta_Alpha_Ratio','F3-M2_Theta_Alpha_Ratio','F4-M1_Theta_Alpha_Ratio', + 'C3-M2_Theta_Beta_Ratio','C4-M1_Theta_Beta_Ratio','F3-M2_Theta_Beta_Ratio','F4-M1_Theta_Beta_Ratio','C3-M2_kurtosis_Alpha', + 'C3-M2_kurtosis_Beta','C4-M1_kurtosis_Beta','F3-M2_kurtosis_Beta','F4-M1_kurtosis_Beta','C3-M2_kurtosis_Delta','C4-M1_kurtosis_Delta', + 'F3-M2_kurtosis_Delta','F4-M1_kurtosis_Delta','C3-M2_kurtosis_Sigma','C4-M1_kurtosis_Sigma','F3-M2_kurtosis_Sigma','F4-M1_kurtosis_Sigma', + 'C3-M2_kurtosis_Theta','C4-M1_kurtosis_Theta','F3-M2_kurtosis_Theta','F4-M1_kurtosis_Theta','C3-M2_variability_Delta', + 'C4-M1_variability_Delta','F3-M2_variability_Delta','F4-M1_variability_Delta','PIP_med','PIP_std','PNNSS_med', + 'PNNSS_std','AVNN_med','AVNN_std','SDNN_med','SDNN_std','RMSSD_std','HF_med','HF_std','ECTOPIC_med','ECTOPIC_std']] + + + # Ajusta el tipo de dato, todas deben ser numericas excepto Sex, Label y ID. Sex y label tendría que ser logical? o categorical? + df_model2['Sex'] = df_model2['Sex'].astype('category') + df_model2['label'] = df_model2['label'].astype(int) + + #Unificar los valores faltantes, que sean consistentes + df_model2 = df_model2.replace([-999, "NA", "null", "None", "nan", ""], np.nan) + # Eliminar filas con demasiados NaNs (mínimo 50% de datos válidos) + umbral = int(len(df_model2.columns) * 0.5) + df_model2 = df_model2.dropna(thresh=umbral) + + #Procesa los datos, imputar nans y estandarizar por la media y desviacion estandar (zscore) + y = df_model2['label'] + X = df_model2.drop(columns=[ 'label','ID']) # Excluimos ID, label del entrenamiento + + return {"X": X, + "y": y} + +def prepare_multimodal_data(demo, df_model, testsize=0.1): + + proc_data=preprocess_multimodal_data(demo, df_model) + X = proc_data["X"] + y = proc_data["y"] + X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=testsize, random_state=1999, stratify=y) + + cols_categoricas = ['Sex'] + cols_numericas = [c for c in X_train.columns if c not in cols_categoricas] + + preprocessor = ColumnTransformer([ + ("num", Pipeline([ + ("imputer", KNNImputer(n_neighbors=5)), + ("scaler", StandardScaler()) + ]), cols_numericas), + + ("cat", Pipeline([ + ("imputer", SimpleImputer(strategy="most_frequent")), + ("encoder", OneHotEncoder(handle_unknown="ignore", sparse_output=False)) + ]), cols_categoricas) + ]) + preprocessor.set_output(transform="pandas") + + X_train = preprocessor.fit_transform(X_train) + X_test = preprocessor.transform(X_test) + + feature_names = preprocessor.get_feature_names_out() + resp_mask = np.array([ any(k in col for k in ["Age","Sex","Nasal_Peakedness", "Chest_Peakedness", "Abdomen_Peakedness", "Flow_Peakedness", "SpO2", "CET90", "ODI" ]) + for col in feature_names]) + eeg_mask = np.array([ any(k in col for k in ["Age","Sex","C3-M2", "C4-M1", "F3-M2", "F4-M1"]) + for col in feature_names]) + ecg_mask = np.array([ any(k in col for k in ["Age","Sex","PIP_", "PNNSS_", "AVNN_", "SDNN_", "RMSSD_", "HF_", "ECTOPIC_"]) + for col in feature_names]) + + # Subsets finales + return { + "X_train": X_train, + "X_test": X_test, + "y_train": y_train, + "y_test": y_test, + "resp": (X_train.loc[:, resp_mask], X_test.loc[:, resp_mask]), + "eeg": (X_train.loc[:, eeg_mask], X_test.loc[:, eeg_mask]), + "ecg": (X_train.loc[:, ecg_mask], X_test.loc[:, ecg_mask]), + "preprocessor": preprocessor + } + +def prepare_multimodal_validationdata(demo, df_model, preprocessor): + + #Llamamos al dataset de entrada que contiene las variables calculadas y confirma que no hay IDs duplicados + df_model = df_model.drop_duplicates(subset='ID') + demo=demo.drop_duplicates(subset='ID') + #Haz el merge con las demograficas del paciente, Age, Sex y Label + # Asegurar tipo string en ID + for d in [demo, df_model]: + d['ID'] = d['ID'].astype(str).str.strip() # .strip() por si hay espacios invisibles + + df_model2= (demo.merge(df_model, on="ID", how="inner")) + + # Extrae las columnas que se usarán + columnas_deseadas=['ID', 'Age','Sex','Nasal_Peakedness_Max', 'Nasal_Peakedness_Min', 'Nasal_Peakedness_Median','Nasal_Peakedness_Std', + 'Chest_Peakedness_Max', 'Chest_Peakedness_Min','Abdomen_Peakedness_Max','Abdomen_Peakedness_Min','Abdomen_Peakedness_Std', + 'Flow_Peakedness_Max','Flow_Peakedness_Min','Flow_Peakedness_Median','Flow_Peakedness_Std','SpO2_Max','SpO2_Min','SpO2_Std', + 'CET90','ODI_Mean','ODI_deepness','C3-M2_Hjorth_Complexity','C4-M1_Hjorth_Complexity','F3-M2_Hjorth_Complexity', + 'F4-M1_Hjorth_Complexity','C3-M2_Hjorth_Mobility','F3-M2_Hjorth_Mobility','F4-M1_Hjorth_Mobility','C3-M2_Ratio_Slow_Fast', + 'C4-M1_Ratio_Slow_Fast','F3-M2_Ratio_Slow_Fast','F4-M1_Ratio_Slow_Fast','C3-M2_Rel_Beta','F3-M2_Rel_Beta','F4-M1_Rel_Beta', + 'C4-M1_Rel_Sigma','F3-M2_Rel_Sigma','C3-M2_Relative_Delta_Power','C4-M1_Relative_Delta_Power','F3-M2_Relative_Delta_Power', + 'F4-M1_Relative_Delta_Power','C3-M2_Theta_Alpha_Ratio','C4-M1_Theta_Alpha_Ratio','F3-M2_Theta_Alpha_Ratio','F4-M1_Theta_Alpha_Ratio', + 'C3-M2_Theta_Beta_Ratio','C4-M1_Theta_Beta_Ratio','F3-M2_Theta_Beta_Ratio','F4-M1_Theta_Beta_Ratio','C3-M2_kurtosis_Alpha', + 'C3-M2_kurtosis_Beta','C4-M1_kurtosis_Beta','F3-M2_kurtosis_Beta','F4-M1_kurtosis_Beta','C3-M2_kurtosis_Delta','C4-M1_kurtosis_Delta', + 'F3-M2_kurtosis_Delta','F4-M1_kurtosis_Delta','C3-M2_kurtosis_Sigma','C4-M1_kurtosis_Sigma','F3-M2_kurtosis_Sigma','F4-M1_kurtosis_Sigma', + 'C3-M2_kurtosis_Theta','C4-M1_kurtosis_Theta','F3-M2_kurtosis_Theta','F4-M1_kurtosis_Theta','C3-M2_variability_Delta', + 'C4-M1_variability_Delta','F3-M2_variability_Delta','F4-M1_variability_Delta','PIP_med','PIP_std','PNNSS_med', + 'PNNSS_std','AVNN_med','AVNN_std','SDNN_med','SDNN_std','RMSSD_std','HF_med','HF_std','ECTOPIC_med','ECTOPIC_std'] + #Filtrar extraer solo las que existen en el modelo + columnas_existentes = [col for col in columnas_deseadas if col in df_model2.columns] + df_model2 = df_model2[columnas_existentes] + + # Ajusta el tipo de dato, todas deben ser numericas excepto Sex, Label y ID. Sex y label tendría que ser logical? o categorical? + df_model2['Sex'] = df_model2['Sex'].astype('category') + #df_model2['label'] = df_model2['label'].astype(int) + + #Unificar los valores faltantes, que sean consistentes + df_model2 = df_model2.replace([-999, "NA", "null", "None", "nan", ""], np.nan) + # Eliminar filas con demasiados NaNs (mínimo 50% de datos válidos) + umbral = int(len(df_model2.columns) * 0.5) + df_model2 = df_model2.dropna(thresh=umbral) + + keys_resp = ["Nasal_Peakedness", "Chest_Peakedness", "Abdomen_Peakedness", "Flow_Peakedness", "SpO2", "CET90", "ODI"] + keys_eeg = ["C3-M2", "C4-M1", "F3-M2", "F4-M1"] + keys_ecg = ["PIP_", "PNNSS_", "AVNN_", "SDNN_", "RMSSD_", "HF_", "ECTOPIC_"] + # Identificar qué columnas de 'columnas_deseadas' pertenecen a cada señal + cols_resp_esperadas = [c for c in columnas_deseadas if any(c.startswith(k) for k in keys_resp)] + cols_eeg_esperadas = [c for c in columnas_deseadas if any(c.startswith(k) for k in keys_eeg)] + cols_ecg_esperadas = [c for c in columnas_deseadas if any(c.startswith(k) for k in keys_ecg)] + + # Verificación Estricta: ¿Están TODAS en el dataframe de entrada? + presencia_original = { + "resp": all(c in df_model2.columns for c in cols_resp_esperadas), + "eeg": all(c in df_model2.columns for c in cols_eeg_esperadas), + "ecg": all(c in df_model2.columns for c in cols_ecg_esperadas) + } + + #Procesa los datos, imputar nans y estandarizar por la media y desviacion estandar (zscore) + #y = df_model2['label'] + #X = df_model2.drop(columns=[ 'label','ID']) # Excluimos ID, label del entrenamiento + X = df_model2.drop(columns=[ 'ID']) # Excluimos ID, label del entrenamiento + + # Reconstrucción técnica (solo para que transform no falle, luego lo filtraremos) + columnas_que_espera = preprocessor.feature_names_in_ + for col in columnas_que_espera: + if col not in X.columns: + X[col] = np.nan + + X = X[columnas_que_espera] + X_val_array = preprocessor.transform(X) + feature_names = preprocessor.get_feature_names_out() + X_validation = pd.DataFrame(X_val_array, columns=feature_names, index=X.index) + + # 6. Extracción Final (Si no estaba completa originalmente, devuelve None) + def validar_y_extraer(cols_esperadas, existe_completa): + if not existe_completa: + return None + + # Filtramos en los nombres transformados (ej: num__Nasal...) + cols_senal = [c for c in feature_names if any(k in c for k in cols_esperadas)] + cols_demo = [c for c in feature_names if any(d in c for d in ["Age", "Sex"])] + + orden_final = [c for c in feature_names if c in (cols_demo + cols_senal)] + return X_validation[orden_final] + + return { + "X_valid": X_validation, + "resp": validar_y_extraer(cols_resp_esperadas, presencia_original["resp"]), + "eeg": validar_y_extraer(cols_eeg_esperadas, presencia_original["eeg"]), + "ecg": validar_y_extraer(cols_ecg_esperadas, presencia_original["ecg"]) + } + +def reconstruct_multimodal_data(demo, df_model): + data = prepare_multimodal_data(demo, df_model) + X_train = data["X_train"] + X_test = data["X_test"] + X_train_resp, X_test_resp = data["resp"] + X_train_EEG, X_test_EEG = data["eeg"] + X_train_ECG, X_test_ECG = data["ecg"] + y_train = data["y_train"] + y_test = data["y_test"] + preprocessor=data["preprocessor"] + + # Configuramos Los parámetros base de los modelos + est=300 # Número de árboles + depth=4 # Profundidad (evita overfitting) + lr=0.05 # Paso de aprendizaje + ss=0.8 # Usa el 80% de los datos para cada árbol (evita memorizar) + rs=42 # Para que sea replicable + categ=False + met='auc' # Métrica de error interna + stopi=20 # early stopping si la metrica no mejora en tantas epocas + neg = (y_train == 0).sum() + pos = (y_train == 1).sum() + scale_pos_weight = neg / pos + + all_xgb = XGBClassifier(scale_pos_weight=scale_pos_weight,n_estimators=est, max_depth=depth, learning_rate=lr, subsample=ss, random_state=rs, enable_categorical=categ, eval_metric=met, early_stopping_rounds=stopi) + ECG_xgb = XGBClassifier(scale_pos_weight=scale_pos_weight,n_estimators=est, max_depth=depth, learning_rate=lr, subsample=ss, random_state=rs, enable_categorical=categ, eval_metric=met, early_stopping_rounds=stopi) + resp_xgb = XGBClassifier(scale_pos_weight=scale_pos_weight,n_estimators=est, max_depth=depth, learning_rate=lr, subsample=ss, random_state=rs, enable_categorical=categ, eval_metric=met, early_stopping_rounds=stopi) + EEG_xgb = XGBClassifier(scale_pos_weight=scale_pos_weight,n_estimators=est, max_depth=depth, learning_rate=lr, subsample=ss, random_state=rs, enable_categorical=categ, eval_metric=met, early_stopping_rounds=stopi) + + # Entrenamiento + all_xgb.fit(X_train, y_train,eval_set=[(X_test, y_test)], verbose=False) + ECG_xgb.fit(X_train_ECG, y_train,eval_set=[(X_test_ECG, y_test)], verbose=False) + resp_xgb.fit(X_train_resp, y_train,eval_set=[(X_test_resp, y_test)], verbose=False) + EEG_xgb.fit(X_train_EEG, y_train,eval_set=[(X_test_EEG, y_test)], verbose=False) + + res = prepare_multimodal_validationdata(demo, dfv, preprocessor) + + # Cálculo dinámico de probabilidades + probas = [] + probas_individuales = {} + + if res["resp"] is not None: + vproba_resp = resp_xgb.predict_proba(res["resp"])[:, 1] + probas.append(vproba_resp) + probas_individuales["RESP"] = vproba_resp + + if res["eeg"] is not None: + vproba_eeg = EEG_xgb.predict_proba(res["eeg"])[:, 1] + probas.append(vproba_eeg) + probas_individuales["EEG"] = vproba_eeg + + if res["ecg"] is not None: + vproba_ecg = ECG_xgb.predict_proba(res["ecg"])[:, 1] + probas.append(vproba_ecg) + probas_individuales["ECG"] = vproba_ecg + + # Promedio de lo que sea que hayamos podido ejecutar + vproba_final = np.mean(probas, axis=0) + y_predv = (vproba_final >= best_thr).astype(int) + + #y_v=res["y_valid"] + #evaluar_rendimiento(y_v,y_predv, vproba_final, probas_individuales, nombre_ensemble="Ensemble gboost validation") diff --git a/src/main_ECG_ver2.py b/src/main_ECG_ver2.py index 6206e23..7227f29 100644 --- a/src/main_ECG_ver2.py +++ b/src/main_ECG_ver2.py @@ -5,6 +5,8 @@ from .lib.compute_hrv_hrf import compute_HRV_HRF from .lib.interpolate_NN import interpolate_NN_pchip from .lib.remove_ectopic_beat import remove_ectopic_beats + + def ECGprocessing(ecg_signal, fs, patient_id): all_results = pd.DataFrame() From 38f9671abdccfac523ebca22cdb3ccb799525665 Mon Sep 17 00:00:00 2001 From: sromagnoli-10 Date: Sun, 29 Mar 2026 23:04:30 +0200 Subject: [PATCH 41/93] creating ECG processing branch, changing folder of two functions and rename files --- src/ecg_processing.py | 110 ++++++++++++++++++++++++++++++++++++++ src/lib/ECG_processing.py | 55 ++++++++----------- src/lib/utilities_HRV.py | 80 +++++++++++++++++++++++++++ src/openECGfunction.py | 39 -------------- team_code.py | 14 ++++- 5 files changed, 226 insertions(+), 72 deletions(-) create mode 100644 src/ecg_processing.py create mode 100644 src/lib/utilities_HRV.py delete mode 100644 src/openECGfunction.py diff --git a/src/ecg_processing.py b/src/ecg_processing.py new file mode 100644 index 0000000..be66843 --- /dev/null +++ b/src/ecg_processing.py @@ -0,0 +1,110 @@ +import pyedflib +from .lib.ECG_processing import ECGprocessing +import pandas as pd +import numpy as np + +ECG_KEYWORDS = ['ecg', 'ekg'] + +ECG_FEATURE_NAMES = [ + "PIP_med", + "PIP_std", + "PNNLS_med", + "PNNLS_std", + "PNNSS_med", + "PNNSS_std", + "AVNN_med", + "AVNN_std", + "SDNN_med", + "SDNN_std", + "RMSSD_med", + "RMSSD_std", + "HF_med", + "HF_std", + "ECTOPIC_med", + "ECTOPIC_std" +] +ECG_FEATURE_LENGTH = len(ECG_FEATURE_NAMES) + +def _normalize_label(text): + return ''.join(ch if ch.isalnum() else ' ' for ch in str(text).lower()).strip() + + +def _find_ecg_channel(physiological_data): + for label in physiological_data.keys(): + label_clean = _normalize_label(label) + if any(keyword in label_clean for keyword in ECG_KEYWORDS): + return label + return None + + +def processECG(physiological_data, physiological_fs, csv_path): + results = np.zeros(ECG_FEATURE_LENGTH, dtype=np.float32) + + ecg_label = _find_ecg_channel(physiological_data) + + if ecg_label is None: + return results # no ECG found + + if ecg_label not in physiological_fs: + return results + + ecg_signal = np.asarray(physiological_data[ecg_label], dtype=float) + fs = physiological_fs[ecg_label] + + if ecg_signal.size == 0: + return results + + try: + values = ECGprocessing(ecg_signal, fs, ECG_FEATURE_LENGTH) + + if values is None or len(values) == 0: + return results + + values = values.astype(np.float32) + + if len(values) >= ECG_FEATURE_LENGTH: + results[:] = values[:ECG_FEATURE_LENGTH] + else: + results[:len(values)] = values + + except Exception: + pass + + return results + +""" def openECG(physiological_data_file, patient_id): + + f = pyedflib.EdfReader(physiological_data_file) + + signal_labels = f.getSignalLabels() + print(signal_labels) + + ecg_keywords = ['ecg', 'ekg'] + + idx = None + for i, label in enumerate(signal_labels): + label_clean = label.lower().strip() + + # Check if any ECG keyword is inside the label + if any(keyword in label_clean for keyword in ecg_keywords): + idx = i + break #first ECG channel only + + if idx is None: + raise ValueError("No ECG channel found") + + print("ECG channel:", signal_labels[idx]) + + ecg_signal = f.readSignal(idx) + fs = f.getSampleFrequency(idx) + + f.close() + + all_results = ECGprocessing(ecg_signal, fs, patient_id) + + if all_results is not None: + all_patients_ECGresults = pd.concat( + [all_patients_ECGresults, all_results], + ignore_index=True + ) + return all_patients_ECGresults """ diff --git a/src/lib/ECG_processing.py b/src/lib/ECG_processing.py index 6206e23..37e2013 100644 --- a/src/lib/ECG_processing.py +++ b/src/lib/ECG_processing.py @@ -1,13 +1,13 @@ import numpy as np import pandas as pd from scipy.signal import butter, filtfilt, resample -from .lib.pan_tompkins import pan_tompkin -from .lib.compute_hrv_hrf import compute_HRV_HRF -from .lib.interpolate_NN import interpolate_NN_pchip -from .lib.remove_ectopic_beat import remove_ectopic_beats -def ECGprocessing(ecg_signal, fs, patient_id): +from .pan_tompkins import pan_tompkin +from .compute_hrv_hrf import compute_HRV_HRF +from .interpolate_NN import interpolate_NN_pchip +from .remove_ectopic_beat import remove_ectopic_beats - all_results = pd.DataFrame() + +def ECGprocessing(ecg_signal, fs, ECG_FEATURE_LENGTH): ecg_signal = ecg_signal - np.mean(ecg_signal) @@ -110,21 +110,16 @@ def ECGprocessing(ecg_signal, fs, patient_id): # =============================== res = compute_HRV_HRF(NN, fs) - meanNN = np.mean(NN) - HRV_all.append([ - meanNN, res["PIP"], res["PNNLS"], res["PNNSS"], res["AVNN"], res["SDNN"], res["RMSSD"], res["HF"], ectopic_perc ]) - - # =============================== - # SUBJECT-LEVEL METRICS - # =============================== +# =============================== +# SUBJECT-LEVEL METRICS +# =============================== if len(HRV_all) == 0: - print("No valid windows.") - return None + return np.zeros(ECG_FEATURE_LENGTH, dtype=np.float32) HRV_all = np.array(HRV_all) @@ -132,21 +127,17 @@ def ECGprocessing(ecg_signal, fs, patient_id): std_vals = np.nanstd(HRV_all, axis=0) # =============================== - # SAVE RESULTS (DataFrame row) + # BUILD FIXED FEATURE VECTOR # =============================== - row = pd.DataFrame([{ - "ID": patient_id, - "mNNmed": median_vals[0], "mNNstd": std_vals[0], - "PIP_med": median_vals[1], "PIP_std": std_vals[1], - "PNNLS_med": median_vals[2], "PNNLS_std": std_vals[2], - "PNNSS_med": median_vals[3], "PNNSS_std": std_vals[3], - "AVNN_med": median_vals[4], "AVNN_std": std_vals[4], - "SDNN_med": median_vals[5], "SDNN_std": std_vals[5], - "RMSSD_med": median_vals[6], "RMSSD_std": std_vals[6], - "HF_med": median_vals[7], "HF_std": std_vals[7], - "ECTOPIC_med": median_vals[8], "ECTOPIC_std": std_vals[8], - }]) - - all_results = pd.concat([all_results, row], ignore_index=True) - - return all_results \ No newline at end of file + features = np.array([ + median_vals[0], std_vals[0], # PIP + median_vals[1], std_vals[1], # PNNLS + median_vals[2], std_vals[2], # PNNSS + median_vals[3], std_vals[3], # AVNN + median_vals[4], std_vals[4], # SDNN + median_vals[5], std_vals[5], # RMSSD + median_vals[6], std_vals[6], # HF + median_vals[7], std_vals[7], # ECTOPIC + ], dtype=np.float32) + + return features \ No newline at end of file diff --git a/src/lib/utilities_HRV.py b/src/lib/utilities_HRV.py new file mode 100644 index 0000000..4438c81 --- /dev/null +++ b/src/lib/utilities_HRV.py @@ -0,0 +1,80 @@ +import numpy as np +from scipy.interpolate import PchipInterpolator + +def interpolate_NN_pchip(NN, maxGap): + """ + NN: array of NN intervals (seconds) + maxGap: max number of consecutive NaNs allowed for interpolation + """ + + NN = np.asarray(NN).flatten() + NN_interp = NN.copy() + + nan_idx = np.isnan(NN) + + # Find NaN segments + d = np.diff(np.concatenate(([0], nan_idx.astype(int), [0]))) + start_idx = np.where(d == 1)[0] + end_idx = np.where(d == -1)[0] - 1 + + for k in range(len(start_idx)): + seg_len = end_idx[k] - start_idx[k] + 1 + + if seg_len <= maxGap: + left = start_idx[k] - 1 + right = end_idx[k] + 1 + + # Check bounds + if (left >= 0 and right < len(NN) and + not np.isnan(NN[left]) and not np.isnan(NN[right])): + + x = np.array([left, right]) + y = np.array([NN[left], NN[right]]) + + xi = np.arange(start_idx[k], end_idx[k] + 1) + + # PCHIP interpolation + interpolator = PchipInterpolator(x, y) + NN_interp[xi] = interpolator(xi) + + return NN_interp + +def remove_ectopic_beats(NN, window_size, threshold): + NN = np.asarray(NN).flatten() + NN_corrected = NN.copy() + + half_win = window_size // 2 + ectopic_count = 0 + valid_count = 0 + + for i in range(len(NN)): + + if np.isnan(NN[i]): + continue + + valid_count += 1 + + # Define local window + left = max(0, i - half_win) + right = min(len(NN), i + half_win + 1) # Python slice is exclusive + + local_segment = NN[left:right] + local_segment = local_segment[~np.isnan(local_segment)] + + if local_segment.size == 0: + continue + + med_val = np.median(local_segment) + + # Detect ectopic + if abs(NN[i] - med_val) > threshold * med_val: + NN_corrected[i] = med_val + ectopic_count += 1 + + # Percentage over valid NN + if valid_count > 0: + ectopic_perc = (ectopic_count / valid_count) * 100 + else: + ectopic_perc = np.nan + + return NN_corrected, ectopic_perc \ No newline at end of file diff --git a/src/openECGfunction.py b/src/openECGfunction.py deleted file mode 100644 index 64c72e7..0000000 --- a/src/openECGfunction.py +++ /dev/null @@ -1,39 +0,0 @@ -import pyedflib -from .lib.ECG_processing import ECGprocessing -import pandas as pd -def openECG(physiological_data_file, patient_id): - - f = pyedflib.EdfReader(physiological_data_file) - - signal_labels = f.getSignalLabels() - print(signal_labels) - - ecg_keywords = ['ecg', 'ekg'] - - idx = None - for i, label in enumerate(signal_labels): - label_clean = label.lower().strip() - - # Check if any ECG keyword is inside the label - if any(keyword in label_clean for keyword in ecg_keywords): - idx = i - break #first ECG channel only - - if idx is None: - raise ValueError("No ECG channel found") - - print("ECG channel:", signal_labels[idx]) - - ecg_signal = f.readSignal(idx) - fs = f.getSampleFrequency(idx) - - f.close() - - all_results = ECGprocessing(ecg_signal, fs, patient_id) - - if all_results is not None: - all_patients_ECGresults = pd.concat( - [all_patients_ECGresults, all_results], - ignore_index=True - ) - return all_patients_ECGresults diff --git a/team_code.py b/team_code.py index d5194c9..c4cc81a 100644 --- a/team_code.py +++ b/team_code.py @@ -24,6 +24,7 @@ from helper_code import * from src.resp_processing import RESP_FEATURE_LENGTH, processResp from src.eeg_processing import EEG_FEATURE_LENGTH, processEEG +from src.ecg_processing import ECG_FEATURE_LENGTH, processECG ################################################################################ # Path & Constant Configuration (Added for Robustness) ################################################################################ @@ -108,7 +109,18 @@ def extract_extended_physiological_features(physiological_data, physiological_fs except Exception: eeg_features = np.zeros(EEG_FEATURE_LENGTH, dtype=np.float32) - return np.hstack([base_features, resp_features, eeg_features]).astype(np.float32) + try: + ecg_features = _extract_optional_features( + processECG, + ECG_FEATURE_LENGTH, + physiological_data, + physiological_fs, + csv_path=csv_path, + ) + except Exception: + ecg_features = np.zeros(ECG_FEATURE_LENGTH, dtype=np.float32) + + return np.hstack([base_features, resp_features, eeg_features, ecg_features]).astype(np.float32) def process_training_record(record, data_folder, demographics_cache, diagnosis_cache, csv_path): From 37f57f580c5c2a1546e3ec4c786e4d42db1d8afd Mon Sep 17 00:00:00 2001 From: sromagnoli-10 Date: Mon, 30 Mar 2026 00:09:09 +0200 Subject: [PATCH 42/93] correction in team code for ecg features merging --- team_code.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/team_code.py b/team_code.py index b0346ab..b4f5305 100644 --- a/team_code.py +++ b/team_code.py @@ -114,18 +114,7 @@ def extract_extended_physiological_features(physiological_data, physiological_fs except Exception: eeg_features = np.zeros(EEG_FEATURE_LENGTH, dtype=np.float32) - try: - ecg_features = _extract_optional_features( - openECG, - ECG_FEATURE_LENGTH, - physiological_data, - physiological_fs, - csv_path=csv_path, - ) - except Exception: - ecg_features = np.zeros(ECG_FEATURE_LENGTH, dtype=np.float32) -<<<<<<< HEAD try: ecg_features = _extract_optional_features( processECG, @@ -137,8 +126,6 @@ def extract_extended_physiological_features(physiological_data, physiological_fs except Exception: ecg_features = np.zeros(ECG_FEATURE_LENGTH, dtype=np.float32) -======= ->>>>>>> 19149a208ece438e8ba5283bcaa21c62eb2b7d27 return np.hstack([base_features, resp_features, eeg_features, ecg_features]).astype(np.float32) From 832741ee7f23ca3179a83d1d52611df9a454663b Mon Sep 17 00:00:00 2001 From: sromagnoli-10 Date: Mon, 30 Mar 2026 00:26:04 +0200 Subject: [PATCH 43/93] fixed error after merging --- team_code.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/team_code.py b/team_code.py index b4f5305..f18dbf5 100644 --- a/team_code.py +++ b/team_code.py @@ -24,12 +24,7 @@ from helper_code import * from src.resp_processing import RESP_FEATURE_LENGTH, processResp from src.eeg_processing import EEG_FEATURE_LENGTH, processEEG -<<<<<<< HEAD from src.ecg_processing import ECG_FEATURE_LENGTH, processECG -======= -from src.lib.openECGfunction import ECG_FEATURE_LENGTH,openECG - ->>>>>>> 19149a208ece438e8ba5283bcaa21c62eb2b7d27 ################################################################################ # Path & Constant Configuration (Added for Robustness) ################################################################################ From 778d9250a9e37b14bb084100b6573393ac3fae64 Mon Sep 17 00:00:00 2001 From: dcajal Date: Mon, 30 Mar 2026 12:50:37 +0200 Subject: [PATCH 44/93] remove pyedflib import and update requirements for xgboost --- requirements.txt | 2 +- src/ecg_processing.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index ae49d51..00c7ede 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,4 @@ pandas==2.2.2 scikit-learn==1.6.0 scipy==1.13.1 tqdm==4.67.1 -pyedflib==0.1.42 \ No newline at end of file +xgboost==3.2.0 \ No newline at end of file diff --git a/src/ecg_processing.py b/src/ecg_processing.py index be66843..e50eaac 100644 --- a/src/ecg_processing.py +++ b/src/ecg_processing.py @@ -1,4 +1,3 @@ -import pyedflib from .lib.ECG_processing import ECGprocessing import pandas as pd import numpy as np From da9d2b8022717339192baed27e830733c8c1a431 Mon Sep 17 00:00:00 2001 From: dcajal Date: Mon, 30 Mar 2026 13:02:57 +0200 Subject: [PATCH 45/93] Implement persistent feature caching to optimize data extraction process --- docs/05_optimization_tracking.md | 23 ++++ team_code.py | 182 ++++++++++++++++++++++--------- 2 files changed, 156 insertions(+), 49 deletions(-) diff --git a/docs/05_optimization_tracking.md b/docs/05_optimization_tracking.md index db352fa..f601f8e 100644 --- a/docs/05_optimization_tracking.md +++ b/docs/05_optimization_tracking.md @@ -112,6 +112,29 @@ Riesgo: - Medio. - El acceso paralelo a archivos puede comportarse distinto en discos más lentos o en una infraestructura más limitada del Challenge. +### 5. Caché persistente de features por registro + +Cambio: + +- Se añadió una caché en disco por registro `(site_id, patient_id, session_id)`. +- Antes de extraer features, el pipeline comprueba si ya existe el archivo de caché para ese paciente. +- Si el archivo existe, se reutiliza directamente; si no existe, se calcula y se guarda. + +Motivo: + +- La extracción fisiológica es el cuello de botella dominante. +- En iteraciones sucesivas de entrenamiento o inferencia se estaban recalculando features ya generadas para los mismos registros. + +Efecto esperado: + +- La primera ejecución mantiene un coste parecido al actual. +- Las siguientes ejecuciones sobre el mismo dataset reutilizan las features ya materializadas y reducen de forma importante el tiempo total. + +Riesgo: + +- Bajo a medio. +- Si cambia la lógica de extracción y se quiere regenerar todo, habrá que borrar manualmente la carpeta de caché. +- ## Plan De Rollback Si la submission se comporta distinto en el entorno del Challenge, revertir en este orden: diff --git a/team_code.py b/team_code.py index f18dbf5..20d3174 100644 --- a/team_code.py +++ b/team_code.py @@ -14,6 +14,7 @@ import os import atexit import builtins +import hashlib import pandas as pd import re from concurrent.futures import ThreadPoolExecutor @@ -43,6 +44,14 @@ RUN_PROGRESS_LINE_RE = re.compile(r'^-\s+\d+/\d+:\s') RENAME_RULES_CACHE = {} MAX_TRAIN_WORKERS = max(1, min(4, os.cpu_count() or 1)) +FEATURE_CACHE_FOLDER_NAME = '.feature_cache' +BASE_PHYSIOLOGICAL_FEATURE_LENGTH = 49 +TOTAL_PHYSIOLOGICAL_FEATURE_LENGTH = ( + BASE_PHYSIOLOGICAL_FEATURE_LENGTH + + RESP_FEATURE_LENGTH + + EEG_FEATURE_LENGTH + + ECG_FEATURE_LENGTH +) def build_training_metadata_cache(patient_data_file): @@ -82,6 +91,116 @@ def _extract_optional_features(extractor, expected_length, *args, **kwargs): return vector +def _get_record_file_paths(data_folder, site_id, patient_id, session_id): + physiological_data_file = os.path.join( + data_folder, + PHYSIOLOGICAL_DATA_SUBFOLDER, + site_id, + f"{patient_id}_ses-{session_id}.edf" + ) + algorithmic_annotations_file = os.path.join( + data_folder, + ALGORITHMIC_ANNOTATIONS_SUBFOLDER, + site_id, + f"{patient_id}_ses-{session_id}_caisr_annotations.edf" + ) + return physiological_data_file, algorithmic_annotations_file + + +def _get_feature_cache_file(data_folder, site_id, patient_id, session_id): + folder_hash = hashlib.sha1(os.path.abspath(data_folder).encode('utf-8')).hexdigest()[:12] + cache_dir = os.path.join( + SCRIPT_DIR, + FEATURE_CACHE_FOLDER_NAME, + folder_hash, + site_id, + ) + return os.path.join(cache_dir, f"{patient_id}_ses-{session_id}.sav") + + +def _load_cached_feature_vector(cache_file): + if not os.path.exists(cache_file): + return None + + try: + payload = joblib.load(cache_file) + except Exception: + return None + + if isinstance(payload, dict): + payload = payload.get('features') + + if payload is None: + return None + + return _coerce_feature_vector(payload) + + +def _save_cached_feature_vector(cache_file, feature_vector): + os.makedirs(os.path.dirname(cache_file), exist_ok=True) + temp_cache_file = f"{cache_file}.tmp" + payload = _coerce_feature_vector(feature_vector) + + try: + joblib.dump(payload, temp_cache_file, protocol=0) + os.replace(temp_cache_file, cache_file) + finally: + if os.path.exists(temp_cache_file): + os.remove(temp_cache_file) + + +def _compute_record_feature_vector(patient_data, data_folder, site_id, patient_id, session_id, csv_path, require_physiological_data): + demographic_features = extract_demographic_features(patient_data) + physiological_data_file, algorithmic_annotations_file = _get_record_file_paths( + data_folder, + site_id, + patient_id, + session_id, + ) + + if os.path.exists(physiological_data_file): + physiological_data, physiological_fs = load_signal_data(physiological_data_file) + physiological_features = extract_extended_physiological_features( + physiological_data, + physiological_fs, + csv_path=csv_path, + ) + elif require_physiological_data: + raise FileNotFoundError(f"Missing physiological data for {patient_id}.") + else: + physiological_features = np.zeros(TOTAL_PHYSIOLOGICAL_FEATURE_LENGTH, dtype=np.float32) + + if os.path.exists(algorithmic_annotations_file): + algorithmic_annotations, _ = load_signal_data(algorithmic_annotations_file) + algorithmic_features = extract_algorithmic_annotations_features(algorithmic_annotations) + else: + algorithmic_features = np.zeros(12, dtype=np.float32) + + return np.hstack([demographic_features, physiological_features, algorithmic_features]).astype(np.float32) + + +def get_or_create_record_feature_vector(record, data_folder, patient_data, csv_path=DEFAULT_CSV_PATH, require_physiological_data=True): + patient_id = record[HEADERS['bids_folder']] + site_id = record[HEADERS['site_id']] + session_id = record[HEADERS['session_id']] + cache_file = _get_feature_cache_file(data_folder, site_id, patient_id, session_id) + cached_features = _load_cached_feature_vector(cache_file) + if cached_features is not None: + return cached_features + + feature_vector = _compute_record_feature_vector( + patient_data, + data_folder, + site_id, + patient_id, + session_id, + csv_path, + require_physiological_data, + ) + _save_cached_feature_vector(cache_file, feature_vector) + return feature_vector + + def extract_extended_physiological_features(physiological_data, physiological_fs, csv_path=DEFAULT_CSV_PATH): base_features = _coerce_feature_vector( extract_physiological_features(physiological_data, physiological_fs, csv_path=csv_path) @@ -126,48 +245,27 @@ def extract_extended_physiological_features(physiological_data, physiological_fs def process_training_record(record, data_folder, demographics_cache, diagnosis_cache, csv_path): patient_id = record[HEADERS['bids_folder']] - site_id = record[HEADERS['site_id']] session_id = record[HEADERS['session_id']] try: patient_data = demographics_cache.get((patient_id, session_id), {}) - demographic_features = extract_demographic_features(patient_data) - - physiological_data_file = os.path.join( - data_folder, - PHYSIOLOGICAL_DATA_SUBFOLDER, - site_id, - f"{patient_id}_ses-{session_id}.edf" - ) - if not os.path.exists(physiological_data_file): - return patient_id, None, None, f"Missing physiological data for {patient_id}. Skipping..." - - physiological_data, physiological_fs = load_signal_data(physiological_data_file) - physiological_features = extract_extended_physiological_features( - physiological_data, - physiological_fs, - csv_path=csv_path - ) - algorithmic_annotations_file = os.path.join( + feature_vector = get_or_create_record_feature_vector( + record, data_folder, - ALGORITHMIC_ANNOTATIONS_SUBFOLDER, - site_id, - f"{patient_id}_ses-{session_id}_caisr_annotations.edf" + patient_data, + csv_path=csv_path, + require_physiological_data=True, ) - if os.path.exists(algorithmic_annotations_file): - algorithmic_annotations, _ = load_signal_data(algorithmic_annotations_file) - algorithmic_features = extract_algorithmic_annotations_features(algorithmic_annotations) - else: - algorithmic_features = np.zeros(12, dtype=np.float32) label = diagnosis_cache.get(patient_id) if label == 0 or label == 1: - feature_vector = np.hstack([demographic_features, physiological_features, algorithmic_features]) return patient_id, feature_vector, label, None return patient_id, None, None, f"Invalid label for {patient_id}. Skipping..." + except FileNotFoundError as e: + return patient_id, None, None, f"{e} Skipping..." except Exception as e: return patient_id, None, None, f"Error processing {patient_id}: {e}" @@ -339,27 +437,13 @@ def run_model(model, record, data_folder, verbose): # Load the patient data. patient_data_file = os.path.join(data_folder, DEMOGRAPHICS_FILE) patient_data = load_demographics(patient_data_file, patient_id, session_id) - demographic_features = extract_demographic_features(patient_data) - - # Load signal data. - phys_file = os.path.join(data_folder, PHYSIOLOGICAL_DATA_SUBFOLDER, site_id, f"{patient_id}_ses-{session_id}.edf") - if os.path.exists(phys_file): - phys_data, phys_fs = load_signal_data(phys_file) - # Ensure csv_path is accessible or defined - physiological_features = extract_extended_physiological_features(phys_data, phys_fs) - else: - physiological_features = np.zeros(49 + RESP_FEATURE_LENGTH + EEG_FEATURE_LENGTH, dtype=np.float32) - - # Load Algorithmic Annotations - algo_file = os.path.join(data_folder, ALGORITHMIC_ANNOTATIONS_SUBFOLDER, site_id, f"{patient_id}_ses-{session_id}_caisr_annotations.edf") - if os.path.exists(algo_file): - algo_data, _ = load_signal_data(algo_file) - algorithmic_features = extract_algorithmic_annotations_features(algo_data) - else: - # Fallback to zeros (length 12) - algorithmic_features = np.zeros(12, dtype=np.float32) - - features = np.hstack([demographic_features, physiological_features, algorithmic_features]).reshape(1, -1) + features = get_or_create_record_feature_vector( + record, + data_folder, + patient_data, + csv_path=DEFAULT_CSV_PATH, + require_physiological_data=False, + ).reshape(1, -1) # Get the model outputs. binary_output = model.predict(features)[0] From a33b55e1e521ebf9d233cccb5f5b4befb2871973 Mon Sep 17 00:00:00 2001 From: dcajal Date: Mon, 30 Mar 2026 14:11:59 +0200 Subject: [PATCH 46/93] Add caching for EEG and respiratory alias groups to optimize data processing --- src/eeg_processing.py | 14 ++++++++++++-- src/lib/peakedness.py | 10 +++++++--- src/resp_processing.py | 14 ++++++++++++-- 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/src/eeg_processing.py b/src/eeg_processing.py index cf95237..4e7cb54 100644 --- a/src/eeg_processing.py +++ b/src/eeg_processing.py @@ -50,6 +50,7 @@ 'EEG_Hjorth_Complexity', ] EEG_FEATURE_LENGTH = len(EEG_FEATURE_NAMES) +EEG_ALIASES_CACHE = {} def _normalize_label(text): @@ -69,6 +70,16 @@ def _build_eeg_aliases(channels): return aliases +def _get_eeg_aliases(csv_path): + normalized_csv_path = os.path.abspath(csv_path) + eeg_aliases = EEG_ALIASES_CACHE.get(normalized_csv_path) + if eeg_aliases is None: + channels = pd.read_csv(normalized_csv_path) + eeg_aliases = _build_eeg_aliases(channels) + EEG_ALIASES_CACHE[normalized_csv_path] = eeg_aliases + return eeg_aliases + + def _resample_signal(signal, fs, target_fs): signal = np.asarray(signal, dtype=float) if signal.size == 0: @@ -130,8 +141,7 @@ def _extract_channel_metrics(signal, fs): def processEEG(physiological_data, physiological_fs, csv_path): - channels = pd.read_csv(csv_path) - eeg_aliases = _build_eeg_aliases(channels) + eeg_aliases = _get_eeg_aliases(csv_path) channel_metrics = [] for label, signal in physiological_data.items(): diff --git a/src/lib/peakedness.py b/src/lib/peakedness.py index eaf224a..772c907 100644 --- a/src/lib/peakedness.py +++ b/src/lib/peakedness.py @@ -111,8 +111,10 @@ def extract_interval( x, t, int_ini, int_end ): # t_int = interval [int_ini, int_end] of 't' # indexes = indexes corresponding to returned time interval - x_int = x[(t>=int_ini) & (t <=int_end)] - t_int = t[(t>=int_ini) & (t <=int_end)] + start_idx = np.searchsorted(t, int_ini, side='left') + end_idx = np.searchsorted(t, int_end, side='right') + x_int = x[start_idx:end_idx] + t_int = t[start_idx:end_idx] return [ x_int, t_int ] @@ -439,6 +441,8 @@ def peakednessCost(signals, ts, fs, Setup = {}, title = "", storeGraph = False, if int_Ts_sig.shape[0] < (Tm*100)/2: vars["Skl"][:, ki, ii] = np.zeros((vars["f"].shape[0])) continue + int_Ts_sig = int_Ts_sig.astype(float, copy=False) + ts_has_nan = np.isnan(int_Ts_sig).any() # Number of Tm length subintervals NWm = int(np.floor(2*Ts/Tm)) I=0 @@ -456,7 +460,7 @@ def peakednessCost(signals, ts, fs, Setup = {}, title = "", storeGraph = False, [int_Tm_sig, int_Tm_t] = extract_interval(int_Ts_sig, int_Ts_t, Wm_begin, Wm_end) # Estimate the spectrum only for intervals without NaNs - if ~np.isnan((int_Ts_sig.astype(float))).any(): + if not ts_has_nan: S_i = abs(fftshift(fft(detrend(int_Tm_sig[:-1]), Nfft)))**2 # S_i = abs(fftshift(fft(int_Tm_sig[:-1], Nfft)))**2 S_i = S_i[f_ind] diff --git a/src/resp_processing.py b/src/resp_processing.py index c1e7845..0b4577c 100644 --- a/src/resp_processing.py +++ b/src/resp_processing.py @@ -21,6 +21,7 @@ "ODI_deepness", ] RESP_FEATURE_LENGTH = len(RESP_FEATURE_NAMES) +RESP_ALIAS_GROUPS_CACHE = {} def _normalize_label(text): @@ -45,6 +46,16 @@ def _build_resp_alias_groups(channels): } +def _get_resp_alias_groups(csv_path): + normalized_csv_path = os.path.abspath(csv_path) + alias_groups = RESP_ALIAS_GROUPS_CACHE.get(normalized_csv_path) + if alias_groups is None: + channels = pd.read_csv(normalized_csv_path) + alias_groups = _build_resp_alias_groups(channels) + RESP_ALIAS_GROUPS_CACHE[normalized_csv_path] = alias_groups + return alias_groups + + def _find_resp_group(label, alias_groups): normalized = _normalize_label(label) for group_name, aliases in alias_groups.items(): @@ -126,8 +137,7 @@ def _summarize_spo2(data, fs): def processResp(physiological_data, physiological_fs, csv_path): - channels = pd.read_csv(csv_path) - alias_groups = _build_resp_alias_groups(channels) + alias_groups = _get_resp_alias_groups(csv_path) results = {feature_name: 0.0 for feature_name in RESP_FEATURE_NAMES} best_quality = {group_name: -np.inf for group_name in RESP_CHANNEL_GROUPS} From 2f32d61178b4ac54bf9f5ae1746bbbdef0d56a41 Mon Sep 17 00:00:00 2001 From: dcajal Date: Mon, 30 Mar 2026 14:42:36 +0200 Subject: [PATCH 47/93] Refactor EEG processing and update feature extraction to improve channel metrics handling --- src/eeg_processing.py | 172 +++++++++++++++++++++++++++++++----------- team_code.py | 54 ++++++------- 2 files changed, 151 insertions(+), 75 deletions(-) diff --git a/src/eeg_processing.py b/src/eeg_processing.py index 4e7cb54..89c23ba 100644 --- a/src/eeg_processing.py +++ b/src/eeg_processing.py @@ -39,16 +39,64 @@ sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -EEG_FEATURE_NAMES = [ - 'EEG_Channel_Count', - 'EEG_Rel_Delta', - 'EEG_Rel_Theta', - 'EEG_Rel_Alpha', - 'EEG_Rel_Sigma', - 'EEG_Rel_Beta', - 'EEG_Theta_Alpha_Ratio', - 'EEG_Hjorth_Complexity', +EEG_CHANNEL_SPECS = { + 'C3-M2': {'direct': 'c3-m2', 'positive': 'c3', 'reference': 'm2'}, + 'C4-M1': {'direct': 'c4-m1', 'positive': 'c4', 'reference': 'm1'}, + 'F3-M2': {'direct': 'f3-m2', 'positive': 'f3', 'reference': 'm2'}, + 'F4-M1': {'direct': 'f4-m1', 'positive': 'f4', 'reference': 'm1'}, +} +EEG_FEATURE_SPECS = [ + ('C3-M2', 'Hjorth_Complexity'), + ('C4-M1', 'Hjorth_Complexity'), + ('F3-M2', 'Hjorth_Complexity'), + ('F4-M1', 'Hjorth_Complexity'), + ('C3-M2', 'Hjorth_Mobility'), + ('F3-M2', 'Hjorth_Mobility'), + ('F4-M1', 'Hjorth_Mobility'), + ('C3-M2', 'Ratio_Slow_Fast'), + ('C4-M1', 'Ratio_Slow_Fast'), + ('F3-M2', 'Ratio_Slow_Fast'), + ('F4-M1', 'Ratio_Slow_Fast'), + ('C3-M2', 'Rel_Beta'), + ('F3-M2', 'Rel_Beta'), + ('F4-M1', 'Rel_Beta'), + ('C4-M1', 'Rel_Sigma'), + ('F3-M2', 'Rel_Sigma'), + ('C3-M2', 'Relative_Delta_Power'), + ('C4-M1', 'Relative_Delta_Power'), + ('F3-M2', 'Relative_Delta_Power'), + ('F4-M1', 'Relative_Delta_Power'), + ('C3-M2', 'Theta_Alpha_Ratio'), + ('C4-M1', 'Theta_Alpha_Ratio'), + ('F3-M2', 'Theta_Alpha_Ratio'), + ('F4-M1', 'Theta_Alpha_Ratio'), + ('C3-M2', 'Theta_Beta_Ratio'), + ('C4-M1', 'Theta_Beta_Ratio'), + ('F3-M2', 'Theta_Beta_Ratio'), + ('F4-M1', 'Theta_Beta_Ratio'), + ('C3-M2', 'kurtosis_Alpha'), + ('C3-M2', 'kurtosis_Beta'), + ('C4-M1', 'kurtosis_Beta'), + ('F3-M2', 'kurtosis_Beta'), + ('F4-M1', 'kurtosis_Beta'), + ('C3-M2', 'kurtosis_Delta'), + ('C4-M1', 'kurtosis_Delta'), + ('F3-M2', 'kurtosis_Delta'), + ('F4-M1', 'kurtosis_Delta'), + ('C3-M2', 'kurtosis_Sigma'), + ('C4-M1', 'kurtosis_Sigma'), + ('F3-M2', 'kurtosis_Sigma'), + ('F4-M1', 'kurtosis_Sigma'), + ('C3-M2', 'kurtosis_Theta'), + ('C4-M1', 'kurtosis_Theta'), + ('F3-M2', 'kurtosis_Theta'), + ('F4-M1', 'kurtosis_Theta'), + ('C3-M2', 'variability_Delta'), + ('C4-M1', 'variability_Delta'), + ('F3-M2', 'variability_Delta'), + ('F4-M1', 'variability_Delta'), ] +EEG_FEATURE_NAMES = [f'{channel}_{metric}' for channel, metric in EEG_FEATURE_SPECS] EEG_FEATURE_LENGTH = len(EEG_FEATURE_NAMES) EEG_ALIASES_CACHE = {} @@ -63,11 +111,14 @@ def _split_aliases(raw_aliases): def _build_eeg_aliases(channels): - eeg_rows = channels[channels['Category'].eq('eeg')] - aliases = set() - for _, row in eeg_rows.iterrows(): - aliases.update(_split_aliases(row['Channel_Names'])) - return aliases + alias_lookup = {} + for _, row in channels.iterrows(): + aliases = _split_aliases(row['Channel_Names']) + if not aliases: + continue + canonical_name = _normalize_label(str(row['Channel_Names']).split(';')[0]) + alias_lookup[canonical_name] = aliases + return alias_lookup def _get_eeg_aliases(csv_path): @@ -94,6 +145,41 @@ def _resample_signal(signal, fs, target_fs): return np.interp(time_target, time_original, signal), target_fs +def _find_matching_label(physiological_data, aliases): + for label in physiological_data.keys(): + if _normalize_label(label) in aliases: + return label + return None + + +def _get_channel_signal(channel_name, physiological_data, physiological_fs, eeg_aliases): + channel_spec = EEG_CHANNEL_SPECS[channel_name] + direct_aliases = eeg_aliases.get(_normalize_label(channel_spec['direct']), set()) + direct_label = _find_matching_label(physiological_data, direct_aliases) + if direct_label is not None and direct_label in physiological_fs: + return np.asarray(physiological_data[direct_label], dtype=float), physiological_fs[direct_label] + + positive_aliases = eeg_aliases.get(_normalize_label(channel_spec['positive']), set()) + reference_aliases = eeg_aliases.get(_normalize_label(channel_spec['reference']), set()) + positive_label = _find_matching_label(physiological_data, positive_aliases) + reference_label = _find_matching_label(physiological_data, reference_aliases) + if positive_label is None or reference_label is None: + return None, None + if positive_label not in physiological_fs or reference_label not in physiological_fs: + return None, None + + positive_fs = physiological_fs[positive_label] + reference_fs = physiological_fs[reference_label] + if positive_fs != reference_fs: + return None, None + + return ( + np.asarray(physiological_data[positive_label], dtype=float) + - np.asarray(physiological_data[reference_label], dtype=float), + positive_fs, + ) + + def _extract_channel_metrics(signal, fs): signal = np.nan_to_num(np.asarray(signal, dtype=float), nan=0.0, posinf=0.0, neginf=0.0) if signal.size < max(int(fs * 30), 2): @@ -119,44 +205,42 @@ def _extract_channel_metrics(signal, fs): if band_powers.empty: return None - total_power = band_powers.sum(axis=1).replace(0, np.nan) - relative_powers = band_powers.div(total_power, axis=0).replace([np.inf, -np.inf], np.nan).fillna(0.0).mean() - alpha_power = float(relative_powers.get('Alpha', 0.0)) - theta_power = float(relative_powers.get('Theta', 0.0)) - theta_alpha_ratio = theta_power / alpha_power if alpha_power > 0 else 0.0 - - complexity_mean = float( - complexities['Hjorth_Complexity'].replace([np.inf, -np.inf], np.nan).fillna(0.0).mean() - ) if 'Hjorth_Complexity' in complexities else 0.0 - - return np.array([ - float(relative_powers.get('Delta', 0.0)), - theta_power, - alpha_power, - float(relative_powers.get('Sigma', 0.0)), - float(relative_powers.get('Beta', 0.0)), - float(theta_alpha_ratio), - complexity_mean, - ], dtype=np.float32) + patient_profile = EEG_functions.get_patient_profile(band_powers) + metrics = { + str(name): float(value) + for name, value in patient_profile.replace([np.inf, -np.inf], np.nan).fillna(0.0).items() + } + for complexity_name in ('Hjorth_Mobility', 'Hjorth_Complexity'): + if complexity_name in complexities: + value = complexities[complexity_name].replace([np.inf, -np.inf], np.nan).fillna(0.0).std() + metrics[complexity_name] = float(0.0 if pd.isna(value) else value) + else: + metrics[complexity_name] = 0.0 + return metrics def processEEG(physiological_data, physiological_fs, csv_path): eeg_aliases = _get_eeg_aliases(csv_path) - channel_metrics = [] + channel_profiles = {} - for label, signal in physiological_data.items(): - if label not in physiological_fs: - continue - if _normalize_label(label) not in eeg_aliases: + for channel_name in EEG_CHANNEL_SPECS: + signal, fs = _get_channel_signal(channel_name, physiological_data, physiological_fs, eeg_aliases) + if signal is None or fs is None: continue - metrics = _extract_channel_metrics(signal, physiological_fs[label]) + metrics = _extract_channel_metrics(signal, fs) if metrics is not None: - channel_metrics.append(metrics) + channel_profiles[channel_name] = metrics - if not channel_metrics: + if not channel_profiles: return np.zeros(EEG_FEATURE_LENGTH, dtype=np.float32) - stacked = np.vstack(channel_metrics) - aggregated = np.mean(stacked, axis=0) - return np.hstack([np.array([len(channel_metrics)], dtype=np.float32), aggregated]).astype(np.float32) \ No newline at end of file + values = [] + for channel_name, metric_name in EEG_FEATURE_SPECS: + channel_metrics = channel_profiles.get(channel_name) + if channel_metrics is None: + values.append(0.0) + continue + values.append(float(channel_metrics.get(metric_name, 0.0))) + + return np.asarray(values, dtype=np.float32) \ No newline at end of file diff --git a/team_code.py b/team_code.py index 20d3174..8cbac15 100644 --- a/team_code.py +++ b/team_code.py @@ -18,7 +18,7 @@ import pandas as pd import re from concurrent.futures import ThreadPoolExecutor -from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor +from xgboost import XGBClassifier import sys from tqdm import tqdm @@ -44,11 +44,9 @@ RUN_PROGRESS_LINE_RE = re.compile(r'^-\s+\d+/\d+:\s') RENAME_RULES_CACHE = {} MAX_TRAIN_WORKERS = max(1, min(4, os.cpu_count() or 1)) -FEATURE_CACHE_FOLDER_NAME = '.feature_cache' -BASE_PHYSIOLOGICAL_FEATURE_LENGTH = 49 +FEATURE_CACHE_FOLDER_NAME = '.feature_cache_xgb_v3' TOTAL_PHYSIOLOGICAL_FEATURE_LENGTH = ( - BASE_PHYSIOLOGICAL_FEATURE_LENGTH - + RESP_FEATURE_LENGTH + RESP_FEATURE_LENGTH + EEG_FEATURE_LENGTH + ECG_FEATURE_LENGTH ) @@ -151,7 +149,7 @@ def _save_cached_feature_vector(cache_file, feature_vector): def _compute_record_feature_vector(patient_data, data_folder, site_id, patient_id, session_id, csv_path, require_physiological_data): demographic_features = extract_demographic_features(patient_data) - physiological_data_file, algorithmic_annotations_file = _get_record_file_paths( + physiological_data_file, _ = _get_record_file_paths( data_folder, site_id, patient_id, @@ -170,13 +168,7 @@ def _compute_record_feature_vector(patient_data, data_folder, site_id, patient_i else: physiological_features = np.zeros(TOTAL_PHYSIOLOGICAL_FEATURE_LENGTH, dtype=np.float32) - if os.path.exists(algorithmic_annotations_file): - algorithmic_annotations, _ = load_signal_data(algorithmic_annotations_file) - algorithmic_features = extract_algorithmic_annotations_features(algorithmic_annotations) - else: - algorithmic_features = np.zeros(12, dtype=np.float32) - - return np.hstack([demographic_features, physiological_features, algorithmic_features]).astype(np.float32) + return np.hstack([demographic_features, physiological_features]).astype(np.float32) def get_or_create_record_feature_vector(record, data_folder, patient_data, csv_path=DEFAULT_CSV_PATH, require_physiological_data=True): @@ -202,10 +194,6 @@ def get_or_create_record_feature_vector(record, data_folder, patient_data, csv_p def extract_extended_physiological_features(physiological_data, physiological_fs, csv_path=DEFAULT_CSV_PATH): - base_features = _coerce_feature_vector( - extract_physiological_features(physiological_data, physiological_fs, csv_path=csv_path) - ) - try: resp_features = _extract_optional_features( processResp, @@ -227,7 +215,6 @@ def extract_extended_physiological_features(physiological_data, physiological_fs ) except Exception: eeg_features = np.zeros(EEG_FEATURE_LENGTH, dtype=np.float32) - try: ecg_features = _extract_optional_features( @@ -238,9 +225,9 @@ def extract_extended_physiological_features(physiological_data, physiological_fs csv_path=csv_path, ) except Exception: - ecg_features = np.zeros(ECG_FEATURE_LENGTH, dtype=np.float32) + ecg_features = np.zeros(ECG_FEATURE_LENGTH, dtype=np.float32) - return np.hstack([base_features, resp_features, eeg_features, ecg_features]).astype(np.float32) + return np.hstack([resp_features, eeg_features, ecg_features]).astype(np.float32) def process_training_record(record, data_folder, demographics_cache, diagnosis_cache, csv_path): @@ -360,7 +347,7 @@ def train_model(data_folder, model_folder, verbose, csv_path=DEFAULT_CSV_PATH): pbar.close() features = np.asarray(features, dtype=np.float32) - labels = np.asarray(labels, dtype=bool) + labels = np.asarray(labels, dtype=np.int32) if features.size == 0 or features.ndim != 2 or features.shape[0] == 0: raise ValueError('No valid training samples were extracted. Review feature extraction logs for the skipped records.') @@ -369,16 +356,21 @@ def train_model(data_folder, model_folder, verbose, csv_path=DEFAULT_CSV_PATH): if verbose: print('Training the model on the data...') - # This very simple model trains a random forest model with very simple features. - - # Define the parameters for the random forest classifier and regressor. - n_estimators = 12 # Number of trees in the forest. - max_leaf_nodes = 34 # Maximum number of leaf nodes in each tree. - random_state = 56 # Random state; set for reproducibility. - - # Fit the model. - model = RandomForestClassifier( - n_estimators=n_estimators, max_leaf_nodes=max_leaf_nodes, random_state=random_state).fit(features, labels) + neg = int(np.sum(labels == 0)) + pos = int(np.sum(labels == 1)) + scale_pos_weight = (neg / pos) if pos > 0 else 1.0 + + model = XGBClassifier( + scale_pos_weight=scale_pos_weight, + n_estimators=300, + max_depth=4, + learning_rate=0.05, + subsample=0.8, + random_state=42, + eval_metric='auc', + tree_method='hist', + ) + model.fit(features, labels) # Create a folder for the model if it does not already exist. os.makedirs(model_folder, exist_ok=True) From ddabe8ba4d949800e28040e8fc3a7714f848e95a Mon Sep 17 00:00:00 2001 From: dcajal Date: Mon, 30 Mar 2026 15:13:39 +0200 Subject: [PATCH 48/93] Update feature extraction to focus on demographic data and refine cache folder naming --- team_code.py | 326 ++------------------------------------------------- 1 file changed, 7 insertions(+), 319 deletions(-) diff --git a/team_code.py b/team_code.py index 8cbac15..6f9098c 100644 --- a/team_code.py +++ b/team_code.py @@ -44,7 +44,7 @@ RUN_PROGRESS_LINE_RE = re.compile(r'^-\s+\d+/\d+:\s') RENAME_RULES_CACHE = {} MAX_TRAIN_WORKERS = max(1, min(4, os.cpu_count() or 1)) -FEATURE_CACHE_FOLDER_NAME = '.feature_cache_xgb_v3' +FEATURE_CACHE_FOLDER_NAME = '.feature_cache' TOTAL_PHYSIOLOGICAL_FEATURE_LENGTH = ( RESP_FEATURE_LENGTH + EEG_FEATURE_LENGTH @@ -457,338 +457,26 @@ def run_model(model, record, data_folder, verbose): def extract_demographic_features(data): """ - Extracts and encodes demographic features from a metadata dictionary. + Extracts the demographic subset used by the current XGBoost model. Inputs: data (dict): A dictionary containing patient metadata (e.g., from a CSV row). Returns: - np.array: A feature vector of length 11: - - [0]: Age (Continuous) - - [1:4]: Sex (One-hot: Female, Male, Other/Unknown) - - [4:9]: Race (One-hot: Asian, Black, Other, Unavailable, White) - - [9]: BMI (Continuous) + np.array: A feature vector of length 4 with age and sex only. """ - # 1. Age Feature (1 dimension) - # Convert 'Age' to a float; default to 0 if missing age = np.array([load_age(data)]) - # 2. Sex One-Hot Encoding (3 dimensions: Female, Male, Other/Unknown) - # Uses lowercase prefix matching to handle variants like 'F', 'Female', 'M', or 'Male' sex = load_sex(data) sex_vec = np.zeros(3) if sex == 'Female': - sex_vec[0] = 1 # Index 0: Female + sex_vec[0] = 1 elif sex == 'Male': - sex_vec[1] = 1 # Index 1: Male + sex_vec[1] = 1 else: - sex_vec[2] = 1 # Index 2: Other/Unknown + sex_vec[2] = 1 - # 3. Race One-Hot Encoding (6 dimensions) - # Standardizes the raw text into one of six categories using the helper function - race_category = get_standardized_race(data).lower() - race_vec = np.zeros(5) - # Pre-defined mapping for index consistency - race_mapping = {'asian': 0, 'black': 1, 'others': 2, 'unavailable': 3, 'white': 4} - race_vec[race_mapping.get(race_category, 2)] = 1 - - # 4. Body Mass Index (BMI) Feature (1 dimension) - # Extracts the pre-calculated mean BMI; handles strings, NaNs, and missing keys - bmi = np.array([load_bmi(data)]) - - # 5. Concatenate all components into a single vector (1 + 3 + 5 + 1 = 10) - - return np.concatenate([age, sex_vec, race_vec, bmi]) - - -def extract_physiological_features(physiological_data, physiological_fs, csv_path=DEFAULT_CSV_PATH): - """ - Standardizes channels and extracts statistical/spectral features. - """ - original_labels = list(physiological_data.keys()) - - # Step 1: Load rules and standardize names - # Note: Use script-relative path or absolute path for robustness - rename_rules = get_rename_rules(csv_path) - rename_map, cols_to_drop = standardize_channel_names_rename_only(original_labels, rename_rules) - - # Step 2: Apply renaming to BOTH signals and their corresponding FS - processed_channels = {} - processed_fs = {} - for old_label, data in physiological_data.items(): - if old_label in cols_to_drop: - continue - new_label = rename_map.get(old_label, old_label.lower()) - processed_channels[new_label] = data - # Mapping the sampling rate to the new label - if old_label in physiological_fs: - processed_fs[new_label] = physiological_fs[old_label] - else: - # Report error and stop if no FS is found for a kept channel - raise KeyError(f"Sampling frequency (fs) not found for channel '{old_label}' ") - - if 'physiological_data' in locals(): del physiological_data - - # Step 3: Construct Bipolar Derivations - bipolar_configs = [ - ('f3-m2', 'f3', ['m2']), ('f4-m1', 'f4', ['m1']), - ('c3-m2', 'c3', ['m2']), ('c4-m1', 'c4', ['m1']), - ('o1-m2', 'o1', ['m2']), ('o2-m1', 'o2', ['m1']), - ('e1-m2', 'e1', ['m2']), ('e2-m1', 'e2', ['m1']), - ('chin1-chin2', 'chin 1', ['chin 2']), - ('lat', 'lleg+', ['lleg-']), ('rat', 'rleg+', ['rleg-']) - ] - - for target, pos, neg_list in bipolar_configs: - # 1. Skip if target already exists or pos channel missing - if target in processed_channels or pos not in processed_channels: - continue - - # 2. Check all neg channels exist - if not all(n in processed_channels for n in neg_list): - continue - - # 3. Check sampling rate consistency - all_involved = [pos] + neg_list - fs_values = [processed_fs[ch] for ch in all_involved] - - if len(set(fs_values)) > 1: - raise ValueError(f"Sampling rate mismatch for {target}: {dict(zip(all_involved, fs_values))}") - - # 4. Derive bipolar signal - ref_sig = processed_channels[neg_list[0]] if len(neg_list) == 1 else tuple(processed_channels[n] for n in neg_list) - - derived = derive_bipolar_signal(processed_channels[pos], ref_sig) - - if derived is not None: - processed_channels[target] = derived - processed_fs[target] = processed_fs[pos] - - leads_to_check = { - 'eeg': ['f3-m2', 'f4-m1', 'c3-m2', 'c4-m1'], - 'eog': ['e1-m2', 'e2-m1'], - 'chin': ['chin1-chin2', 'chin'], - 'leg': ['lat', 'rat'], - 'ecg': ['ecg', 'ekg'], - 'resp': ['airflow', 'ptaf', 'abd', 'chest'], - 'spo2': ['spo2', 'sao2'] # Added sao2 as fallback for spo2 - } - - final_features = [] - for lead_type, candidates in leads_to_check.items(): - sig = None - fs = None - - # Identify the first available candidate - for candidate in candidates: - if candidate in processed_channels and processed_channels[candidate] is not None: - sig = processed_channels[candidate] - fs = processed_fs.get(candidate) - break - - # if sig is not None and len(sig) > 0 and fs is not None: - # # --- 1. Time Domain Features --- - # std_val = np.std(sig) - # mav_val = np.mean(np.abs(sig)) - # energy_val = np.sum(sig**2) / len(sig) - - # # --- 2. Frequency Domain Features (Spectral) --- - # n = len(sig) - # # Correct spacing for frequency axis based on channel-specific fs - # freqs = np.fft.rfftfreq(n, d=1/fs) - - # # Compute Power Spectral Density (PSD) - # # Multiplied by 2 for rfft (except DC/Nyquist) and divided by fs for density - # fft_res = np.abs(np.fft.rfft(sig)) - # psd = (fft_res**2) / (n * fs) - - # # Define band masks - # delta_mask = (freqs >= 0.5) & (freqs <= 4) - # theta_mask = (freqs > 4) & (freqs <= 8) - # alpha_mask = (freqs > 8) & (freqs <= 12) - - # # Calculate power in bands using trapezoidal integration for physical accuracy - # delta_p = np.trapezoid(psd[delta_mask], freqs[delta_mask]) if np.any(delta_mask) else 0.0 - # theta_p = np.trapezoid(psd[theta_mask], freqs[theta_mask]) if np.any(theta_mask) else 0.0 - # alpha_p = np.trapezoid(psd[alpha_mask], freqs[alpha_mask]) if np.any(alpha_mask) else 0.0 - - # # Ratio biomarker: Delta/Theta (Indicator of cognitive slowing) - # dt_ratio = delta_p / theta_p if theta_p > 0 else 0.0 - - # final_features.extend([std_val, mav_val, energy_val, delta_p, theta_p, alpha_p, dt_ratio]) - - if sig is not None and len(sig) > 1: - # --- Time Domain Features (Very Fast) --- - std_val = np.std(sig) - mav_val = np.mean(np.abs(sig)) - - # Zero Crossing Rate (Proxy for frequency/slowing) - zcr = np.mean(np.diff(np.sign(sig)) != 0) - - # Root Mean Square - rms = np.sqrt(np.mean(sig**2)) - - # Signal Activity (Variance) - activity = np.var(sig) - - # Mobility (Hjorth Parameter) - Proxy for mean frequency - # sqrt(var(diff(sig)) / var(sig)) - diff_sig = np.diff(sig) - mobility = np.sqrt(np.var(diff_sig) / activity) if activity > 0 else 0.0 - - # Complexity (Hjorth Parameter) - Proxy for bandwidth - diff2_sig = np.diff(diff_sig) - var_d2 = np.var(diff2_sig) - var_d1 = np.var(diff_sig) - complexity = (np.sqrt(var_d2 / var_d1) / mobility) if (var_d1 > 0 and mobility > 0) else 0.0 - - final_features.extend([std_val, mav_val, zcr, rms, activity, mobility, complexity]) - - else: - # Padding: 7 features per lead type - final_features.extend([0.0] * 7) - - if 'processed_channels' in locals(): del processed_channels - - return np.array(final_features) - -def extract_algorithmic_annotations_features(algo_data): - """ - Extracts sleep architecture and event density features from CAISR outputs. - Output vector length: 12 - """ - if not algo_data: - return np.zeros(12) - - features = [] - - # --- 1. Respiratory & Arousal Event Densities --- - # Total duration in hours (assuming 1Hz for event traces) - # If the signal exists, we calculate events per hour (Index) - total_hours = len(algo_data.get('resp_caisr', [])) / 3600.0 - - def count_discrete_events(key): - if key not in algo_data or total_hours <= 0: - return 0.0 - - sig = algo_data[key].astype(float) - # Create a binary mask: 1 if there is an event, 0 if not - binary_sig = (sig > 0).astype(int) - - # Detect rising edges: 0 to 1 transition - # diff will be 1 at the start of an event, -1 at the end - diff = np.diff(binary_sig, prepend=0) - num_events = np.count_nonzero(diff == 1) - - return num_events / total_hours - - ahi_auto = count_discrete_events('resp_caisr') # Automated Apnea-Hypopnea Index - arousal_auto = count_discrete_events('arousal_caisr') # Automated Arousal Index - limb_auto = count_discrete_events('limb_caisr') # Automated Limb Movement Index - - features.extend([ahi_auto, arousal_auto, limb_auto]) - - # --- 2. Sleep Architecture (from stage_caisr) --- - # Standard labels: 5=W, 4=R, 3=N1, 2=N2, 1=N3 (or similar mapping) - stages = algo_data.get('stage_caisr', np.array([])) - # Filter out invalid/background values (like the 9.0 in your sample) - valid_stages = stages[stages < 9.0] - - if len(valid_stages) > 0: - total_epochs = len(valid_stages) - # Percentage of each stage - w_pct = np.mean(valid_stages == 5) - r_pct = np.mean(valid_stages == 4) - n1_pct = np.mean(valid_stages == 3) - n2_pct = np.mean(valid_stages == 2) - n3_pct = np.mean(valid_stages == 1) - - # Sleep Efficiency: (N1+N2+N3+R) / Total - efficiency = np.mean((valid_stages >= 1) & (valid_stages <= 4)) - else: - w_pct = n1_pct = n2_pct = n3_pct = r_pct = efficiency = 0.0 - - features.extend([w_pct, n1_pct, n2_pct, n3_pct, r_pct, efficiency]) - - # --- 3. Model Confidence / Uncertainty --- - # Mean probability of Wake and REM (indicators of sleep stability) - # We use the raw probability traces - prob_w = np.mean(algo_data.get('caisr_prob_w', [0])) - prob_n3 = np.mean(algo_data.get('caisr_prob_n3', [0])) - prob_arous = np.mean(algo_data.get('caisr_prob_arous', [0])) - - # Standardize '9.0' or other filler values to 0 - clean_prob = lambda x: x if x < 1.0 else 0.0 - features.extend([clean_prob(prob_w), clean_prob(prob_n3), clean_prob(prob_arous)]) - - return np.array(features) - -def extract_human_annotations_features(human_data): - """ - Extracts features from expert-scored human annotations. - Output vector length: 12 (to match algorithmic feature length) - """ - # If data is missing (common in hidden test sets), return a zero vector - if not human_data or 'resp_expert' not in human_data: - return np.zeros(12) - - features = [] - - # --- 1. Human Event Indices (Events per Hour) --- - # Total duration in hours based on 1Hz signal - total_seconds = len(human_data.get('resp_expert', [])) - total_hours = total_seconds / 3600.0 - - def count_discrete_events(key): - if key not in human_data or total_hours <= 0: - return 0.0 - sig = (human_data[key] > 0).astype(int) - # Identify the start of each continuous event block - diff = np.diff(sig, prepend=0) - return np.count_nonzero(diff == 1) / total_hours - - ahi_human = count_discrete_events('resp_expert') # Human AHI - arousal_human = count_discrete_events('arousal_expert') # Human Arousal Index - limb_human = count_discrete_events('limb_expert') # Human PLMI - - features.extend([ahi_human, arousal_human, limb_human]) - - # --- 2. Human Sleep Architecture --- - # Standard labels: 0=W, 1=N1, 2=N2, 3=N3, 4=R, 5=Unknown/Movement - stages = human_data.get('stage_expert', np.array([])) - - # Filter out label 5 (often used by experts for movement/unscored) - valid_mask = (stages < 9.0) - valid_stages = stages[valid_mask] - - if len(valid_stages) > 0: - w_pct = np.mean(valid_stages == 5) - r_pct = np.mean(valid_stages == 4) - n1_pct = np.mean(valid_stages == 3) - n2_pct = np.mean(valid_stages == 2) - n3_pct = np.mean(valid_stages == 1) - efficiency = np.mean(valid_stages > 0) - else: - w_pct = n1_pct = n2_pct = n3_pct = r_pct = efficiency = 0.0 - - features.extend([w_pct, n1_pct, n2_pct, n3_pct, r_pct, efficiency]) - - # --- 3. Fragmentation & Stability (Replacing Probabilities) --- - # These metrics quantify how "broken" the sleep is, which is a key marker. - if len(valid_stages) > 1: - # Number of stage transitions - transitions = np.count_nonzero(np.diff(valid_stages)) / total_hours - # Wake After Sleep Onset (WASO) proxy: non-zero stages followed by zero - waso_minutes = (np.count_nonzero(valid_stages == 0) * 30) / 60.0 - # REM Latency (epochs until first REM) - rem_indices = np.where(valid_stages == 4)[0] - rem_latency = rem_indices[0] if len(rem_indices) > 0 else 0.0 - else: - transitions = waso_minutes = rem_latency = 0.0 - - features.extend([transitions, waso_minutes, rem_latency]) - - return np.array(features) + return np.concatenate([age, sex_vec]) # Save your trained model. From 4c41cf2e6c4ac018818df2b04ecf915048432ab6 Mon Sep 17 00:00:00 2001 From: dcajal Date: Mon, 30 Mar 2026 15:35:43 +0200 Subject: [PATCH 49/93] Add signal alias handling and improve data loading for EEG and respiratory signals --- team_code.py | 65 ++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 61 insertions(+), 4 deletions(-) diff --git a/team_code.py b/team_code.py index 6f9098c..ece7784 100644 --- a/team_code.py +++ b/team_code.py @@ -14,6 +14,7 @@ import os import atexit import builtins +import edfio import hashlib import pandas as pd import re @@ -23,9 +24,9 @@ from tqdm import tqdm from helper_code import * -from src.resp_processing import RESP_FEATURE_LENGTH, processResp -from src.eeg_processing import EEG_FEATURE_LENGTH, processEEG -from src.ecg_processing import ECG_FEATURE_LENGTH, processECG +from src.resp_processing import RESP_FEATURE_LENGTH, processResp, _get_resp_alias_groups +from src.eeg_processing import EEG_CHANNEL_SPECS, EEG_FEATURE_LENGTH, processEEG, _get_eeg_aliases, _normalize_label as _normalize_eeg_label +from src.ecg_processing import ECG_FEATURE_LENGTH, ECG_KEYWORDS, processECG ################################################################################ # Path & Constant Configuration (Added for Robustness) ################################################################################ @@ -43,6 +44,7 @@ PRINT_FILTER_ACTIVE = False RUN_PROGRESS_LINE_RE = re.compile(r'^-\s+\d+/\d+:\s') RENAME_RULES_CACHE = {} +REQUIRED_SIGNAL_ALIASES_CACHE = {} MAX_TRAIN_WORKERS = max(1, min(4, os.cpu_count() or 1)) FEATURE_CACHE_FOLDER_NAME = '.feature_cache' TOTAL_PHYSIOLOGICAL_FEATURE_LENGTH = ( @@ -105,6 +107,61 @@ def _get_record_file_paths(data_folder, site_id, patient_id, session_id): return physiological_data_file, algorithmic_annotations_file +def _normalize_signal_label(text): + normalized = ''.join(ch if ch.isalnum() else ' ' for ch in str(text).lower()) + return ' '.join(normalized.split()) + + +def _get_required_signal_aliases(csv_path): + normalized_csv_path = os.path.abspath(csv_path) + required_aliases = REQUIRED_SIGNAL_ALIASES_CACHE.get(normalized_csv_path) + if required_aliases is not None: + return required_aliases + + resp_alias_groups = _get_resp_alias_groups(normalized_csv_path) + eeg_aliases = _get_eeg_aliases(normalized_csv_path) + + required_aliases = set() + for aliases in resp_alias_groups.values(): + required_aliases.update(aliases) + + for channel_spec in EEG_CHANNEL_SPECS.values(): + required_aliases.update(eeg_aliases.get(_normalize_eeg_label(channel_spec['direct']), set())) + required_aliases.update(eeg_aliases.get(_normalize_eeg_label(channel_spec['positive']), set())) + required_aliases.update(eeg_aliases.get(_normalize_eeg_label(channel_spec['reference']), set())) + + REQUIRED_SIGNAL_ALIASES_CACHE[normalized_csv_path] = required_aliases + return required_aliases + + +def _load_required_signal_data(edf_path, csv_path): + required_aliases = _get_required_signal_aliases(csv_path) + channel_dict = {} + fs_dict = {} + + try: + edf = edfio.read_edf(edf_path, lazy_load_data=True) + except Exception: + return load_signal_data(edf_path) + + for signal in edf.signals: + label = signal.label.lower().strip() + normalized_label = _normalize_signal_label(label) + is_required_signal = normalized_label in required_aliases + is_ecg_signal = any(keyword in normalized_label for keyword in ECG_KEYWORDS) + + if not is_required_signal and not is_ecg_signal: + continue + + fs_dict[label] = float(signal.sampling_frequency) + channel_dict[label] = signal.data + + if channel_dict: + return channel_dict, fs_dict + + return load_signal_data(edf_path) + + def _get_feature_cache_file(data_folder, site_id, patient_id, session_id): folder_hash = hashlib.sha1(os.path.abspath(data_folder).encode('utf-8')).hexdigest()[:12] cache_dir = os.path.join( @@ -157,7 +214,7 @@ def _compute_record_feature_vector(patient_data, data_folder, site_id, patient_i ) if os.path.exists(physiological_data_file): - physiological_data, physiological_fs = load_signal_data(physiological_data_file) + physiological_data, physiological_fs = _load_required_signal_data(physiological_data_file, csv_path) physiological_features = extract_extended_physiological_features( physiological_data, physiological_fs, From 486b36f40a51f71137e4f07d08c67f71bc14034f Mon Sep 17 00:00:00 2001 From: dcajal Date: Mon, 30 Mar 2026 15:59:02 +0200 Subject: [PATCH 50/93] Add initial pipeline modules and refactor training logic --- src/pipeline/__init__.py | 1 + src/pipeline/config.py | 16 ++ src/pipeline/features.py | 234 ++++++++++++++++++++++++ src/pipeline/training.py | 115 ++++++++++++ team_code.py | 383 +-------------------------------------- 5 files changed, 370 insertions(+), 379 deletions(-) create mode 100644 src/pipeline/__init__.py create mode 100644 src/pipeline/config.py create mode 100644 src/pipeline/features.py create mode 100644 src/pipeline/training.py diff --git a/src/pipeline/__init__.py b/src/pipeline/__init__.py new file mode 100644 index 0000000..beafece --- /dev/null +++ b/src/pipeline/__init__.py @@ -0,0 +1 @@ +"""Internal pipeline modules for the active Challenge submission.""" \ No newline at end of file diff --git a/src/pipeline/config.py b/src/pipeline/config.py new file mode 100644 index 0000000..9845da8 --- /dev/null +++ b/src/pipeline/config.py @@ -0,0 +1,16 @@ +import os + +from src.ecg_processing import ECG_FEATURE_LENGTH +from src.eeg_processing import EEG_FEATURE_LENGTH +from src.resp_processing import RESP_FEATURE_LENGTH + + +SCRIPT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')) +DEFAULT_CSV_PATH = os.path.join(SCRIPT_DIR, 'channel_table.csv') +FEATURE_CACHE_FOLDER_NAME = '.feature_cache' +MAX_TRAIN_WORKERS = max(1, min(4, os.cpu_count() or 1)) +TOTAL_PHYSIOLOGICAL_FEATURE_LENGTH = ( + RESP_FEATURE_LENGTH + + EEG_FEATURE_LENGTH + + ECG_FEATURE_LENGTH +) \ No newline at end of file diff --git a/src/pipeline/features.py b/src/pipeline/features.py new file mode 100644 index 0000000..c202fe8 --- /dev/null +++ b/src/pipeline/features.py @@ -0,0 +1,234 @@ +import hashlib +import os + +import edfio +import joblib +import numpy as np + +from helper_code import HEADERS, PHYSIOLOGICAL_DATA_SUBFOLDER, load_age, load_sex, load_signal_data +from src.ecg_processing import ECG_FEATURE_LENGTH, ECG_KEYWORDS, processECG +from src.eeg_processing import EEG_CHANNEL_SPECS, EEG_FEATURE_LENGTH, processEEG, _get_eeg_aliases, _normalize_label as _normalize_eeg_label +from src.resp_processing import RESP_FEATURE_LENGTH, processResp, _get_resp_alias_groups + +from .config import DEFAULT_CSV_PATH, FEATURE_CACHE_FOLDER_NAME, SCRIPT_DIR, TOTAL_PHYSIOLOGICAL_FEATURE_LENGTH + + +REQUIRED_SIGNAL_ALIASES_CACHE = {} + + +def _coerce_feature_vector(features): + vector = np.asarray(features, dtype=np.float32).reshape(-1) + return np.nan_to_num(vector, nan=0.0, posinf=0.0, neginf=0.0) + + +def _extract_optional_features(extractor, expected_length, *args, **kwargs): + vector = _coerce_feature_vector(extractor(*args, **kwargs)) + if vector.size != expected_length: + raise ValueError( + f"{extractor.__name__} returned {vector.size} features; expected {expected_length}." + ) + return vector + + +def _get_physiological_data_file(data_folder, site_id, patient_id, session_id): + return os.path.join( + data_folder, + PHYSIOLOGICAL_DATA_SUBFOLDER, + site_id, + f"{patient_id}_ses-{session_id}.edf", + ) + + +def _normalize_signal_label(text): + normalized = ''.join(ch if ch.isalnum() else ' ' for ch in str(text).lower()) + return ' '.join(normalized.split()) + + +def _get_required_signal_aliases(csv_path): + normalized_csv_path = os.path.abspath(csv_path) + required_aliases = REQUIRED_SIGNAL_ALIASES_CACHE.get(normalized_csv_path) + if required_aliases is not None: + return required_aliases + + resp_alias_groups = _get_resp_alias_groups(normalized_csv_path) + eeg_aliases = _get_eeg_aliases(normalized_csv_path) + + required_aliases = set() + for aliases in resp_alias_groups.values(): + required_aliases.update(aliases) + + for channel_spec in EEG_CHANNEL_SPECS.values(): + required_aliases.update(eeg_aliases.get(_normalize_eeg_label(channel_spec['direct']), set())) + required_aliases.update(eeg_aliases.get(_normalize_eeg_label(channel_spec['positive']), set())) + required_aliases.update(eeg_aliases.get(_normalize_eeg_label(channel_spec['reference']), set())) + + REQUIRED_SIGNAL_ALIASES_CACHE[normalized_csv_path] = required_aliases + return required_aliases + + +def _load_required_signal_data(edf_path, csv_path): + required_aliases = _get_required_signal_aliases(csv_path) + channel_dict = {} + fs_dict = {} + + try: + edf = edfio.read_edf(edf_path, lazy_load_data=True) + except Exception: + return load_signal_data(edf_path) + + for signal in edf.signals: + label = signal.label.lower().strip() + normalized_label = _normalize_signal_label(label) + is_required_signal = normalized_label in required_aliases + is_ecg_signal = any(keyword in normalized_label for keyword in ECG_KEYWORDS) + + if not is_required_signal and not is_ecg_signal: + continue + + fs_dict[label] = float(signal.sampling_frequency) + channel_dict[label] = signal.data + + if channel_dict: + return channel_dict, fs_dict + + return load_signal_data(edf_path) + + +def _get_feature_cache_file(data_folder, site_id, patient_id, session_id): + folder_hash = hashlib.sha1(os.path.abspath(data_folder).encode('utf-8')).hexdigest()[:12] + cache_dir = os.path.join( + SCRIPT_DIR, + FEATURE_CACHE_FOLDER_NAME, + folder_hash, + site_id, + ) + return os.path.join(cache_dir, f"{patient_id}_ses-{session_id}.sav") + + +def _load_cached_feature_vector(cache_file): + if not os.path.exists(cache_file): + return None + + try: + payload = joblib.load(cache_file) + except Exception: + return None + + if isinstance(payload, dict): + payload = payload.get('features') + + if payload is None: + return None + + return _coerce_feature_vector(payload) + + +def _save_cached_feature_vector(cache_file, feature_vector): + os.makedirs(os.path.dirname(cache_file), exist_ok=True) + temp_cache_file = f"{cache_file}.tmp" + payload = _coerce_feature_vector(feature_vector) + + try: + joblib.dump(payload, temp_cache_file, protocol=0) + os.replace(temp_cache_file, cache_file) + finally: + if os.path.exists(temp_cache_file): + os.remove(temp_cache_file) + + +def extract_demographic_features(data): + age = np.array([load_age(data)]) + + sex = load_sex(data) + sex_vec = np.zeros(3) + if sex == 'Female': + sex_vec[0] = 1 + elif sex == 'Male': + sex_vec[1] = 1 + else: + sex_vec[2] = 1 + + return np.concatenate([age, sex_vec]) + + +def extract_extended_physiological_features(physiological_data, physiological_fs, csv_path=DEFAULT_CSV_PATH): + try: + resp_features = _extract_optional_features( + processResp, + RESP_FEATURE_LENGTH, + physiological_data, + physiological_fs, + csv_path=csv_path, + ) + except Exception: + resp_features = np.zeros(RESP_FEATURE_LENGTH, dtype=np.float32) + + try: + eeg_features = _extract_optional_features( + processEEG, + EEG_FEATURE_LENGTH, + physiological_data, + physiological_fs, + csv_path=csv_path, + ) + except Exception: + eeg_features = np.zeros(EEG_FEATURE_LENGTH, dtype=np.float32) + + try: + ecg_features = _extract_optional_features( + processECG, + ECG_FEATURE_LENGTH, + physiological_data, + physiological_fs, + csv_path=csv_path, + ) + except Exception: + ecg_features = np.zeros(ECG_FEATURE_LENGTH, dtype=np.float32) + + return np.hstack([resp_features, eeg_features, ecg_features]).astype(np.float32) + + +def _compute_record_feature_vector(patient_data, data_folder, site_id, patient_id, session_id, csv_path, require_physiological_data): + demographic_features = extract_demographic_features(patient_data) + physiological_data_file = _get_physiological_data_file( + data_folder, + site_id, + patient_id, + session_id, + ) + + if os.path.exists(physiological_data_file): + physiological_data, physiological_fs = _load_required_signal_data(physiological_data_file, csv_path) + physiological_features = extract_extended_physiological_features( + physiological_data, + physiological_fs, + csv_path=csv_path, + ) + elif require_physiological_data: + raise FileNotFoundError(f"Missing physiological data for {patient_id}.") + else: + physiological_features = np.zeros(TOTAL_PHYSIOLOGICAL_FEATURE_LENGTH, dtype=np.float32) + + return np.hstack([demographic_features, physiological_features]).astype(np.float32) + + +def get_or_create_record_feature_vector(record, data_folder, patient_data, csv_path=DEFAULT_CSV_PATH, require_physiological_data=True): + patient_id = record[HEADERS['bids_folder']] + site_id = record[HEADERS['site_id']] + session_id = record[HEADERS['session_id']] + cache_file = _get_feature_cache_file(data_folder, site_id, patient_id, session_id) + cached_features = _load_cached_feature_vector(cache_file) + if cached_features is not None: + return cached_features + + feature_vector = _compute_record_feature_vector( + patient_data, + data_folder, + site_id, + patient_id, + session_id, + csv_path, + require_physiological_data, + ) + _save_cached_feature_vector(cache_file, feature_vector) + return feature_vector \ No newline at end of file diff --git a/src/pipeline/training.py b/src/pipeline/training.py new file mode 100644 index 0000000..ae2c34a --- /dev/null +++ b/src/pipeline/training.py @@ -0,0 +1,115 @@ +import os +from concurrent.futures import ThreadPoolExecutor + +import numpy as np +import pandas as pd +from tqdm import tqdm +from xgboost import XGBClassifier + +from helper_code import DEMOGRAPHICS_FILE, HEADERS, find_patients, load_label + +from .config import MAX_TRAIN_WORKERS +from .features import get_or_create_record_feature_vector + + +def build_training_metadata_cache(patient_data_file): + metadata = pd.read_csv(patient_data_file) + demographics_cache = {} + diagnosis_cache = {} + + for row in metadata.to_dict('records'): + patient_id = row[HEADERS['bids_folder']] + session_id = row[HEADERS['session_id']] + demographics_cache[(patient_id, session_id)] = row + diagnosis_cache[patient_id] = load_label(row) + + return demographics_cache, diagnosis_cache + + +def process_training_record(record, data_folder, demographics_cache, diagnosis_cache, csv_path): + patient_id = record[HEADERS['bids_folder']] + session_id = record[HEADERS['session_id']] + + try: + patient_data = demographics_cache.get((patient_id, session_id), {}) + feature_vector = get_or_create_record_feature_vector( + record, + data_folder, + patient_data, + csv_path=csv_path, + require_physiological_data=True, + ) + + label = diagnosis_cache.get(patient_id) + + if label == 0 or label == 1: + return patient_id, feature_vector, label, None + + return patient_id, None, None, f"Invalid label for {patient_id}. Skipping..." + + except FileNotFoundError as exc: + return patient_id, None, None, f"{exc} Skipping..." + except Exception as exc: + return patient_id, None, None, f"Error processing {patient_id}: {exc}" + + +def train_xgb_model(data_folder, verbose, csv_path): + patient_data_file = os.path.join(data_folder, DEMOGRAPHICS_FILE) + patient_metadata_list = find_patients(patient_data_file) + demographics_cache, diagnosis_cache = build_training_metadata_cache(patient_data_file) + num_records = len(patient_metadata_list) + + if num_records == 0: + raise FileNotFoundError('No data were provided.') + + features = [] + labels = [] + + with ThreadPoolExecutor(max_workers=MAX_TRAIN_WORKERS) as executor: + results = executor.map( + lambda record: process_training_record( + record, + data_folder, + demographics_cache, + diagnosis_cache, + csv_path, + ), + patient_metadata_list, + ) + + pbar = tqdm(results, total=num_records, desc='Extracting Features', unit='record', disable=not verbose) + for patient_id, feature_vector, label, message in pbar: + if verbose: + pbar.set_postfix({'patient': patient_id}) + + if message is not None: + tqdm.write(f" ! {message}") + continue + + features.append(feature_vector) + labels.append(label) + + pbar.close() + + features = np.asarray(features, dtype=np.float32) + labels = np.asarray(labels, dtype=np.int32) + + if features.size == 0 or features.ndim != 2 or features.shape[0] == 0: + raise ValueError('No valid training samples were extracted. Review feature extraction logs for the skipped records.') + + neg = int(np.sum(labels == 0)) + pos = int(np.sum(labels == 1)) + scale_pos_weight = (neg / pos) if pos > 0 else 1.0 + + model = XGBClassifier( + scale_pos_weight=scale_pos_weight, + n_estimators=300, + max_depth=4, + learning_rate=0.05, + subsample=0.8, + random_state=42, + eval_metric='auc', + tree_method='hist', + ) + model.fit(features, labels) + return model \ No newline at end of file diff --git a/team_code.py b/team_code.py index ece7784..35b1c11 100644 --- a/team_code.py +++ b/team_code.py @@ -10,310 +10,27 @@ ################################################################################ import joblib -import numpy as np import os import atexit import builtins -import edfio -import hashlib -import pandas as pd import re -from concurrent.futures import ThreadPoolExecutor -from xgboost import XGBClassifier import sys from tqdm import tqdm from helper_code import * -from src.resp_processing import RESP_FEATURE_LENGTH, processResp, _get_resp_alias_groups -from src.eeg_processing import EEG_CHANNEL_SPECS, EEG_FEATURE_LENGTH, processEEG, _get_eeg_aliases, _normalize_label as _normalize_eeg_label -from src.ecg_processing import ECG_FEATURE_LENGTH, ECG_KEYWORDS, processECG +from src.pipeline.config import DEFAULT_CSV_PATH +from src.pipeline.features import _get_feature_cache_file, get_or_create_record_feature_vector +from src.pipeline.training import train_xgb_model ################################################################################ # Path & Constant Configuration (Added for Robustness) ################################################################################ -# Get the absolute directory where this script is located -SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) - -# Build the absolute path to the CSV file relative to the script location -DEFAULT_CSV_PATH = os.path.join(SCRIPT_DIR, 'channel_table.csv') - # Progress bar state for run_model (initialized lazily) RUN_MODEL_PBAR = None RUN_MODEL_PBAR_TOTAL = None ORIGINAL_PRINT = builtins.print PRINT_FILTER_ACTIVE = False RUN_PROGRESS_LINE_RE = re.compile(r'^-\s+\d+/\d+:\s') -RENAME_RULES_CACHE = {} -REQUIRED_SIGNAL_ALIASES_CACHE = {} -MAX_TRAIN_WORKERS = max(1, min(4, os.cpu_count() or 1)) -FEATURE_CACHE_FOLDER_NAME = '.feature_cache' -TOTAL_PHYSIOLOGICAL_FEATURE_LENGTH = ( - RESP_FEATURE_LENGTH - + EEG_FEATURE_LENGTH - + ECG_FEATURE_LENGTH -) - - -def build_training_metadata_cache(patient_data_file): - metadata = pd.read_csv(patient_data_file) - demographics_cache = {} - diagnosis_cache = {} - - for row in metadata.to_dict('records'): - patient_id = row[HEADERS['bids_folder']] - session_id = row[HEADERS['session_id']] - demographics_cache[(patient_id, session_id)] = row - diagnosis_cache[patient_id] = load_label(row) - - return demographics_cache, diagnosis_cache - - -def get_rename_rules(csv_path): - normalized_csv_path = os.path.abspath(csv_path) - rename_rules = RENAME_RULES_CACHE.get(normalized_csv_path) - if rename_rules is None: - rename_rules = load_rename_rules(normalized_csv_path) - RENAME_RULES_CACHE[normalized_csv_path] = rename_rules - return rename_rules - - -def _coerce_feature_vector(features): - vector = np.asarray(features, dtype=np.float32).reshape(-1) - return np.nan_to_num(vector, nan=0.0, posinf=0.0, neginf=0.0) - - -def _extract_optional_features(extractor, expected_length, *args, **kwargs): - vector = _coerce_feature_vector(extractor(*args, **kwargs)) - if vector.size != expected_length: - raise ValueError( - f"{extractor.__name__} returned {vector.size} features; expected {expected_length}." - ) - return vector - - -def _get_record_file_paths(data_folder, site_id, patient_id, session_id): - physiological_data_file = os.path.join( - data_folder, - PHYSIOLOGICAL_DATA_SUBFOLDER, - site_id, - f"{patient_id}_ses-{session_id}.edf" - ) - algorithmic_annotations_file = os.path.join( - data_folder, - ALGORITHMIC_ANNOTATIONS_SUBFOLDER, - site_id, - f"{patient_id}_ses-{session_id}_caisr_annotations.edf" - ) - return physiological_data_file, algorithmic_annotations_file - - -def _normalize_signal_label(text): - normalized = ''.join(ch if ch.isalnum() else ' ' for ch in str(text).lower()) - return ' '.join(normalized.split()) - - -def _get_required_signal_aliases(csv_path): - normalized_csv_path = os.path.abspath(csv_path) - required_aliases = REQUIRED_SIGNAL_ALIASES_CACHE.get(normalized_csv_path) - if required_aliases is not None: - return required_aliases - - resp_alias_groups = _get_resp_alias_groups(normalized_csv_path) - eeg_aliases = _get_eeg_aliases(normalized_csv_path) - - required_aliases = set() - for aliases in resp_alias_groups.values(): - required_aliases.update(aliases) - - for channel_spec in EEG_CHANNEL_SPECS.values(): - required_aliases.update(eeg_aliases.get(_normalize_eeg_label(channel_spec['direct']), set())) - required_aliases.update(eeg_aliases.get(_normalize_eeg_label(channel_spec['positive']), set())) - required_aliases.update(eeg_aliases.get(_normalize_eeg_label(channel_spec['reference']), set())) - - REQUIRED_SIGNAL_ALIASES_CACHE[normalized_csv_path] = required_aliases - return required_aliases - - -def _load_required_signal_data(edf_path, csv_path): - required_aliases = _get_required_signal_aliases(csv_path) - channel_dict = {} - fs_dict = {} - - try: - edf = edfio.read_edf(edf_path, lazy_load_data=True) - except Exception: - return load_signal_data(edf_path) - - for signal in edf.signals: - label = signal.label.lower().strip() - normalized_label = _normalize_signal_label(label) - is_required_signal = normalized_label in required_aliases - is_ecg_signal = any(keyword in normalized_label for keyword in ECG_KEYWORDS) - - if not is_required_signal and not is_ecg_signal: - continue - - fs_dict[label] = float(signal.sampling_frequency) - channel_dict[label] = signal.data - - if channel_dict: - return channel_dict, fs_dict - - return load_signal_data(edf_path) - - -def _get_feature_cache_file(data_folder, site_id, patient_id, session_id): - folder_hash = hashlib.sha1(os.path.abspath(data_folder).encode('utf-8')).hexdigest()[:12] - cache_dir = os.path.join( - SCRIPT_DIR, - FEATURE_CACHE_FOLDER_NAME, - folder_hash, - site_id, - ) - return os.path.join(cache_dir, f"{patient_id}_ses-{session_id}.sav") - - -def _load_cached_feature_vector(cache_file): - if not os.path.exists(cache_file): - return None - - try: - payload = joblib.load(cache_file) - except Exception: - return None - - if isinstance(payload, dict): - payload = payload.get('features') - - if payload is None: - return None - - return _coerce_feature_vector(payload) - - -def _save_cached_feature_vector(cache_file, feature_vector): - os.makedirs(os.path.dirname(cache_file), exist_ok=True) - temp_cache_file = f"{cache_file}.tmp" - payload = _coerce_feature_vector(feature_vector) - - try: - joblib.dump(payload, temp_cache_file, protocol=0) - os.replace(temp_cache_file, cache_file) - finally: - if os.path.exists(temp_cache_file): - os.remove(temp_cache_file) - - -def _compute_record_feature_vector(patient_data, data_folder, site_id, patient_id, session_id, csv_path, require_physiological_data): - demographic_features = extract_demographic_features(patient_data) - physiological_data_file, _ = _get_record_file_paths( - data_folder, - site_id, - patient_id, - session_id, - ) - - if os.path.exists(physiological_data_file): - physiological_data, physiological_fs = _load_required_signal_data(physiological_data_file, csv_path) - physiological_features = extract_extended_physiological_features( - physiological_data, - physiological_fs, - csv_path=csv_path, - ) - elif require_physiological_data: - raise FileNotFoundError(f"Missing physiological data for {patient_id}.") - else: - physiological_features = np.zeros(TOTAL_PHYSIOLOGICAL_FEATURE_LENGTH, dtype=np.float32) - - return np.hstack([demographic_features, physiological_features]).astype(np.float32) - - -def get_or_create_record_feature_vector(record, data_folder, patient_data, csv_path=DEFAULT_CSV_PATH, require_physiological_data=True): - patient_id = record[HEADERS['bids_folder']] - site_id = record[HEADERS['site_id']] - session_id = record[HEADERS['session_id']] - cache_file = _get_feature_cache_file(data_folder, site_id, patient_id, session_id) - cached_features = _load_cached_feature_vector(cache_file) - if cached_features is not None: - return cached_features - - feature_vector = _compute_record_feature_vector( - patient_data, - data_folder, - site_id, - patient_id, - session_id, - csv_path, - require_physiological_data, - ) - _save_cached_feature_vector(cache_file, feature_vector) - return feature_vector - - -def extract_extended_physiological_features(physiological_data, physiological_fs, csv_path=DEFAULT_CSV_PATH): - try: - resp_features = _extract_optional_features( - processResp, - RESP_FEATURE_LENGTH, - physiological_data, - physiological_fs, - csv_path=csv_path, - ) - except Exception: - resp_features = np.zeros(RESP_FEATURE_LENGTH, dtype=np.float32) - - try: - eeg_features = _extract_optional_features( - processEEG, - EEG_FEATURE_LENGTH, - physiological_data, - physiological_fs, - csv_path=csv_path, - ) - except Exception: - eeg_features = np.zeros(EEG_FEATURE_LENGTH, dtype=np.float32) - - try: - ecg_features = _extract_optional_features( - processECG, - ECG_FEATURE_LENGTH, - physiological_data, - physiological_fs, - csv_path=csv_path, - ) - except Exception: - ecg_features = np.zeros(ECG_FEATURE_LENGTH, dtype=np.float32) - - return np.hstack([resp_features, eeg_features, ecg_features]).astype(np.float32) - - -def process_training_record(record, data_folder, demographics_cache, diagnosis_cache, csv_path): - patient_id = record[HEADERS['bids_folder']] - session_id = record[HEADERS['session_id']] - - try: - patient_data = demographics_cache.get((patient_id, session_id), {}) - feature_vector = get_or_create_record_feature_vector( - record, - data_folder, - patient_data, - csv_path=csv_path, - require_physiological_data=True, - ) - - label = diagnosis_cache.get(patient_id) - - if label == 0 or label == 1: - return patient_id, feature_vector, label, None - - return patient_id, None, None, f"Invalid label for {patient_id}. Skipping..." - - except FileNotFoundError as e: - return patient_id, None, None, f"{e} Skipping..." - except Exception as e: - return patient_id, None, None, f"Error processing {patient_id}: {e}" - - def _close_run_model_pbar(): global RUN_MODEL_PBAR if RUN_MODEL_PBAR is not None: @@ -362,72 +79,10 @@ def train_model(data_folder, model_folder, verbose, csv_path=DEFAULT_CSV_PATH): if verbose: print('Finding the Challenge data...') - patient_data_file = os.path.join(data_folder, DEMOGRAPHICS_FILE) - patient_metadata_list = find_patients(patient_data_file) - demographics_cache, diagnosis_cache = build_training_metadata_cache(patient_data_file) - num_records = len(patient_metadata_list) - - if num_records == 0: - raise FileNotFoundError('No data were provided.') - - # Extract the features and labels from the data. - if verbose: - print('Extracting features and labels from the data...') - - features = list() - labels = list() - - with ThreadPoolExecutor(max_workers=MAX_TRAIN_WORKERS) as executor: - results = executor.map( - lambda record: process_training_record( - record, - data_folder, - demographics_cache, - diagnosis_cache, - csv_path - ), - patient_metadata_list - ) - - pbar = tqdm(results, total=num_records, desc="Extracting Features", unit="record", disable=not verbose) - for patient_id, feature_vector, label, message in pbar: - if verbose: - pbar.set_postfix({"patient": patient_id}) - - if message is not None: - tqdm.write(f" ! {message}") - continue - - features.append(feature_vector) - labels.append(label) - - pbar.close() - - features = np.asarray(features, dtype=np.float32) - labels = np.asarray(labels, dtype=np.int32) - - if features.size == 0 or features.ndim != 2 or features.shape[0] == 0: - raise ValueError('No valid training samples were extracted. Review feature extraction logs for the skipped records.') - - # Train the models on the features. if verbose: print('Training the model on the data...') - neg = int(np.sum(labels == 0)) - pos = int(np.sum(labels == 1)) - scale_pos_weight = (neg / pos) if pos > 0 else 1.0 - - model = XGBClassifier( - scale_pos_weight=scale_pos_weight, - n_estimators=300, - max_depth=4, - learning_rate=0.05, - subsample=0.8, - random_state=42, - eval_metric='auc', - tree_method='hist', - ) - model.fit(features, labels) + model = train_xgb_model(data_folder, verbose, csv_path) # Create a folder for the model if it does not already exist. os.makedirs(model_folder, exist_ok=True) @@ -506,36 +161,6 @@ def run_model(model, record, data_folder, verbose): return binary_output, probability_output -################################################################################ -# -# Optional functions. You can change or remove these functions and/or add new functions. -# -################################################################################ - -def extract_demographic_features(data): - """ - Extracts the demographic subset used by the current XGBoost model. - - Inputs: - data (dict): A dictionary containing patient metadata (e.g., from a CSV row). - - Returns: - np.array: A feature vector of length 4 with age and sex only. - """ - age = np.array([load_age(data)]) - - sex = load_sex(data) - sex_vec = np.zeros(3) - if sex == 'Female': - sex_vec[0] = 1 - elif sex == 'Male': - sex_vec[1] = 1 - else: - sex_vec[2] = 1 - - return np.concatenate([age, sex_vec]) - - # Save your trained model. def save_model(model_folder, model): d = {'model': model} From 1370123cdced4ad2d0fce31c176dbd54846370d5 Mon Sep 17 00:00:00 2001 From: dcajal Date: Mon, 30 Mar 2026 15:59:14 +0200 Subject: [PATCH 51/93] delete lolai_models --- src/lolai_models.py | 2214 ------------------------------------------- 1 file changed, 2214 deletions(-) delete mode 100644 src/lolai_models.py diff --git a/src/lolai_models.py b/src/lolai_models.py deleted file mode 100644 index c93eb98..0000000 --- a/src/lolai_models.py +++ /dev/null @@ -1,2214 +0,0 @@ -import torch -import torch.nn as nn -import torch.nn.functional as F -import numpy as np - -import torch.nn as nn -import torch -import torch.nn as nn -import torch.nn.functional as F -import numpy as np -import matplotlib.pyplot as plt -from captum.attr import ( - IntegratedGradients, - LayerGradCam, - LayerAttribution, - Occlusion, - GradientShap -) - -from typing import Dict, List, Tuple, Union, Optional - - -class CNN_LSTM_Classifier(nn.Module): - def __init__(self, input_channels=3, hidden_dim=64, num_classes=3, dropout=0.3): - super(CNN_LSTM_Classifier, self).__init__() - - self.cnn = nn.Sequential( - nn.Conv1d(input_channels, 6, kernel_size=5, padding=2), - nn.BatchNorm1d(6), - nn.ReLU(), - nn.MaxPool1d(kernel_size=2), - nn.Dropout(dropout), - - nn.Conv1d(6, 9, kernel_size=3, padding=1), - nn.BatchNorm1d(9), - nn.ReLU(), - nn.MaxPool1d(kernel_size=2), - nn.Dropout(dropout), - - - nn.Conv1d(9, 18, kernel_size=3, padding=1), - nn.BatchNorm1d(18), - nn.ReLU(), - nn.MaxPool1d(kernel_size=2), - nn.Dropout(dropout) - ) - - self.lstm = nn.LSTM( - input_size=18, - hidden_size=hidden_dim, - batch_first=True, - bidirectional=True # Use bidirectional LSTM for better context - ) - - self.classifier = nn.Sequential( - nn.Linear(2 * hidden_dim, 64), # Adjust for bidirectional LSTM - nn.ReLU(), - nn.Dropout(dropout), - nn.Linear(64, num_classes) - ) - - def forward(self, x): - # x: (batch, channels, time) - x = self.cnn(x) # (batch, features, time) - x = x.permute(0, 2, 1) # (batch, time, features) - _, (h_n, _) = self.lstm(x) # h_n: (num_layers * num_directions, batch, hidden_dim) - h_n = torch.cat((h_n[-2], h_n[-1]), dim=1) # Concatenate forward and backward states - out = self.classifier(h_n) # (batch, num_classes) - return out - -import torch -import torch.nn as nn -import torch.nn.functional as F -import matplotlib.pyplot as plt -import numpy as np - -import torch -import torch.nn as nn -import torch.nn.functional as F - -class CNN_LSTM_Classifier_XAI(nn.Module): - def __init__(self, input_channels=3, hidden_dim=32, num_classes=3, dropout=0.4): - super(CNN_LSTM_Classifier_XAI, self).__init__() - - self.cnn_activations = [] - self.lstm_activations = None - self.attention_weights = None - self.gradients = None - self.last_cnn_output = None - self.input = None - - # CNN - self.conv1 = nn.Conv1d(input_channels, 16, kernel_size=5, padding=2) - self.bn1 = nn.BatchNorm1d(16) - self.conv2 = nn.Conv1d(16, 32, kernel_size=3, padding=1) - self.bn2 = nn.BatchNorm1d(32) - self.conv3 = nn.Conv1d(32, 64, kernel_size=3, padding=1) - self.bn3 = nn.BatchNorm1d(64) - self.pool = nn.MaxPool1d(kernel_size=2) - self.dropout = nn.Dropout(dropout) - - # LSTM - self.lstm = nn.LSTM(input_size=64, hidden_size=hidden_dim, - batch_first=True, bidirectional=True) - - # Atención - self.attention = nn.Linear(2 * hidden_dim, 1) - - # Clasificador - self.classifier = nn.Sequential( - nn.Linear(2 * hidden_dim, 32), - nn.ReLU(), - nn.Dropout(dropout), - nn.Linear(32, num_classes) - ) - - def activations_hook(self, grad): - self.gradients = grad - - def forward(self, x, return_attention=False, track_gradients=False): - self.input = x - - x = self.conv1(x) - x = self.bn1(x) - x = F.relu(x) - self.cnn_activations.append(x.detach()) - x = self.pool(x) - x = self.dropout(x) - - x = self.conv2(x) - x = self.bn2(x) - x = F.relu(x) - self.cnn_activations.append(x.detach()) - x = self.pool(x) - x = self.dropout(x) - - x = self.conv3(x) - x = self.bn3(x) - x = F.relu(x) - cnn_output = x - - if track_gradients and cnn_output.requires_grad: - cnn_output.register_hook(self.activations_hook) - - self.last_cnn_output = cnn_output # necesario para Grad-CAM - self.cnn_activations.append(cnn_output.detach()) - - x = self.pool(cnn_output) - x = self.dropout(x) - - x = x.permute(0, 2, 1) # (batch, time, features) - lstm_out, _ = self.lstm(x) - self.lstm_activations = lstm_out.detach() - - attention_scores = self.attention(lstm_out).squeeze(-1) - attention_weights = F.softmax(attention_scores, dim=1) - self.attention_weights = attention_weights.detach() - - context_vector = torch.bmm(attention_weights.unsqueeze(1), lstm_out).squeeze(1) - out = self.classifier(context_vector) - - if return_attention: - return out, attention_weights - return out - - def reset_activation_storage(self): - self.cnn_activations = [] - self.lstm_activations = None - self.attention_weights = None - self.gradients = None - self.last_cnn_output = None - self.input = None - - def interpret(self, x, class_idx=None): - self.reset_activation_storage() - - was_training = self.training - lstm_was_training = self.lstm.training - - self.eval() - self.lstm.train() # necesario para CuDNN backward - - x.requires_grad_() - logits, attention = self.forward(x, return_attention=True, track_gradients=True) - pred = torch.softmax(logits, dim=1) - - if class_idx is None: - class_idx = pred.argmax(dim=1) - - for i in range(x.shape[0]): - pred[i, class_idx[i]].backward(retain_graph=True) - - self.train(was_training) - self.lstm.train(lstm_was_training) - - feature_importance = self.get_feature_importance() - temporal_channel_importance = self.get_temporal_channel_importance() - channel_imp=self.get_channel_importance() - - self.input = None # limpieza para evitar problemas de memoria - torch.cuda.empty_cache() - - return { - 'prediction': pred.detach(), - 'class_idx': class_idx, - 'attention_weights': self.attention_weights, - 'feature_importance': feature_importance, - 'cnn_activations': self.cnn_activations, - 'temporal_channel_importance': temporal_channel_importance, - 'channel_importance': channel_imp - } - - def get_feature_importance(self): - """ - Grad-CAM temporal sobre la salida del último bloque CNN. - Devuelve tensor (batch, time) - """ - if self.gradients is None or self.last_cnn_output is None: - return None - - pooled_gradients = torch.mean(self.gradients, dim=[0, 2]) # (channels,) - cam = self.last_cnn_output.clone() - - for i in range(cam.shape[1]): - cam[:, i, :] *= pooled_gradients[i] - - heatmap = torch.mean(cam, dim=1).detach() # (batch, time) - return heatmap - - def get_channel_importance(self): - """ - Importancia por canal: (batch, channels) - """ - if self.input.grad is None: - raise ValueError("Gradientes de la entrada no están disponibles. Llama primero a interpret().") - return self.input.grad.abs().mean(dim=2) - - def get_temporal_channel_importance(self): - """ - Importancia canal-temporal: (batch, channels, time) - """ - if self.input.grad is None: - raise ValueError("Gradients of the input are not available. Call interpret() first.") - return self.input.grad.abs().detach() - - - - - - -class ContrastiveVAE(nn.Module): - def __init__(self, in_channels=4, latent_dim=32, lstm_hidden=64, n_classes=3, use_classifier=False): - super().__init__() - self.use_classifier = use_classifier - - # Encoder - self.encoder = nn.Sequential( - nn.Conv1d(in_channels, 32, kernel_size=3, padding=1), - nn.ReLU(), - nn.Conv1d(32, 64, kernel_size=3, padding=1), - nn.ReLU() - ) - - self.global_pool = nn.AdaptiveAvgPool1d(1) # for VAE path - self.fc_mu = nn.Linear(64, latent_dim) - self.fc_logvar = nn.Linear(64, latent_dim) - - # Decoder (for reconstruction) - self.decoder_input = nn.Linear(latent_dim, 64) - self.decoder = nn.Sequential( - nn.ConvTranspose1d(64, 32, kernel_size=3, padding=1), - nn.ReLU(), - nn.ConvTranspose1d(32, in_channels, kernel_size=3, padding=1) - ) - - # LSTM + Classifier always initialized (but optionally used) - self.lstm = nn.LSTM(input_size=64, hidden_size=lstm_hidden, batch_first=True, bidirectional=True) - self.classifier = nn.Sequential( - nn.Linear(lstm_hidden * 2, 64), - nn.ReLU(), - nn.Linear(64, n_classes) - ) - - def encode(self, x): - h = self.encoder(x) # (B, 64, T) - pooled = self.global_pool(h).squeeze(-1) # (B, 64) - mu = self.fc_mu(pooled) - logvar = self.fc_logvar(pooled) - return mu, logvar, h # h is (B, 64, T) - - def reparameterize(self, mu, logvar): - std = torch.exp(0.5 * logvar) - eps = torch.randn_like(std) - return mu + eps * std - - def decode(self, z, length): - # Mejora la reconstrucción con una capa de proyección inicial - h = self.decoder_input(z).unsqueeze(-1) # (B, 64, 1) - # Usar interpolación para un escalado más suave en lugar de expand - h = F.interpolate(h, size=length, mode='linear', align_corners=False) - x_recon = self.decoder(h) - return x_recon - - def forward(self, x): - B, C, T = x.shape - mu, logvar, features = self.encode(x) # features: (B, 64, T) - z = self.reparameterize(mu, logvar) - x_recon = self.decode(z, T) - - logits = None - if self.use_classifier: - features_t = features.permute(0, 2, 1) # (B, T, 64) - lstm_out, _ = self.lstm(features_t) # (B, T, 2*hidden) - lstm_feat = lstm_out.mean(dim=1) # (B, 2*hidden) - logits = self.classifier(lstm_feat) # (B, n_classes) - - return x_recon, mu, logvar, z, logits - - def get_latents(self, x, use_mean=True): - mu, logvar, _ = self.encode(x) - return mu if use_mean else self.reparameterize(mu, logvar) - - def classify(self, x): - """Forward through the classifier only (requires use_classifier = True).""" - assert self.use_classifier, "Classifier is not enabled. Set model.use_classifier = True before calling classify." - _, _, features = self.encode(x) - features_t = features.permute(0, 2, 1) - lstm_out, _ = self.lstm(features_t) - lstm_feat = lstm_out.mean(dim=1) - return self.classifier(lstm_feat) - - -def vae_loss(recon_x, x, mu, logvar): - recon_loss = F.mse_loss(recon_x, x, reduction='mean') - kl_div = -0.5 * torch.mean(1 + logvar - mu.pow(2) - logvar.exp()) - return recon_loss + kl_div, recon_loss, kl_div - - -def contrastive_loss(z, ids, temperature=0.1): - z = F.normalize(z, dim=1) - sim = torch.mm(z, z.T) / temperature - labels = ids.view(-1, 1) - mask = torch.eq(labels, labels.T).float().to(z.device) - mask = mask - torch.eye(len(z), device=z.device) - exp_sim = torch.exp(sim) * (1 - torch.eye(len(z), device=z.device)) - log_prob = sim - torch.log(exp_sim.sum(1, keepdim=True) + 1e-8) - mean_log_prob_pos = (mask * log_prob).sum(1) / (mask.sum(1) + 1e-8) - return -mean_log_prob_pos.mean() - - -def intra_patient_loss(latents, patient_ids): - """Versión optimizada que evita bucles explícitos""" - # Convertir IDs a tensor si no lo son ya - if not isinstance(patient_ids, torch.Tensor): - patient_ids = torch.tensor(patient_ids, device=latents.device) - - # Crear matriz de similaridad de pacientes (1 donde son iguales) - patient_sim = (patient_ids.unsqueeze(1) == patient_ids.unsqueeze(0)).float() - # Quitar diagonal (mismo ejemplo) - mask = patient_sim - torch.eye(len(latents), device=latents.device) - # Calcular distancias entre latentes - latent_dists = torch.cdist(latents, latents, p=2) - # Aplicar máscara y promediar - valid_pairs = mask.sum() - if valid_pairs > 0: - return (mask * latent_dists).sum() / valid_pairs - return torch.tensor(0.0, device=latents.device) - - -def training_step(model, batch1, batch2, patient_ids, optimizer, alpha=0.1, beta=1.0): - model.train() - x1, x2 = batch1, batch2 - x1_recon, mu1, logvar1, z1, _ = model(x1) - x2_recon, mu2, logvar2, z2, _ = model(x2) - - recon1, r1, kl1 = vae_loss(x1_recon, x1, mu1, logvar1) - recon2, r2, kl2 = vae_loss(x2_recon, x2, mu2, logvar2) - vae_total = (recon1 + recon2) / 2 - - z_all = torch.cat([z1, z2], dim=0) - # Asegurar que IDs sean tensores - ids = torch.arange(len(z1), device=z1.device).repeat(2) - contrastive = contrastive_loss(z_all, ids) - - # Extender patient_ids correctamente - if isinstance(patient_ids, torch.Tensor): - p_ids = torch.cat([patient_ids, patient_ids], dim=0) - else: - p_ids = patient_ids + patient_ids # Si es una lista - - patient_reg = intra_patient_loss(z_all, p_ids) - - total = vae_total + alpha * contrastive + beta * patient_reg - - optimizer.zero_grad() - total.backward() - optimizer.step() - - return { - 'total_loss': total.item(), - 'recon_loss': vae_total.item(), - 'contrastive': contrastive.item(), - 'patient_reg': patient_reg.item(), - 'kl_loss': (kl1 + kl2).item() / 2 - } - - -def fine_tune_step(model, x, y, optimizer, criterion=nn.CrossEntropyLoss()): - model.train() - model.use_classifier = True - logits = model.classify(x) - loss = criterion(logits, y) - - optimizer.zero_grad() - loss.backward() - optimizer.step() - - preds = torch.argmax(logits, dim=1) - acc = (preds == y).float().mean().item() - - return { - 'classification_loss': loss.item(), - 'accuracy': acc - } - -class ImprovedPainClassifier(nn.Module): - def __init__(self, input_channels=3, hidden_dim=128, num_classes=3, dropout=0.4): - super(ImprovedPainClassifier, self).__init__() - - # Increased regularization and feature extraction for small datasets - self.cnn = nn.Sequential( - # Layer 1: More filters to capture diverse patterns - nn.Conv1d(input_channels, 64, kernel_size=3, padding=1), - nn.BatchNorm1d(64), - nn.LeakyReLU(0.1), # LeakyReLU helps with gradient flow - nn.MaxPool1d(kernel_size=2), - - # Layer 2: Increased complexity - nn.Conv1d(64, 128, kernel_size=3, padding=1), - nn.BatchNorm1d(128), - nn.LeakyReLU(0.1), - nn.MaxPool1d(kernel_size=2), - nn.Dropout(dropout), - - # Layer 3: Additional layer for better feature extraction - nn.Conv1d(128, 128, kernel_size=3, padding=1), - nn.BatchNorm1d(128), - nn.LeakyReLU(0.1), - nn.Dropout(dropout) - ) - - # Attention mechanism to focus on important temporal patterns - self.attention = nn.Sequential( - nn.Linear(256, 64), - nn.Tanh(), - nn.Linear(64, 1) - ) - - # Bidirectional LSTM with residual connections - self.lstm = nn.LSTM( - input_size=128, - hidden_size=hidden_dim, - num_layers=2, # Multiple layers for complex temporal patterns - batch_first=True, - bidirectional=True, - dropout=dropout # Apply dropout between LSTM layers - ) - - # Classifier with additional regularization - self.classifier = nn.Sequential( - nn.Linear(2 * hidden_dim, hidden_dim), - nn.BatchNorm1d(hidden_dim), # Normalize activations - nn.LeakyReLU(0.1), - nn.Dropout(dropout), - nn.Linear(hidden_dim, hidden_dim // 2), - nn.LeakyReLU(0.1), - nn.Dropout(dropout), - nn.Linear(hidden_dim // 2, num_classes) - ) - - def forward(self, x): - # x: (batch, channels, time) - BVP, EDA, and respiratory signals - - # Extract features with CNN - cnn_out = self.cnn(x) # (batch, 128, time') - - # Reshape for LSTM - cnn_out = cnn_out.permute(0, 2, 1) # (batch, time', 128) - - # Process with LSTM - lstm_out, (h_n, _) = self.lstm(cnn_out) # lstm_out: (batch, time', 2*hidden_dim) - - # Apply attention to focus on relevant parts of the signal - attn_weights = self.attention(lstm_out).softmax(dim=1) # (batch, time', 1) - context = torch.sum(attn_weights * lstm_out, dim=1) # (batch, 2*hidden_dim) - - # Alternative: Use concatenated hidden states from both directions - # h_n = torch.cat((h_n[-2], h_n[-1]), dim=1) # (batch, 2*hidden_dim) - - # Classify - out = self.classifier(context) # (batch, num_classes) - return out - - -import matplotlib.pyplot as plt -import numpy as np -import torch - -class ExplainabilityVisualizer: - def __init__(self, channel_names=None): - """ - channel_names: lista opcional con nombres de los canales de entrada - """ - self.channel_names = channel_names - - def plot_attention_weights(self, attention_weights, title="Atención temporal"): - attention = attention_weights.squeeze().cpu().numpy() - plt.figure(figsize=(10, 2)) - plt.plot(attention) - plt.title(title) - plt.xlabel("Timestep") - plt.ylabel("Weight") - plt.grid(True) - plt.tight_layout() - plt.show() - - def plot_gradcam_heatmap(self, heatmap, title="Grad-CAM temporal"): - heat = heatmap.squeeze().cpu().numpy() - plt.figure(figsize=(10, 2)) - plt.plot(heat) - plt.title(title) - plt.xlabel("Timestep") - plt.ylabel("Importance") - plt.grid(True) - plt.tight_layout() - plt.show() - - def plot_channel_importance(self, channel_importance, title="Importancia por canal"): - values = channel_importance.squeeze().cpu().numpy() - channels = self.channel_names if self.channel_names else [f"Channel {i}" for i in range(len(values))] - plt.figure(figsize=(6, 3)) - plt.bar(channels, values) - plt.title(title) - plt.ylabel("Importancia media") - plt.xticks(rotation=45) - plt.grid(axis='y') - plt.tight_layout() - plt.show() - - def plot_signals_with_attention_highlight(self, x, importance, threshold=0.85, title="Señal con zonas de atención"): - """ - Dibuja las señales multicanal y sombreado rojo donde la importancia temporal supera el umbral. - - Args: - x: Tensor (channels, time) - importance: Tensor (time,) - threshold: percentil (0-1) o valor absoluto - title: título del gráfico - """ - x = x.detach().cpu().numpy() - importance = importance.detach().cpu().numpy() - time = np.arange(x.shape[1]) - n_channels = x.shape[0] - - if threshold <= 1.0: - threshold_value = np.quantile(importance, threshold) - else: - threshold_value = threshold - - high_attention_mask = importance >= threshold_value - - fig, axs = plt.subplots(n_channels, 1, figsize=(12, 2.5 * n_channels), sharex=True) - if n_channels == 1: - axs = [axs] - - for i in range(n_channels): - axs[i].plot(time, x[i], label=self.channel_names[i] if self.channel_names else f"Canal {i}", color="black") - axs[i].set_ylabel("Valor") - axs[i].grid(True) - - in_high = False - start = 0 - for t in range(len(high_attention_mask)): - if high_attention_mask[t] and not in_high: - start = t - in_high = True - elif not high_attention_mask[t] and in_high: - axs[i].axvspan(start, t, color='red', alpha=0.25) - in_high = False - if in_high: - axs[i].axvspan(start, len(high_attention_mask), color='red', alpha=0.25) - - axs[i].legend(loc="upper right") - - axs[-1].set_xlabel("Tiempo (muestras)") - plt.suptitle(title) - plt.tight_layout() - plt.show() - - - -class FocalLoss(nn.Module): - """ - Focal Loss para clasificación binaria y multiclase. - - Parámetros: - - alpha: Factor de ponderación para manejar desequilibrio de clases. - Puede ser un escalar (mismo valor para todas las clases) o - un tensor (valores específicos por clase). - - gamma: Factor de modulación para enfocar en ejemplos difíciles (>= 0). - - reduction: 'none' | 'mean' | 'sum' - - eps: Pequeño valor para estabilidad numérica - - Referencias: - - Paper original: "Focal Loss for Dense Object Detection" por Lin et al. - """ - def __init__(self, alpha=0.25, gamma=2.0, reduction='mean', eps=1e-6): - super(FocalLoss, self).__init__() - self.alpha = alpha - self.gamma = gamma - self.reduction = reduction - self.eps = eps - - def forward(self, inputs, targets): - """ - Args: - inputs: Logits de forma [B, C] donde B es el tamaño del batch y C es el número de clases. - Para clasificación binaria, C puede ser 1. - targets: Etiquetas de objetivos de forma [B] para multiclase o [B, 1] para binaria. - Valores enteros para multiclase (clases indexadas desde 0 a C-1). - Valores continuos entre 0 y 1 para binaria. - """ - # Determinar si es clasificación binaria o multiclase - if inputs.shape[1] == 1 or inputs.shape[1] == 2: # Binaria - # Aplicar sigmoide para obtener probabilidades - probs = torch.sigmoid(inputs.view(-1)) - targets = targets.view(-1) - - # Calcular pt (probabilidad del objetivo correcto) - pt = probs * targets + (1 - probs) * (1 - targets) - - # Aplicar factores de ponderación - if isinstance(self.alpha, (float, int)): - alpha_t = self.alpha * targets + (1 - self.alpha) * (1 - targets) - else: - # Si alpha es un tensor, usar indexación - alpha_t = self.alpha if self.alpha is not None else torch.ones_like(pt) - - # Calcular la focal loss - focal_weight = (1 - pt).pow(self.gamma) - loss = -alpha_t * focal_weight * torch.log(pt.clamp(min=self.eps)) - - else: # Multiclase - # Convertir logits a distribución de probabilidad - log_softmax = F.log_softmax(inputs, dim=1) - - # Obtener log probabilidad para las clases objetivo - targets = targets.view(-1, 1) - log_pt = log_softmax.gather(1, targets).view(-1) - pt = log_pt.exp() # Obtener probabilidades - - # Aplicar factores de ponderación - if isinstance(self.alpha, (list, tuple, torch.Tensor)): - # Si alpha es específico por clase - alpha = torch.tensor(self.alpha, device=inputs.device) - alpha_t = alpha.gather(0, targets.view(-1)) - else: - alpha_t = self.alpha if self.alpha is not None else 1.0 - - # Calcular focal loss - focal_weight = (1 - pt).pow(self.gamma) - loss = -alpha_t * focal_weight * log_pt - - # Aplicar reduction - if self.reduction == 'mean': - return loss.mean() - elif self.reduction == 'sum': - return loss.sum() - else: # 'none' - return loss - - - - -class CNN_LSTM_Classifier_XAI_2(nn.Module): - def __init__(self, input_channels=3, hidden_dim=32, num_classes=3, dropout=0.1): - super(CNN_LSTM_Classifier_XAI_2, self).__init__() - - self.cnn_activations = [] - self.lstm_activations = None - self.attention_weights = None - self.gradients = None - self.last_cnn_output = None - self.input = None - self.input_channels = input_channels - - # CNN - self.conv1 = nn.Conv1d(input_channels, 16, kernel_size=5, padding=2) - self.bn1 = nn.BatchNorm1d(16) - self.conv2 = nn.Conv1d(16, 32, kernel_size=3, padding=1) - self.bn2 = nn.BatchNorm1d(32) - self.conv3 = nn.Conv1d(32, 64, kernel_size=3, padding=1) - self.bn3 = nn.BatchNorm1d(64) - self.pool = nn.MaxPool1d(kernel_size=2) - self.dropout = nn.Dropout(dropout) - - # LSTM - self.lstm = nn.LSTM(input_size=64, hidden_size=hidden_dim, - batch_first=True, bidirectional=True) - - # Attention - self.attention = nn.Linear(2 * hidden_dim, 1) - - # Classifier - self.classifier = nn.Sequential( - nn.Linear(2 * hidden_dim, 32), - nn.ReLU(), - nn.Dropout(dropout), - nn.Linear(32, num_classes) - ) - - def activations_hook(self, grad): - self.gradients = grad - - - - def forward(self, x, return_attention=False, track_gradients=False): - # Reset activation storage at the beginning of each forward pass - self.reset_activation_storage() - - self.input = x - - x = self.conv1(x) - x = self.bn1(x) - x = F.relu(x) - self.cnn_activations.append(x.detach()) - x = self.pool(x) - x = self.dropout(x) - - x = self.conv2(x) - x = self.bn2(x) - x = F.relu(x) - self.cnn_activations.append(x.detach()) - x = self.pool(x) - x = self.dropout(x) - - x = self.conv3(x) - x = self.bn3(x) - x = F.relu(x) - cnn_output = x - - if track_gradients and cnn_output.requires_grad: - cnn_output.register_hook(self.activations_hook) - - self.last_cnn_output = cnn_output # needed for Grad-CAM - self.cnn_activations.append(cnn_output.detach()) - - x = self.pool(cnn_output) - x = self.dropout(x) - - x = x.permute(0, 2, 1) # (batch, time, features) - lstm_out, (h_n, c_n) = self.lstm(x) - self.lstm_activations = lstm_out.detach() - - attention_scores = self.attention(lstm_out).squeeze(-1) - attention_weights = F.softmax(attention_scores, dim=1) - self.attention_weights = attention_weights.detach() - - context_vector = torch.bmm(attention_weights.unsqueeze(1), lstm_out).squeeze(1) - out = self.classifier(context_vector) - - if return_attention: - return out, attention_weights - return out - - def reset_activation_storage(self): - self.cnn_activations = [] - self.lstm_activations = None - self.attention_weights = None - self.gradients = None - self.last_cnn_output = None - - def interpret(self, x, class_idx=None, methods=None): - """ - Enhanced interpretation method with multiple explainability techniques - - Args: - x: Input data tensor - class_idx: Target class indices to explain (defaults to predicted class) - methods: List of methods to use, options: ['gradcam', 'integrated_gradients', - 'occlusion', 'shap', 'feature_ablation', 'all'] - - Returns: - Dictionary with various interpretability outputs - """ - if methods is None: - methods = ['gradcam', 'attention'] # Default methods - if 'all' in methods: - methods = ['gradcam', 'integrated_gradients', 'occlusion', 'shap', - 'feature_ablation', 'attention', 'layer_importance'] - - # Store original training state - was_training = self.training - lstm_was_training = self.lstm.training - - # Set model to evaluation mode for interpretability - self.eval() - self.lstm.train() # needed for CuDNN backward compatibility - - # Base prediction - x.requires_grad_() - self.input = x # Store input for interpretability methods - - logits, attention = self.forward(x, return_attention=True, track_gradients=True) - pred = torch.softmax(logits, dim=1) - - if class_idx is None: - class_idx = pred.argmax(dim=1) - - # Initialize results dictionary - results = { - 'prediction': pred.detach(), - 'class_idx': class_idx, - 'attention_weights': self.attention_weights, - } - - # Apply selected interpretability methods - if 'gradcam' in methods: - for i in range(x.shape[0]): - pred[i, class_idx[i]].backward(retain_graph=True if i < x.shape[0]-1 else False) - - results['feature_importance'] = self.get_feature_importance() - results['temporal_channel_importance'] = self.get_temporal_channel_importance() - results['channel_importance'] = self.get_channel_importance() - results['cnn_activations'] = self.cnn_activations - - # Integrated Gradients - if 'integrated_gradients' in methods: - ig = IntegratedGradients(self.forward_wrapper) - results['integrated_gradients'] = self._compute_integrated_gradients( - ig, x, class_idx) - - # Occlusion analysis - if 'occlusion' in methods: - occlusion = Occlusion(self.forward_wrapper) - results['occlusion'] = self._compute_occlusion(occlusion, x, class_idx) - - # SHAP (GradientSHAP implementation) - if 'shap' in methods: - gradient_shap = GradientShap(self.forward_wrapper) - results['gradient_shap'] = self._compute_gradient_shap(gradient_shap, x, class_idx) - - # Feature ablation (sensitivity analysis) - if 'feature_ablation' in methods: - results['feature_ablation'] = self._feature_ablation_analysis(x, class_idx) - - # Layer importance analysis - if 'layer_importance' in methods: - results['layer_importance'] = self._compute_layer_importance(x, class_idx) - - # Restore original training states - self.train(was_training) - self.lstm.train(lstm_was_training) - - # Clean up to avoid memory issues - self.input = None - torch.cuda.empty_cache() - - return results - - def forward_wrapper(self, x): - """Wrapper for Captum compatibility""" - return self.forward(x) - - def get_feature_importance(self): - """ - Grad-CAM temporal over the output of the last CNN block. - Returns tensor (batch, time) - """ - if self.gradients is None or self.last_cnn_output is None: - return None - - pooled_gradients = torch.mean(self.gradients, dim=[0, 2]) # (channels,) - cam = self.last_cnn_output.clone() - - for i in range(cam.shape[1]): - cam[:, i, :] *= pooled_gradients[i] - - heatmap = torch.mean(cam, dim=1).detach() # (batch, time) - - # Apply ReLU to highlight only positive influences - heatmap = F.relu(heatmap) - - # Normalize heatmap for better visualization - if heatmap.max() > 0: - heatmap = heatmap / heatmap.max() - - return heatmap - - def get_channel_importance(self): - """ - Channel importance: (batch, channels) - """ - if self.input is None or self.input.grad is None: - raise ValueError("Input gradients not available. Call interpret() first.") - return self.input.grad.abs().mean(dim=2).detach() - - def get_temporal_channel_importance(self): - """ - Temporal-channel importance: (batch, channels, time) - """ - if self.input is None or self.input.grad is None: - raise ValueError("Input gradients not available. Call interpret() first.") - return self.input.grad.abs().detach() - - def _compute_integrated_gradients(self, ig, x, class_idx): - """Compute integrated gradients attribution""" - batch_size = x.shape[0] - attributions = [] - - for i in range(batch_size): - baseline = torch.zeros_like(x[i:i+1]) - attr = ig.attribute( - x[i:i+1], baseline, target=class_idx[i].item(), n_steps=50 - ) - attributions.append(attr) - - return torch.cat(attributions).detach() - - def _compute_occlusion(self, occlusion_algo, x, class_idx): - """Compute occlusion-based feature attribution""" - batch_size = x.shape[0] - attributions = [] - - # Define sliding window parameters for temporal data - window_size = min(5, x.shape[2] // 4) # Adapt window size to input length - - for i in range(batch_size): - attr = occlusion_algo.attribute( - x[i:i+1], - sliding_window_shapes=(1, window_size), - target=class_idx[i].item(), - strides=(1, max(1, window_size // 2)) - ) - attributions.append(attr) - - return torch.cat(attributions).detach() - - def _compute_gradient_shap(self, shap_algo, x, class_idx): - """Compute GradientSHAP attributions""" - batch_size = x.shape[0] - attributions = [] - - for i in range(batch_size): - # Create random baselines (typically 10-50 for good estimates) - baselines = torch.randn(10, *x[i:i+1].shape[1:]) * 0.001 - - # Ensure baselines device matches input - baselines = baselines.to(x.device) - - attr = shap_algo.attribute( - x[i:i+1], baselines=baselines, target=class_idx[i].item() - ) - attributions.append(attr) - - return torch.cat(attributions).detach() - - def _feature_ablation_analysis(self, x, class_idx): - """Analyze model by systematically ablating input features""" - batch_size = x.shape[0] - results = [] - - for i in range(batch_size): - # Store original prediction - with torch.no_grad(): - orig_output = self.forward(x[i:i+1]) - orig_prob = torch.softmax(orig_output, dim=1)[0, class_idx[i]].item() - - # Test ablation of each channel - channel_importance = [] - for c in range(self.input_channels): - # Create ablated input (zero out one channel) - ablated_input = x[i:i+1].clone() - ablated_input[:, c, :] = 0 - - # Get prediction on ablated input - with torch.no_grad(): - ablated_output = self.forward(ablated_input) - ablated_prob = torch.softmax(ablated_output, dim=1)[0, class_idx[i]].item() - - # Impact is reduction in probability - channel_impact = orig_prob - ablated_prob - channel_importance.append(channel_impact) - - results.append(torch.tensor(channel_importance)) - - return torch.stack(results) - - def _compute_layer_importance(self, x, class_idx): - """Compute importance of each layer using Layer GradCAM""" - batch_size = x.shape[0] - layer_importance = {} - - # Define layers to analyze - layers = { - 'conv1': self.conv1, - 'conv2': self.conv2, - 'conv3': self.conv3 - } - - for layer_name, layer in layers.items(): - layer_gradcam = LayerGradCam(self.forward_wrapper, layer) - layer_attrs = [] - - for i in range(batch_size): - attr = layer_gradcam.attribute( - x[i:i+1], target=class_idx[i].item() - ) - # Process attribution to create a single importance score per sample - pooled_attr = torch.mean(attr, dim=1) - - layer_attrs.append(pooled_attr) - - layer_importance[layer_name] = torch.cat(layer_attrs).detach() - - return layer_importance - - def visualize_attributions(self, sample_idx, interpretations, time_axis=None, - channel_names=None, class_names=None): - """ - Visualize the various interpretation results - - Args: - sample_idx: Index of the sample to visualize - interpretations: Dictionary returned by interpret() method - time_axis: Optional array/list with time points for x-axis - channel_names: Optional list of channel names - class_names: Optional list of class names - """ - if not channel_names: - channel_names = [f'Channel {i}' for i in range(self.input_channels)] - - if not class_names: - class_idx = interpretations['class_idx'][sample_idx].item() - class_name = f'Class {class_idx}' - else: - class_idx = interpretations['class_idx'][sample_idx].item() - class_name = class_names[class_idx] - - # Set up figure - plt.figure(figsize=(15, 12)) - - # Original input visualization (top row, first column) - plt.subplot(3, 3, 1) - if self.input is not None: - input_data = self.input[sample_idx].cpu().detach().numpy() - if time_axis is not None: - for i in range(input_data.shape[0]): - plt.plot(time_axis, input_data[i], label=channel_names[i]) - else: - for i in range(input_data.shape[0]): - plt.plot(input_data[i], label=channel_names[i]) - plt.legend(loc='best') - plt.title('Input Signal') - plt.xlabel('Time') - plt.ylabel('Value') - - # GradCAM feature importance (top row, second column) - if 'feature_importance' in interpretations and interpretations['feature_importance'] is not None: - plt.subplot(3, 3, 2) - heatmap = interpretations['feature_importance'][sample_idx].cpu().numpy() - if time_axis is not None: - plt.plot(time_axis, heatmap) - else: - plt.plot(heatmap) - plt.title('GradCAM Feature Importance') - plt.xlabel('Time') - plt.ylabel('Importance') - - # Attention weights (top row, third column) - if 'attention_weights' in interpretations and interpretations['attention_weights'] is not None: - plt.subplot(3, 3, 3) - attention = interpretations['attention_weights'][sample_idx].cpu().numpy() - - if time_axis is not None: - # Need to match attention time axis to input time axis - # (account for pooling in the network) - x_points = np.linspace(time_axis[0], time_axis[-1], len(attention)) - plt.plot(x_points, attention) - else: - plt.plot(attention) - plt.title('Attention Weights') - plt.xlabel('Time') - plt.ylabel('Attention') - - # Channel importance (middle row, first column) - if 'channel_importance' in interpretations and interpretations['channel_importance'] is not None: - plt.subplot(3, 3, 4) - ch_importance = interpretations['channel_importance'][sample_idx].cpu().numpy() - plt.bar(channel_names, ch_importance) - plt.title('Channel Importance') - plt.ylabel('Importance') - plt.xticks(rotation=45) - - # Integrated Gradients (middle row, second column) - if 'integrated_gradients' in interpretations: - plt.subplot(3, 3, 5) - ig_attr = interpretations['integrated_gradients'][sample_idx].cpu().numpy() - ig_attr_mean = np.mean(ig_attr, axis=0) # Average across channels for visualization - - if time_axis is not None: - plt.plot(time_axis, ig_attr_mean) - else: - plt.plot(ig_attr_mean) - plt.title('Integrated Gradients') - plt.xlabel('Time') - plt.ylabel('Attribution') - - # Feature Ablation (middle row, third column) - if 'feature_ablation' in interpretations: - plt.subplot(3, 3, 6) - ablation_scores = interpretations['feature_ablation'][sample_idx].cpu().numpy() - plt.bar(channel_names, ablation_scores) - plt.title('Feature Ablation Impact') - plt.ylabel('Probability Change') - plt.xticks(rotation=45) - - # SHAP values (bottom row, first column) - if 'gradient_shap' in interpretations: - plt.subplot(3, 3, 7) - shap_attr = interpretations['gradient_shap'][sample_idx].cpu().numpy() - # Visualize average SHAP value over time - shap_avg = np.mean(shap_attr, axis=0) - - if time_axis is not None: - plt.plot(time_axis, shap_avg) - else: - plt.plot(shap_avg) - plt.title('GradientSHAP Values') - plt.xlabel('Time') - plt.ylabel('SHAP Value') - - # Occlusion analysis (bottom row, second column) - if 'occlusion' in interpretations: - plt.subplot(3, 3, 8) - occlusion_attr = interpretations['occlusion'][sample_idx].cpu().numpy() - occlusion_avg = np.mean(occlusion_attr, axis=0) - - if time_axis is not None: - plt.plot(time_axis, occlusion_avg) - else: - plt.plot(occlusion_avg) - plt.title('Occlusion Analysis') - plt.xlabel('Time') - plt.ylabel('Attribution') - - # Prediction summary (bottom row, third column) - plt.subplot(3, 3, 9) - pred_probs = interpretations['prediction'][sample_idx].cpu().numpy() - classes = list(range(len(pred_probs))) - if class_names: - classes = class_names - plt.bar(classes, pred_probs) - plt.title(f'Prediction: {class_name}') - plt.ylabel('Probability') - plt.ylim([0, 1]) - - plt.tight_layout() - return plt.gcf() - - def generate_interpretation_report(self, input_data, class_idx=None, - channel_names=None, class_names=None, - time_axis=None, methods='all'): - """ - Generate a comprehensive interpretation report for the given input - - Args: - input_data: Input tensor to analyze - class_idx: Target class indices (optional) - channel_names: Names of input channels (optional) - class_names: Names of output classes (optional) - time_axis: Time points for x-axis (optional) - methods: Explainability methods to use - - Returns: - Dictionary containing interpretations and visualization figure - """ - # Run all interpretation methods - interpretations = self.interpret(input_data, class_idx, methods=methods) - - # Generate visualizations for each sample - figures = [] - for i in range(input_data.shape[0]): - fig = self.visualize_attributions( - i, interpretations, - time_axis=time_axis, - channel_names=channel_names, - class_names=class_names - ) - figures.append(fig) - plt.close(fig) # Close to avoid display in notebooks - - return { - 'interpretations': interpretations, - 'figures': figures - } - - - -class CNN_LSTM_Classifier_XAI_2(nn.Module): - def __init__( - self, - input_channels=3, - num_classes=3, - cnn_channels=(16, 32, 64), - kernel_sizes=(5, 3, 3), - pool_type="max", # or 'avg' - dropout=0.1, - lstm_hidden_dim=32, - lstm_num_layers=1, - bidirectional=True, - classifier_hidden_dim=32, - attention_dim=None, # None = default: 2 * lstm_hidden_dim - ): - super(CNN_LSTM_Classifier_XAI_2, self).__init__() - - self.input_channels = input_channels - self.pool_type = pool_type - self.dropout_rate = dropout - self.bidirectional = bidirectional - self.num_directions = 2 if bidirectional else 1 - - self.cnn_activations = [] - self.lstm_activations = None - self.attention_weights = None - self.gradients = None - self.last_cnn_output = None - self.input = None - - # CNN Blocks - self.conv1 = nn.Conv1d(input_channels, cnn_channels[0], kernel_size=kernel_sizes[0], padding=kernel_sizes[0] // 2) - self.bn1 = nn.BatchNorm1d(cnn_channels[0]) - - self.conv2 = nn.Conv1d(cnn_channels[0], cnn_channels[1], kernel_size=kernel_sizes[1], padding=kernel_sizes[1] // 2) - self.bn2 = nn.BatchNorm1d(cnn_channels[1]) - - self.conv3 = nn.Conv1d(cnn_channels[1], cnn_channels[2], kernel_size=kernel_sizes[2], padding=kernel_sizes[2] // 2) - self.bn3 = nn.BatchNorm1d(cnn_channels[2]) - - self.pool = nn.MaxPool1d(kernel_size=2) if pool_type == "max" else nn.AvgPool1d(kernel_size=2) - self.dropout = nn.Dropout(dropout) - - # LSTM - self.lstm = nn.LSTM( - input_size=cnn_channels[2], - hidden_size=lstm_hidden_dim, - num_layers=lstm_num_layers, - batch_first=True, - bidirectional=bidirectional - ) - - # Attention - attention_dim = attention_dim or self.num_directions * lstm_hidden_dim - self.attention = nn.Linear(self.num_directions * lstm_hidden_dim, 1) - - # Classifier - self.classifier = nn.Sequential( - nn.Linear(self.num_directions * lstm_hidden_dim, classifier_hidden_dim), - nn.ReLU(), - nn.Dropout(dropout), - nn.Linear(classifier_hidden_dim, num_classes) - ) - - def activations_hook(self, grad): - self.gradients = grad - - def reset_activation_storage(self): - self.cnn_activations = [] - self.lstm_activations = None - self.attention_weights = None - self.gradients = None - self.last_cnn_output = None - - def forward(self, x, return_attention=False, track_gradients=False): - self.reset_activation_storage() - self.input = x - - x = self.pool(F.relu(self.bn1(self.conv1(x)))) - self.cnn_activations.append(x.detach()) - x = self.dropout(x) - - x = self.pool(F.relu(self.bn2(self.conv2(x)))) - self.cnn_activations.append(x.detach()) - x = self.dropout(x) - - x = F.relu(self.bn3(self.conv3(x))) - cnn_output = x - - if track_gradients and cnn_output.requires_grad: - cnn_output.register_hook(self.activations_hook) - - self.last_cnn_output = cnn_output - self.cnn_activations.append(cnn_output.detach()) - - x = self.pool(cnn_output) - x = self.dropout(x) - - x = x.permute(0, 2, 1) # (batch, time, features) - lstm_out, _ = self.lstm(x) - self.lstm_activations = lstm_out.detach() - - attention_scores = self.attention(lstm_out).squeeze(-1) - attention_weights = F.softmax(attention_scores, dim=1) - self.attention_weights = attention_weights.detach() - - context_vector = torch.bmm(attention_weights.unsqueeze(1), lstm_out).squeeze(1) - out = self.classifier(context_vector) - - if return_attention: - return out, attention_weights - return out - - -class CNN_LSTM_Classifier_Tunable(nn.Module): - def __init__( - self, - config: Optional[Dict] = None, - input_channels: int = 3, - seq_length: int = None, - num_classes: int = 3, - cnn_channels: Tuple[int, ...] = (16, 32, 64), - kernel_sizes: Tuple[int, ...] = (5, 3, 3), - pool_type: str = "max", # or 'avg' - pool_sizes: Tuple[int, ...] = (2, 2, 2), - use_batch_norm: bool = True, - activation: str = "relu", # "relu", "leaky_relu", "elu", "gelu" - dropout: float = 0.1, - cnn_dropout: Optional[float] = None, # Separate dropout for CNN - lstm_hidden_dim: int = 32, - lstm_num_layers: int = 1, - bidirectional: bool = True, - lstm_dropout: Optional[float] = None, # Separate dropout for LSTM - classifier_hidden_dims: List[int] = [32], # Multiple hidden layers - attention_dim: Optional[int] = None, # None = default: 2 * lstm_hidden_dim - attention_type: str = "basic", # "basic", "scaled_dot", "multi_head" - multi_head_num: int = 4, # For multi-head attention - residual_connections: bool = False, - layer_normalization: bool = False, - weight_init: str = "default", # "default", "xavier", "kaiming" - ): - """ - Enhanced CNN-LSTM model with attention mechanism designed for tuning flexibility. - - Args: - config: Optional dictionary with all hyperparameters to override other arguments - input_channels: Number of input channels - seq_length: Length of input sequence (needed for some operations) - num_classes: Number of output classes - cnn_channels: Tuple of CNN output channels for each layer - kernel_sizes: Tuple of kernel sizes for each CNN layer - pool_type: Pooling type ("max" or "avg") - pool_sizes: Pooling sizes for each layer - use_batch_norm: Whether to use batch normalization - activation: Activation function type - dropout: Default dropout rate - cnn_dropout: CNN-specific dropout (if None, uses dropout) - lstm_hidden_dim: LSTM hidden dimension - lstm_num_layers: Number of LSTM layers - bidirectional: Whether LSTM is bidirectional - lstm_dropout: LSTM-specific dropout (if None, uses dropout) - classifier_hidden_dims: List of hidden dimensions for classifier - attention_dim: Attention dimension - attention_type: Type of attention mechanism - multi_head_num: Number of heads for multi-head attention - residual_connections: Whether to use residual connections - layer_normalization: Whether to use layer normalization - weight_init: Weight initialization strategy - """ - super(CNN_LSTM_Classifier_Tunable, self).__init__() - - # Override with config if provided - if config is not None: - # Set all attributes from config - for key, value in config.items(): - if hasattr(self, key): - setattr(self, key, value) - elif key in locals(): - locals()[key] = value - - # Store parameters - self.input_channels = input_channels - self.seq_length = seq_length - self.num_classes = num_classes - self.cnn_channels = cnn_channels - self.kernel_sizes = kernel_sizes - self.pool_type = pool_type - self.pool_sizes = pool_sizes - self.use_batch_norm = use_batch_norm - self.activation_type = activation - self.dropout_rate = dropout - self.cnn_dropout_rate = cnn_dropout if cnn_dropout is not None else dropout - self.lstm_hidden_dim = lstm_hidden_dim - self.lstm_num_layers = lstm_num_layers - self.bidirectional = bidirectional - self.lstm_dropout_rate = lstm_dropout if lstm_dropout is not None else dropout - self.classifier_hidden_dims = classifier_hidden_dims - self.residual_connections = residual_connections - self.layer_normalization = layer_normalization - self.weight_init = weight_init - self.attention_type = attention_type - self.multi_head_num = multi_head_num - - # Calculate directions - self.num_directions = 2 if bidirectional else 1 - - # Default attention dimension if not provided - self.attention_dim = attention_dim or self.num_directions * lstm_hidden_dim - - # Precalcular dimensiones de secuencia después de las capas CNN - self.input_seq_length = seq_length - self.output_seq_length = None - - if seq_length is not None: - # Calcular reducción de secuencia por pooling - seq_reduction = 1 - current_length = seq_length - - for pool_size in self.pool_sizes: - current_length = (current_length + pool_size - 1) // pool_size # Ceil division - seq_reduction *= pool_size - - self.output_seq_length = current_length - - # Verificar dimensiones válidas - if self.output_seq_length <= 0: - raise ValueError(f"La secuencia resultante después del pooling es demasiado corta. " - f"Secuencia entrada: {seq_length}, reducción: {seq_reduction}") - - # For visualization and explanation - self.cnn_activations = [] - self.lstm_activations = None - self.attention_weights = None - self.gradients = None - self.last_cnn_output = None - self.input = None - - # Create activation function - self.activation = self._get_activation() - - # Create CNN layers - self.cnn_blocks = nn.ModuleList() - in_channels = input_channels - - for i, (out_channels, kernel_size, pool_size) in enumerate(zip(cnn_channels, kernel_sizes, pool_sizes)): - block = nn.ModuleDict() - block["conv"] = nn.Conv1d(in_channels, out_channels, kernel_size=kernel_size, padding=kernel_size // 2) - - if use_batch_norm: - block["bn"] = nn.BatchNorm1d(out_channels) - - if layer_normalization: - # Usamos LayerNorm correctamente para normalizar sobre la dimensión de características - block["ln"] = nn.LayerNorm([out_channels]) - - if pool_type == "max": - block["pool"] = nn.MaxPool1d(kernel_size=pool_size) - else: - block["pool"] = nn.AvgPool1d(kernel_size=pool_size) - - block["dropout"] = nn.Dropout(self.cnn_dropout_rate) - - # Determinar si este bloque puede usar conexión residual - # Solo si tienen mismas dimensiones de entrada y salida - if residual_connections and in_channels == out_channels: - block["has_residual"] = True - else: - block["has_residual"] = False - - self.cnn_blocks.append(block) - in_channels = out_channels - - # LSTM layer - self.lstm = nn.LSTM( - input_size=cnn_channels[-1], - hidden_size=lstm_hidden_dim, - num_layers=lstm_num_layers, - batch_first=True, - bidirectional=bidirectional, - dropout=self.lstm_dropout_rate if lstm_num_layers > 1 else 0 - ) - - # Attention mechanism - lstm_output_dim = self.num_directions * lstm_hidden_dim - if attention_type == "basic": - self.attention = nn.Linear(lstm_output_dim, 1) - elif attention_type == "scaled_dot": - self.query = nn.Linear(lstm_output_dim, self.attention_dim) - self.key = nn.Linear(lstm_output_dim, self.attention_dim) - self.value = nn.Linear(lstm_output_dim, lstm_output_dim) - elif attention_type == "multi_head": - self.mha = nn.MultiheadAttention( - embed_dim=lstm_output_dim, - num_heads=multi_head_num, - batch_first=True - ) - self.attention_ln = nn.LayerNorm(lstm_output_dim) - else: - # Caso por defecto para evitar errores - self.attention = nn.Linear(lstm_output_dim, 1) - print(f"ADVERTENCIA: Tipo de atención '{attention_type}' no reconocido. Usando 'basic'.") - - # Classifier - classifier_layers = [] - in_dim = lstm_output_dim - - for hidden_dim in classifier_hidden_dims: - classifier_layers.append(nn.Linear(in_dim, hidden_dim)) - classifier_layers.append(self.activation) - classifier_layers.append(nn.Dropout(dropout)) - in_dim = hidden_dim - - classifier_layers.append(nn.Linear(in_dim, num_classes)) - self.classifier = nn.Sequential(*classifier_layers) - - # Initialize weights - self._initialize_weights() - - - def _get_activation(self): - if self.activation_type == "relu": - return nn.ReLU() - elif self.activation_type == "leaky_relu": - return nn.LeakyReLU(0.1) - elif self.activation_type == "elu": - return nn.ELU() - elif self.activation_type == "gelu": - return nn.GELU() - else: - return nn.ReLU() - - def _initialize_weights(self): - if self.weight_init == "xavier": - for m in self.modules(): - if isinstance(m, nn.Conv1d) or isinstance(m, nn.Linear): - nn.init.xavier_uniform_(m.weight) - if m.bias is not None: - nn.init.zeros_(m.bias) - elif self.weight_init == "kaiming": - for m in self.modules(): - if isinstance(m, nn.Conv1d) or isinstance(m, nn.Linear): - nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu') - if m.bias is not None: - nn.init.zeros_(m.bias) - - def activations_hook(self, grad): - self.gradients = grad - - def reset_activation_storage(self): - self.cnn_activations = [] - self.lstm_activations = None - self.attention_weights = None - self.gradients = None - self.last_cnn_output = None - - def forward(self, x, return_attention=False, track_gradients=False): - """ - Forward pass del modelo - - Args: - x: Input tensor de forma (batch, channels, seq_length) - return_attention: Si True, devuelve también weights de atención - track_gradients: Si True, registra hook para gradientes (para explicabilidad) - - Returns: - Logits de clasificación y opcionalmente pesos de atención - """ - self.reset_activation_storage() - self.input = x - - # Guardar dimensiones originales para debugging - batch_size, channels, orig_seq_len = x.shape - - # Verificar entrada coherente con la configuración del modelo - if channels != self.input_channels: - print(f"ADVERTENCIA: Número de canales de entrada ({channels}) " - f"difiere del configurado ({self.input_channels})") - - # Almacenar dimensiones después de cada bloque para debugging - dims_after_each_block = [] - - # Process through CNN blocks - for i, block in enumerate(self.cnn_blocks): - # Guardar entrada para posible conexión residual - residual = x if block["has_residual"] else None - - # Forward pass por operaciones del bloque - x = block["conv"](x) - - if "bn" in block: - x = block["bn"](x) - - if "ln" in block: - # Transponemos correctamente para aplicar layer norm - x_transposed = x.transpose(1, 2) # (batch, seq, channels) - x_normalized = block["ln"](x_transposed) - x = x_normalized.transpose(1, 2) # Volver a (batch, channels, seq) - - x = self.activation(x) - - # Aplicar conexión residual si está disponible para este bloque - if residual is not None: - x = x + residual - - # Aplicar pooling (con la lógica corregida) - if not (i == len(self.cnn_blocks) - 1 and self.residual_connections): - x = block["pool"](x) - - x = block["dropout"](x) - self.cnn_activations.append(x.detach()) - - # Guardar dimensiones actuales - dims_after_each_block.append(tuple(x.shape)) - - cnn_output = x - - # Verificar dimensiones finales CNN - final_seq_len = x.shape[2] - if self.output_seq_length is not None and final_seq_len != self.output_seq_length: - print(f"ADVERTENCIA: Longitud secuencia después de CNN ({final_seq_len}) " - f"difiere de la esperada ({self.output_seq_length})") - - if track_gradients and cnn_output.requires_grad: - cnn_output.register_hook(self.activations_hook) - - self.last_cnn_output = cnn_output - - # Reshape for LSTM: (batch, channels, seq) -> (batch, seq, channels) - x = cnn_output.permute(0, 2, 1) - - # LSTM - lstm_out, _ = self.lstm(x) - self.lstm_activations = lstm_out.detach() - - # Declarar vectores que usaremos para todos los tipos de atención - attention_weights = None - context_vector = None - - # Apply attention mechanism según tipo configurado - if self.attention_type == "basic": - attention_scores = self.attention(lstm_out).squeeze(-1) - attention_weights = F.softmax(attention_scores, dim=1) - context_vector = torch.bmm(attention_weights.unsqueeze(1), lstm_out).squeeze(1) - - elif self.attention_type == "scaled_dot": - Q = self.query(lstm_out) - K = self.key(lstm_out) - V = self.value(lstm_out) - - scores = torch.bmm(Q, K.transpose(1, 2)) / np.sqrt(self.attention_dim) - attention_weights = F.softmax(scores, dim=-1) - context_vector = torch.bmm(attention_weights, V).mean(dim=1) - - elif self.attention_type == "multi_head": - # MultiheadAttention devuelve (attn_output, attn_output_weights) - attn_output, attn_output_weights = self.mha(lstm_out, lstm_out, lstm_out) - attention_weights = attn_output_weights - - if self.layer_normalization: - attn_output = self.attention_ln(attn_output + lstm_out) - - context_vector = attn_output.mean(dim=1) - else: - # Caso por defecto: atención uniforme - attention_weights = torch.ones(lstm_out.shape[0], lstm_out.shape[1]).to(lstm_out.device) - attention_weights = attention_weights / lstm_out.shape[1] # Normalizar - context_vector = lstm_out.mean(dim=1) - - # Verificar que attention_weights existe - if attention_weights is None: - attention_weights = torch.ones(lstm_out.shape[0], lstm_out.shape[1]).to(lstm_out.device) - attention_weights = attention_weights / lstm_out.shape[1] # Normalizar - - # Guardar pesos de atención para visualización/interpretación - self.attention_weights = attention_weights.detach() - - # Verificar dimensiones del vector de contexto - if context_vector is None: - context_vector = lstm_out.mean(dim=1) - - # Asegurar dimensionalidad correcta: (batch_size, features) - if len(context_vector.shape) > 2: - print(f"ADVERTENCIA: Vector contexto tiene forma inesperada {context_vector.shape}. " - f"Aplicando mean en dim 1.") - context_vector = context_vector.mean(dim=1) - elif len(context_vector.shape) == 1: - context_vector = context_vector.unsqueeze(0) - - # Verificación final - if len(context_vector.shape) != 2: - print(f"ERROR: Vector contexto debe ser 2D pero es {context_vector.shape}") - # Intentar corregir - if len(context_vector.shape) > 2: - context_vector = context_vector.reshape(batch_size, -1) - - # Classification - out = self.classifier(context_vector) - - if return_attention: - return out, attention_weights - return out - - def get_config(self): - """Returns the current configuration as a dictionary""" - return { - "input_channels": self.input_channels, - "seq_length": self.seq_length, - "num_classes": self.num_classes, - "cnn_channels": self.cnn_channels, - "kernel_sizes": self.kernel_sizes, - "pool_type": self.pool_type, - "pool_sizes": self.pool_sizes, - "use_batch_norm": self.use_batch_norm, - "activation": self.activation_type, - "dropout": self.dropout_rate, - "cnn_dropout": self.cnn_dropout_rate, - "lstm_hidden_dim": self.lstm_hidden_dim, - "lstm_num_layers": self.lstm_num_layers, - "bidirectional": self.bidirectional, - "lstm_dropout": self.lstm_dropout_rate, - "classifier_hidden_dims": self.classifier_hidden_dims, - "attention_dim": self.attention_dim, - "attention_type": self.attention_type, - "multi_head_num": self.multi_head_num, - "residual_connections": self.residual_connections, - "layer_normalization": self.layer_normalization, - "weight_init": self.weight_init - } - - def count_parameters(self): - """Count and return the number of trainable parameters""" - return sum(p.numel() for p in self.parameters() if p.requires_grad) - - def get_intermediate_outputs(self, x): - """Get all intermediate activations for a given input""" - _ = self.forward(x, track_gradients=True) - return { - "cnn_activations": self.cnn_activations, - "lstm_activations": self.lstm_activations, - "attention_weights": self.attention_weights - } - - def visualize_attention(self, x, return_fig=False): - """Visualize attention weights for a given input""" - try: - import matplotlib.pyplot as plt - - _, attention_weights = self.forward(x, return_attention=True) - - if attention_weights is None: - print("No attention weights available") - return None - - batch_size = attention_weights.size(0) - seq_len = attention_weights.size(1) - - fig, axes = plt.subplots(batch_size, 1, figsize=(10, 2*batch_size)) - if batch_size == 1: - axes = [axes] - - for i, ax in enumerate(axes): - weights = attention_weights[i].cpu().detach().numpy() - ax.bar(range(seq_len), weights) - ax.set_title(f"Sample {i+1}") - ax.set_xlabel("Sequence position") - ax.set_ylabel("Attention weight") - - plt.tight_layout() - - if return_fig: - return fig - plt.show() - return None - except ImportError: - print("matplotlib is required for visualization") - return None - - def interpret(self, x, class_idx=None, methods=None): - """ - Enhanced interpretation method with multiple explainability techniques - - Args: - x: Input data tensor - class_idx: Target class indices to explain (defaults to predicted class) - methods: List of methods to use, options: ['gradcam', 'integrated_gradients', - 'occlusion', 'shap', 'feature_ablation', 'all'] - - Returns: - Dictionary with various interpretability outputs - """ - try: - # Try to import Captum components - from captum.attr import IntegratedGradients, Occlusion, GradientShap, LayerGradCam - except ImportError: - raise ImportError("This method requires the 'captum' package. Install with: pip install captum") - - if methods is None: - methods = ['gradcam', 'attention'] # Default methods - if 'all' in methods: - methods = ['gradcam', 'integrated_gradients', 'occlusion', 'shap', - 'feature_ablation', 'attention', 'layer_importance'] - - # Store original training state - was_training = self.training - lstm_was_training = self.lstm.training - - # Set model to evaluation mode for interpretability - self.eval() - self.lstm.train() # needed for CuDNN backward compatibility - - # Base prediction - x.requires_grad_() - self.input = x # Store input for interpretability methods - - logits, attention = self.forward(x, return_attention=True, track_gradients=True) - pred = torch.softmax(logits, dim=1) - - if class_idx is None: - class_idx = pred.argmax(dim=1) - - # Initialize results dictionary - results = { - 'prediction': pred.detach(), - 'class_idx': class_idx, - 'attention_weights': self.attention_weights, - } - - # Apply selected interpretability methods - if 'gradcam' in methods: - for i in range(x.shape[0]): - pred[i, class_idx[i]].backward(retain_graph=True if i < x.shape[0]-1 else False) - - results['feature_importance'] = self.get_feature_importance() - results['temporal_channel_importance'] = self.get_temporal_channel_importance() - results['channel_importance'] = self.get_channel_importance() - results['cnn_activations'] = self.cnn_activations - - # Integrated Gradients - if 'integrated_gradients' in methods: - ig = IntegratedGradients(self.forward_wrapper) - results['integrated_gradients'] = self._compute_integrated_gradients( - ig, x, class_idx) - - # Occlusion analysis - if 'occlusion' in methods: - occlusion = Occlusion(self.forward_wrapper) - results['occlusion'] = self._compute_occlusion(occlusion, x, class_idx) - - # SHAP (GradientSHAP implementation) - if 'shap' in methods: - gradient_shap = GradientShap(self.forward_wrapper) - results['gradient_shap'] = self._compute_gradient_shap(gradient_shap, x, class_idx) - - # Feature ablation (sensitivity analysis) - if 'feature_ablation' in methods: - results['feature_ablation'] = self._feature_ablation_analysis(x, class_idx) - - # Layer importance analysis - if 'layer_importance' in methods: - results['layer_importance'] = self._compute_layer_importance(x, class_idx) - - # Restore original training states - self.train(was_training) - self.lstm.train(lstm_was_training) - - # Clean up to avoid memory issues - self.input = None - torch.cuda.empty_cache() - - return results - - def forward_wrapper(self, x): - """Wrapper for Captum compatibility""" - return self.forward(x) - - def get_feature_importance(self): - """ - Grad-CAM temporal over the output of the last CNN block. - Returns tensor (batch, time) - """ - if self.gradients is None or self.last_cnn_output is None: - return None - - pooled_gradients = torch.mean(self.gradients, dim=[0, 2]) # (channels,) - cam = self.last_cnn_output.clone() - - for i in range(cam.shape[1]): - cam[:, i, :] *= pooled_gradients[i] - - heatmap = torch.mean(cam, dim=1).detach() # (batch, time) - - # Apply ReLU to highlight only positive influences - heatmap = F.relu(heatmap) - - # Normalize heatmap for better visualization - if heatmap.max() > 0: - heatmap = heatmap / heatmap.max() - - return heatmap - - def get_channel_importance(self): - """ - Channel importance: (batch, channels) - """ - if self.input is None or self.input.grad is None: - raise ValueError("Input gradients not available. Call interpret() first.") - return self.input.grad.abs().mean(dim=2).detach() - - def get_temporal_channel_importance(self): - """ - Temporal-channel importance: (batch, channels, time) - """ - if self.input is None or self.input.grad is None: - raise ValueError("Input gradients not available. Call interpret() first.") - return self.input.grad.abs().detach() - - def _compute_integrated_gradients(self, ig, x, class_idx): - """Compute integrated gradients attribution""" - batch_size = x.shape[0] - attributions = [] - - for i in range(batch_size): - baseline = torch.zeros_like(x[i:i+1]) - attr = ig.attribute( - x[i:i+1], baseline, target=class_idx[i].item(), n_steps=50 - ) - attributions.append(attr) - - return torch.cat(attributions).detach() - - def _compute_occlusion(self, occlusion_algo, x, class_idx): - """Compute occlusion-based feature attribution""" - batch_size = x.shape[0] - attributions = [] - - # Define sliding window parameters for temporal data - window_size = min(5, x.shape[2] // 4) # Adapt window size to input length - - for i in range(batch_size): - attr = occlusion_algo.attribute( - x[i:i+1], - sliding_window_shapes=(1, window_size), - target=class_idx[i].item(), - strides=(1, max(1, window_size // 2)) - ) - attributions.append(attr) - - return torch.cat(attributions).detach() - - def _compute_gradient_shap(self, shap_algo, x, class_idx): - """Compute GradientSHAP attributions""" - batch_size = x.shape[0] - attributions = [] - - for i in range(batch_size): - # Create random baselines (typically 10-50 for good estimates) - baselines = torch.randn(10, *x[i:i+1].shape[1:]) * 0.001 - - # Ensure baselines device matches input - baselines = baselines.to(x.device) - - attr = shap_algo.attribute( - x[i:i+1], baselines=baselines, target=class_idx[i].item() - ) - attributions.append(attr) - - return torch.cat(attributions).detach() - - def _feature_ablation_analysis(self, x, class_idx): - """Analyze model by systematically ablating input features""" - batch_size = x.shape[0] - results = [] - - for i in range(batch_size): - # Store original prediction - with torch.no_grad(): - orig_output = self.forward(x[i:i+1]) - orig_prob = torch.softmax(orig_output, dim=1)[0, class_idx[i]].item() - - # Test ablation of each channel - channel_importance = [] - for c in range(self.input_channels): - # Create ablated input (zero out one channel) - ablated_input = x[i:i+1].clone() - ablated_input[:, c, :] = 0 - - # Get prediction on ablated input - with torch.no_grad(): - ablated_output = self.forward(ablated_input) - ablated_prob = torch.softmax(ablated_output, dim=1)[0, class_idx[i]].item() - - # Impact is reduction in probability - channel_impact = orig_prob - ablated_prob - channel_importance.append(channel_impact) - - results.append(torch.tensor(channel_importance)) - - return torch.stack(results) - - def _compute_layer_importance(self, x, class_idx): - """Compute importance of each layer using Layer GradCAM""" - try: - from captum.attr import LayerGradCam - except ImportError: - raise ImportError("This method requires the 'captum' package.") - - batch_size = x.shape[0] - layer_importance = {} - - # Define layers to analyze - adapted for our new ModuleList structure - layers = {} - for i, block in enumerate(self.cnn_blocks): - layers[f'conv{i+1}'] = block['conv'] - - for layer_name, layer in layers.items(): - layer_gradcam = LayerGradCam(self.forward_wrapper, layer) - layer_attrs = [] - - for i in range(batch_size): - attr = layer_gradcam.attribute( - x[i:i+1], target=class_idx[i].item() - ) - # Process attribution to create a single importance score per sample - pooled_attr = torch.mean(attr, dim=1) - - layer_attrs.append(pooled_attr) - - layer_importance[layer_name] = torch.cat(layer_attrs).detach() - - return layer_importance - - def visualize_attributions(self, sample_idx, interpretations, time_axis=None, - channel_names=None, class_names=None): - """ - Visualize the various interpretation results - - Args: - sample_idx: Index of the sample to visualize - interpretations: Dictionary returned by interpret() method - time_axis: Optional array/list with time points for x-axis - channel_names: Optional list of channel names - class_names: Optional list of class names - """ - try: - import matplotlib.pyplot as plt - import numpy as np - except ImportError: - raise ImportError("This method requires matplotlib and numpy for visualization") - - if not channel_names: - channel_names = [f'Channel {i}' for i in range(self.input_channels)] - - if not class_names: - class_idx = interpretations['class_idx'][sample_idx].item() - class_name = f'Class {class_idx}' - else: - class_idx = interpretations['class_idx'][sample_idx].item() - class_name = class_names[class_idx] - - # Set up figure - plt.figure(figsize=(15, 12)) - - # Original input visualization (top row, first column) - plt.subplot(3, 3, 1) - if self.input is not None: - input_data = self.input[sample_idx].cpu().detach().numpy() - if time_axis is not None: - for i in range(input_data.shape[0]): - plt.plot(time_axis, input_data[i], label=channel_names[i]) - else: - for i in range(input_data.shape[0]): - plt.plot(input_data[i], label=channel_names[i]) - plt.legend(loc='best') - plt.title('Input Signal') - plt.xlabel('Time') - plt.ylabel('Value') - - # GradCAM feature importance (top row, second column) - if 'feature_importance' in interpretations and interpretations['feature_importance'] is not None: - plt.subplot(3, 3, 2) - heatmap = interpretations['feature_importance'][sample_idx].cpu().numpy() - if time_axis is not None: - plt.plot(time_axis, heatmap) - else: - plt.plot(heatmap) - plt.title('GradCAM Feature Importance') - plt.xlabel('Time') - plt.ylabel('Importance') - - # Attention weights (top row, third column) - if 'attention_weights' in interpretations and interpretations['attention_weights'] is not None: - plt.subplot(3, 3, 3) - attention = interpretations['attention_weights'][sample_idx].cpu().numpy() - - if time_axis is not None: - # Need to match attention time axis to input time axis - # (account for pooling in the network) - x_points = np.linspace(time_axis[0], time_axis[-1], len(attention)) - plt.plot(x_points, attention) - else: - plt.plot(attention) - plt.title('Attention Weights') - plt.xlabel('Time') - plt.ylabel('Attention') - - # Channel importance (middle row, first column) - if 'channel_importance' in interpretations and interpretations['channel_importance'] is not None: - plt.subplot(3, 3, 4) - ch_importance = interpretations['channel_importance'][sample_idx].cpu().numpy() - plt.bar(channel_names, ch_importance) - plt.title('Channel Importance') - plt.ylabel('Importance') - plt.xticks(rotation=45) - - # Integrated Gradients (middle row, second column) - if 'integrated_gradients' in interpretations: - plt.subplot(3, 3, 5) - ig_attr = interpretations['integrated_gradients'][sample_idx].cpu().numpy() - ig_attr_mean = np.mean(ig_attr, axis=0) # Average across channels for visualization - - if time_axis is not None: - plt.plot(time_axis, ig_attr_mean) - else: - plt.plot(ig_attr_mean) - plt.title('Integrated Gradients') - plt.xlabel('Time') - plt.ylabel('Attribution') - - # Feature Ablation (middle row, third column) - if 'feature_ablation' in interpretations: - plt.subplot(3, 3, 6) - ablation_scores = interpretations['feature_ablation'][sample_idx].cpu().numpy() - plt.bar(channel_names, ablation_scores) - plt.title('Feature Ablation Impact') - plt.ylabel('Probability Change') - plt.xticks(rotation=45) - - # SHAP values (bottom row, first column) - if 'gradient_shap' in interpretations: - plt.subplot(3, 3, 7) - shap_attr = interpretations['gradient_shap'][sample_idx].cpu().numpy() - # Visualize average SHAP value over time - shap_avg = np.mean(shap_attr, axis=0) - - if time_axis is not None: - plt.plot(time_axis, shap_avg) - else: - plt.plot(shap_avg) - plt.title('GradientSHAP Values') - plt.xlabel('Time') - plt.ylabel('SHAP Value') - - # Occlusion analysis (bottom row, second column) - if 'occlusion' in interpretations: - plt.subplot(3, 3, 8) - occlusion_attr = interpretations['occlusion'][sample_idx].cpu().numpy() - occlusion_avg = np.mean(occlusion_attr, axis=0) - - if time_axis is not None: - plt.plot(time_axis, occlusion_avg) - else: - plt.plot(occlusion_avg) - plt.title('Occlusion Analysis') - plt.xlabel('Time') - plt.ylabel('Attribution') - - # Prediction summary (bottom row, third column) - plt.subplot(3, 3, 9) - pred_probs = interpretations['prediction'][sample_idx].cpu().numpy() - classes = list(range(len(pred_probs))) - if class_names: - classes = class_names - plt.bar(classes, pred_probs) - plt.title(f'Prediction: {class_name}') - plt.ylabel('Probability') - plt.ylim([0, 1]) - - plt.tight_layout() - return plt.gcf() - - -# Example of creating a model with custom hyperparameters -def create_model_with_config(**kwargs): - """Helper function to create a model with specified config""" - config = { - "input_channels": 3, - "num_classes": 3, - "cnn_channels": (16, 32, 64), - "kernel_sizes": (5, 3, 3), - "pool_type": "max", - "dropout": 0.1, - "lstm_hidden_dim": 32, - "lstm_num_layers": 1, - "bidirectional": True, - "classifier_hidden_dims": [32], - "attention_type": "basic", - "residual_connections": False, - "layer_normalization": False - } - - # Update config with provided kwargs - config.update(kwargs) - - return CNN_LSTM_Classifier_Tunable(config=config) - From ad2164ffee0435ed3c7fc5fed3d543742faf8362 Mon Sep 17 00:00:00 2001 From: dcajal Date: Mon, 30 Mar 2026 16:05:26 +0200 Subject: [PATCH 52/93] Add channel and signal utility functions for processing EEG and respiratory data --- src/common/__init__.py | 1 + src/common/channel_utils.py | 31 ++++++++++++ src/common/signal_utils.py | 15 ++++++ src/ecg_processing.py | 45 +----------------- src/eeg_processing.py | 95 +++++++------------------------------ src/lib/utilities_HRV.py | 80 ------------------------------- src/pipeline/features.py | 18 +++---- src/resp_processing.py | 47 ++++-------------- 8 files changed, 81 insertions(+), 251 deletions(-) create mode 100644 src/common/__init__.py create mode 100644 src/common/channel_utils.py create mode 100644 src/common/signal_utils.py delete mode 100644 src/lib/utilities_HRV.py diff --git a/src/common/__init__.py b/src/common/__init__.py new file mode 100644 index 0000000..5baf7d3 --- /dev/null +++ b/src/common/__init__.py @@ -0,0 +1 @@ +"""Shared helpers for active signal-processing modules.""" \ No newline at end of file diff --git a/src/common/channel_utils.py b/src/common/channel_utils.py new file mode 100644 index 0000000..a15aa18 --- /dev/null +++ b/src/common/channel_utils.py @@ -0,0 +1,31 @@ +import os + +import pandas as pd + + +CHANNEL_TABLE_CACHE = {} + + +def normalize_channel_label(text): + normalized = ''.join(ch if ch.isalnum() else ' ' for ch in str(text).lower()) + return ' '.join(normalized.split()) + + +def split_channel_aliases(raw_aliases): + return {normalize_channel_label(alias) for alias in str(raw_aliases).split(';') if alias} + + +def get_cached_channel_table(csv_path): + normalized_csv_path = os.path.abspath(csv_path) + channels = CHANNEL_TABLE_CACHE.get(normalized_csv_path) + if channels is None: + channels = pd.read_csv(normalized_csv_path) + CHANNEL_TABLE_CACHE[normalized_csv_path] = channels + return channels, normalized_csv_path + + +def find_matching_label(signal_dict, aliases): + for label in signal_dict.keys(): + if normalize_channel_label(label) in aliases: + return label + return None \ No newline at end of file diff --git a/src/common/signal_utils.py b/src/common/signal_utils.py new file mode 100644 index 0000000..385ec8f --- /dev/null +++ b/src/common/signal_utils.py @@ -0,0 +1,15 @@ +import numpy as np + + +def resample_signal(signal, fs, target_fs): + signal = np.asarray(signal, dtype=float) + if signal.size == 0: + return signal, target_fs + if fs == target_fs: + return signal, target_fs + + duration = signal.size / fs + target_samples = max(1, int(round(duration * target_fs))) + time_original = np.linspace(0, duration, signal.size) + time_target = np.linspace(0, duration, target_samples) + return np.interp(time_target, time_original, signal), target_fs \ No newline at end of file diff --git a/src/ecg_processing.py b/src/ecg_processing.py index e50eaac..134a3e7 100644 --- a/src/ecg_processing.py +++ b/src/ecg_processing.py @@ -1,6 +1,6 @@ from .lib.ECG_processing import ECGprocessing -import pandas as pd import numpy as np +from src.common.channel_utils import normalize_channel_label ECG_KEYWORDS = ['ecg', 'ekg'] @@ -24,13 +24,9 @@ ] ECG_FEATURE_LENGTH = len(ECG_FEATURE_NAMES) -def _normalize_label(text): - return ''.join(ch if ch.isalnum() else ' ' for ch in str(text).lower()).strip() - - def _find_ecg_channel(physiological_data): for label in physiological_data.keys(): - label_clean = _normalize_label(label) + label_clean = normalize_channel_label(label) if any(keyword in label_clean for keyword in ECG_KEYWORDS): return label return None @@ -70,40 +66,3 @@ def processECG(physiological_data, physiological_fs, csv_path): pass return results - -""" def openECG(physiological_data_file, patient_id): - - f = pyedflib.EdfReader(physiological_data_file) - - signal_labels = f.getSignalLabels() - print(signal_labels) - - ecg_keywords = ['ecg', 'ekg'] - - idx = None - for i, label in enumerate(signal_labels): - label_clean = label.lower().strip() - - # Check if any ECG keyword is inside the label - if any(keyword in label_clean for keyword in ecg_keywords): - idx = i - break #first ECG channel only - - if idx is None: - raise ValueError("No ECG channel found") - - print("ECG channel:", signal_labels[idx]) - - ecg_signal = f.readSignal(idx) - fs = f.getSampleFrequency(idx) - - f.close() - - all_results = ECGprocessing(ecg_signal, fs, patient_id) - - if all_results is not None: - all_patients_ECGresults = pd.concat( - [all_patients_ECGresults, all_results], - ignore_index=True - ) - return all_patients_ECGresults """ diff --git a/src/eeg_processing.py b/src/eeg_processing.py index 89c23ba..112855b 100644 --- a/src/eeg_processing.py +++ b/src/eeg_processing.py @@ -1,44 +1,9 @@ -"""EEG_processing.py - -Este módulo contiene funciones para procesar datos EEG de los -hospitales incluidos en el desafío CincChallenge 2026. La principal -función definida es `MetricasHospitlal`, que recorre los archivos EDF -correspondientes a un hospital concreto, extrae las señales EEG, -las filtra, normaliza, crea épocas y calcula potencias de banda y -complejidades. Los resultados se guardan en un CSV resumen por -hospital. - -Características principales: - -- Soporta datos tanto del conjunto de entrenamiento como del - conjunto suplementario. -- Selección automática de canales EEG a partir de la tabla - `notebooks/channel_table.csv`. -- Creación de canales bipolares si están disponibles. -- Filtrado de banda 0.3-35 Hz y normalización de la señal. -- Re-muestreo a 200 Hz si fuese necesario. -- Cálculo de potencias de banda y complejidades usando - funciones auxiliares (`lib/EEG_functions.py`). -- Exportación de resultados en `results_summaryEEG_{hospital}.csv`. - -Uso típico: - ->>> from src.scripts.EEG_processing import MetricasHospitlal ->>> MetricasHospitlal('I0002') - -El módulo depende de `numpy`, `pandas`, `matplotlib`, `plotly` y de -las utilidades definidas en `lib/helper_code` y `lib/EEG_functions`. -""" -import sys -import os import pandas as pd import numpy as np -import helper_code as helper_code +from src.common.channel_utils import find_matching_label, get_cached_channel_table, normalize_channel_label, split_channel_aliases +from src.common.signal_utils import resample_signal from .lib import EEG_functions - -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) - EEG_CHANNEL_SPECS = { 'C3-M2': {'direct': 'c3-m2', 'positive': 'c3', 'reference': 'm2'}, 'C4-M1': {'direct': 'c4-m1', 'positive': 'c4', 'reference': 'm1'}, @@ -101,68 +66,37 @@ EEG_ALIASES_CACHE = {} -def _normalize_label(text): - normalized = ''.join(ch if ch.isalnum() else ' ' for ch in str(text).lower()) - return ' '.join(normalized.split()) - - -def _split_aliases(raw_aliases): - return {_normalize_label(alias) for alias in str(raw_aliases).split(';') if alias} - - def _build_eeg_aliases(channels): alias_lookup = {} for _, row in channels.iterrows(): - aliases = _split_aliases(row['Channel_Names']) + aliases = split_channel_aliases(row['Channel_Names']) if not aliases: continue - canonical_name = _normalize_label(str(row['Channel_Names']).split(';')[0]) + canonical_name = normalize_channel_label(str(row['Channel_Names']).split(';')[0]) alias_lookup[canonical_name] = aliases return alias_lookup def _get_eeg_aliases(csv_path): - normalized_csv_path = os.path.abspath(csv_path) + channels, normalized_csv_path = get_cached_channel_table(csv_path) eeg_aliases = EEG_ALIASES_CACHE.get(normalized_csv_path) if eeg_aliases is None: - channels = pd.read_csv(normalized_csv_path) eeg_aliases = _build_eeg_aliases(channels) EEG_ALIASES_CACHE[normalized_csv_path] = eeg_aliases return eeg_aliases -def _resample_signal(signal, fs, target_fs): - signal = np.asarray(signal, dtype=float) - if signal.size == 0: - return signal, target_fs - if fs == target_fs: - return signal, target_fs - - duration = signal.size / fs - target_samples = max(1, int(round(duration * target_fs))) - time_original = np.linspace(0, duration, signal.size) - time_target = np.linspace(0, duration, target_samples) - return np.interp(time_target, time_original, signal), target_fs - - -def _find_matching_label(physiological_data, aliases): - for label in physiological_data.keys(): - if _normalize_label(label) in aliases: - return label - return None - - def _get_channel_signal(channel_name, physiological_data, physiological_fs, eeg_aliases): channel_spec = EEG_CHANNEL_SPECS[channel_name] - direct_aliases = eeg_aliases.get(_normalize_label(channel_spec['direct']), set()) - direct_label = _find_matching_label(physiological_data, direct_aliases) + direct_aliases = eeg_aliases.get(normalize_channel_label(channel_spec['direct']), set()) + direct_label = find_matching_label(physiological_data, direct_aliases) if direct_label is not None and direct_label in physiological_fs: return np.asarray(physiological_data[direct_label], dtype=float), physiological_fs[direct_label] - positive_aliases = eeg_aliases.get(_normalize_label(channel_spec['positive']), set()) - reference_aliases = eeg_aliases.get(_normalize_label(channel_spec['reference']), set()) - positive_label = _find_matching_label(physiological_data, positive_aliases) - reference_label = _find_matching_label(physiological_data, reference_aliases) + positive_aliases = eeg_aliases.get(normalize_channel_label(channel_spec['positive']), set()) + reference_aliases = eeg_aliases.get(normalize_channel_label(channel_spec['reference']), set()) + positive_label = find_matching_label(physiological_data, positive_aliases) + reference_label = find_matching_label(physiological_data, reference_aliases) if positive_label is None or reference_label is None: return None, None if positive_label not in physiological_fs or reference_label not in physiological_fs: @@ -186,7 +120,7 @@ def _extract_channel_metrics(signal, fs): return None if fs != 200: - signal, fs = _resample_signal(signal, fs, 200) + signal, fs = resample_signal(signal, fs, 200) filtered = EEG_functions.butter_bandpass_filter(signal, lowcut=0.3, highcut=35, fs=fs, order=4) signal_std = np.std(filtered) @@ -243,4 +177,7 @@ def processEEG(physiological_data, physiological_fs, csv_path): continue values.append(float(channel_metrics.get(metric_name, 0.0))) - return np.asarray(values, dtype=np.float32) \ No newline at end of file + return np.asarray(values, dtype=np.float32) + + +_normalize_label = normalize_channel_label \ No newline at end of file diff --git a/src/lib/utilities_HRV.py b/src/lib/utilities_HRV.py deleted file mode 100644 index 4438c81..0000000 --- a/src/lib/utilities_HRV.py +++ /dev/null @@ -1,80 +0,0 @@ -import numpy as np -from scipy.interpolate import PchipInterpolator - -def interpolate_NN_pchip(NN, maxGap): - """ - NN: array of NN intervals (seconds) - maxGap: max number of consecutive NaNs allowed for interpolation - """ - - NN = np.asarray(NN).flatten() - NN_interp = NN.copy() - - nan_idx = np.isnan(NN) - - # Find NaN segments - d = np.diff(np.concatenate(([0], nan_idx.astype(int), [0]))) - start_idx = np.where(d == 1)[0] - end_idx = np.where(d == -1)[0] - 1 - - for k in range(len(start_idx)): - seg_len = end_idx[k] - start_idx[k] + 1 - - if seg_len <= maxGap: - left = start_idx[k] - 1 - right = end_idx[k] + 1 - - # Check bounds - if (left >= 0 and right < len(NN) and - not np.isnan(NN[left]) and not np.isnan(NN[right])): - - x = np.array([left, right]) - y = np.array([NN[left], NN[right]]) - - xi = np.arange(start_idx[k], end_idx[k] + 1) - - # PCHIP interpolation - interpolator = PchipInterpolator(x, y) - NN_interp[xi] = interpolator(xi) - - return NN_interp - -def remove_ectopic_beats(NN, window_size, threshold): - NN = np.asarray(NN).flatten() - NN_corrected = NN.copy() - - half_win = window_size // 2 - ectopic_count = 0 - valid_count = 0 - - for i in range(len(NN)): - - if np.isnan(NN[i]): - continue - - valid_count += 1 - - # Define local window - left = max(0, i - half_win) - right = min(len(NN), i + half_win + 1) # Python slice is exclusive - - local_segment = NN[left:right] - local_segment = local_segment[~np.isnan(local_segment)] - - if local_segment.size == 0: - continue - - med_val = np.median(local_segment) - - # Detect ectopic - if abs(NN[i] - med_val) > threshold * med_val: - NN_corrected[i] = med_val - ectopic_count += 1 - - # Percentage over valid NN - if valid_count > 0: - ectopic_perc = (ectopic_count / valid_count) * 100 - else: - ectopic_perc = np.nan - - return NN_corrected, ectopic_perc \ No newline at end of file diff --git a/src/pipeline/features.py b/src/pipeline/features.py index c202fe8..4f03ef1 100644 --- a/src/pipeline/features.py +++ b/src/pipeline/features.py @@ -5,9 +5,10 @@ import joblib import numpy as np +from src.common.channel_utils import normalize_channel_label from helper_code import HEADERS, PHYSIOLOGICAL_DATA_SUBFOLDER, load_age, load_sex, load_signal_data from src.ecg_processing import ECG_FEATURE_LENGTH, ECG_KEYWORDS, processECG -from src.eeg_processing import EEG_CHANNEL_SPECS, EEG_FEATURE_LENGTH, processEEG, _get_eeg_aliases, _normalize_label as _normalize_eeg_label +from src.eeg_processing import EEG_CHANNEL_SPECS, EEG_FEATURE_LENGTH, processEEG, _get_eeg_aliases from src.resp_processing import RESP_FEATURE_LENGTH, processResp, _get_resp_alias_groups from .config import DEFAULT_CSV_PATH, FEATURE_CACHE_FOLDER_NAME, SCRIPT_DIR, TOTAL_PHYSIOLOGICAL_FEATURE_LENGTH @@ -37,13 +38,6 @@ def _get_physiological_data_file(data_folder, site_id, patient_id, session_id): site_id, f"{patient_id}_ses-{session_id}.edf", ) - - -def _normalize_signal_label(text): - normalized = ''.join(ch if ch.isalnum() else ' ' for ch in str(text).lower()) - return ' '.join(normalized.split()) - - def _get_required_signal_aliases(csv_path): normalized_csv_path = os.path.abspath(csv_path) required_aliases = REQUIRED_SIGNAL_ALIASES_CACHE.get(normalized_csv_path) @@ -58,9 +52,9 @@ def _get_required_signal_aliases(csv_path): required_aliases.update(aliases) for channel_spec in EEG_CHANNEL_SPECS.values(): - required_aliases.update(eeg_aliases.get(_normalize_eeg_label(channel_spec['direct']), set())) - required_aliases.update(eeg_aliases.get(_normalize_eeg_label(channel_spec['positive']), set())) - required_aliases.update(eeg_aliases.get(_normalize_eeg_label(channel_spec['reference']), set())) + required_aliases.update(eeg_aliases.get(normalize_channel_label(channel_spec['direct']), set())) + required_aliases.update(eeg_aliases.get(normalize_channel_label(channel_spec['positive']), set())) + required_aliases.update(eeg_aliases.get(normalize_channel_label(channel_spec['reference']), set())) REQUIRED_SIGNAL_ALIASES_CACHE[normalized_csv_path] = required_aliases return required_aliases @@ -78,7 +72,7 @@ def _load_required_signal_data(edf_path, csv_path): for signal in edf.signals: label = signal.label.lower().strip() - normalized_label = _normalize_signal_label(label) + normalized_label = normalize_channel_label(label) is_required_signal = normalized_label in required_aliases is_ecg_signal = any(keyword in normalized_label for keyword in ECG_KEYWORDS) diff --git a/src/resp_processing.py b/src/resp_processing.py index 0b4577c..7e94301 100644 --- a/src/resp_processing.py +++ b/src/resp_processing.py @@ -1,10 +1,7 @@ from .lib import Resp_features -import sys -import os -import pandas as pd import numpy as np - -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) +from src.common.channel_utils import get_cached_channel_table, normalize_channel_label, split_channel_aliases +from src.common.signal_utils import resample_signal RESP_CHANNEL_GROUPS = ("Abdomen", "Chest", "Nasal", "Flow") RESP_FEATURE_NAMES = [ @@ -24,60 +21,36 @@ RESP_ALIAS_GROUPS_CACHE = {} -def _normalize_label(text): - normalized = ''.join(ch if ch.isalnum() else ' ' for ch in str(text).lower()) - return ' '.join(normalized.split()) - - -def _split_aliases(raw_aliases): - return {_normalize_label(alias) for alias in str(raw_aliases).split(';') if alias} - - def _build_resp_alias_groups(channels): resp_rows = channels[channels['Category'].eq('resp')].reset_index(drop=True) if len(resp_rows) < 7: return {} return { - 'Abdomen': _split_aliases(resp_rows.iloc[0]['Channel_Names']), - 'Chest': _split_aliases(resp_rows.iloc[1]['Channel_Names']), - 'Nasal': _split_aliases(resp_rows.iloc[2]['Channel_Names']), - 'Flow': _split_aliases(resp_rows.iloc[3]['Channel_Names']), - 'SpO2': _split_aliases(resp_rows.iloc[6]['Channel_Names']), + 'Abdomen': split_channel_aliases(resp_rows.iloc[0]['Channel_Names']), + 'Chest': split_channel_aliases(resp_rows.iloc[1]['Channel_Names']), + 'Nasal': split_channel_aliases(resp_rows.iloc[2]['Channel_Names']), + 'Flow': split_channel_aliases(resp_rows.iloc[3]['Channel_Names']), + 'SpO2': split_channel_aliases(resp_rows.iloc[6]['Channel_Names']), } def _get_resp_alias_groups(csv_path): - normalized_csv_path = os.path.abspath(csv_path) + channels, normalized_csv_path = get_cached_channel_table(csv_path) alias_groups = RESP_ALIAS_GROUPS_CACHE.get(normalized_csv_path) if alias_groups is None: - channels = pd.read_csv(normalized_csv_path) alias_groups = _build_resp_alias_groups(channels) RESP_ALIAS_GROUPS_CACHE[normalized_csv_path] = alias_groups return alias_groups def _find_resp_group(label, alias_groups): - normalized = _normalize_label(label) + normalized = normalize_channel_label(label) for group_name, aliases in alias_groups.items(): if normalized in aliases: return group_name return None -def _resample_signal(signal, fs, target_fs): - signal = np.asarray(signal, dtype=float) - if signal.size == 0: - return signal, target_fs - if fs == target_fs: - return signal, target_fs - - duration = signal.size / fs - target_samples = max(1, int(round(duration * target_fs))) - time_original = np.linspace(0, duration, signal.size) - time_target = np.linspace(0, duration, target_samples) - return np.interp(time_target, time_original, signal), target_fs - - def _compute_resp_quality(used, hat_br): used_array = np.asarray(used, dtype=float) if used_array.size: @@ -149,7 +122,7 @@ def processResp(physiological_data, physiological_fs, csv_path): if group_name is None: continue - resampled, fs = _resample_signal(signal, physiological_fs[label], 25) + resampled, fs = resample_signal(signal, physiological_fs[label], 25) resampled = np.nan_to_num(resampled, nan=0.0, posinf=0.0, neginf=0.0) if group_name == 'SpO2': From a778efd3dd6df739812d5ec0df70b92ca236d89b Mon Sep 17 00:00:00 2001 From: dcajal Date: Mon, 30 Mar 2026 16:14:52 +0200 Subject: [PATCH 53/93] remove unused files --- src/lib/model_functions.py | 295 ------------------------------------- src/results_analysis.py | 67 --------- src/segmentation.py | 193 ------------------------ 3 files changed, 555 deletions(-) delete mode 100644 src/lib/model_functions.py delete mode 100644 src/results_analysis.py delete mode 100644 src/segmentation.py diff --git a/src/lib/model_functions.py b/src/lib/model_functions.py deleted file mode 100644 index 94b8432..0000000 --- a/src/lib/model_functions.py +++ /dev/null @@ -1,295 +0,0 @@ -import numpy as np -import pandas as pd -from xgboost import XGBClassifier -import numpy as np -import pandas as pd -from sklearn.model_selection import train_test_split -from sklearn.compose import ColumnTransformer -from sklearn.pipeline import Pipeline -from sklearn.impute import KNNImputer, SimpleImputer -from sklearn.preprocessing import StandardScaler, OneHotEncoder -from sklearn.metrics import classification_report, roc_auc_score, f1_score - -def best_threshold(proba_final,y_test): - thresholds = np.linspace(0, 1, 100) - best_thr = 0 - best_f1 = 0 - for t in thresholds: - preds = (proba_final >= t).astype(int) - f1 = f1_score(y_test, preds) - if f1 > best_f1: - best_f1 = f1 - best_thr = t - return best_thr - -def evaluar_rendimiento(y_real,y_pred, proba_final, dict_probas, nombre_ensemble="Ensemble"): - """ - Imprime un reporte completo de métricas de clasificación y AUC. - - Args: - y_real: Etiquetas verdaderas (y_test o y_val). - proba_final: Probabilidades del modelo combinado. - dict_probas: Diccionario con formato {'Nombre': lista_probas} para modelos individuales. - nombre_ensemble: Nombre personalizado para el modelo principal. - """ - - # Impresión de métricas principales - print(f"Reporte de clasificacion {nombre_ensemble}") - print(classification_report(y_real, y_pred)) - - # AUC del Ensemble - roc_auc = roc_auc_score(y_real, proba_final) - print(f"AUC {nombre_ensemble}: {roc_auc:.2%}") - print("\n--- Comparativa AUC Modelos Individuales ---") - - # 4. AUC de modelos individuales usando el diccionario - for nombre, probas in dict_probas.items(): - auc_score = roc_auc_score(y_real, probas) - print(f"AUC {nombre:<8}: {auc_score:.2%}") - print("-" * 30) - -def preprocess_multimodal_data(demo, df_model): - - #Llamamos al dataset de entrada que contiene las variables calculadas y confirma que no hay IDs duplicados - df_model = df_model.drop_duplicates(subset='ID') - demo=demo.drop_duplicates(subset='ID') - #Haz el merge con las demograficas del paciente, Age, Sex y Label - # Asegurar tipo string en ID - for d in [demo, df_model]: - d['ID'] = d['ID'].astype(str).str.strip() # .strip() por si hay espacios invisibles - - df_model2= (demo.merge(df_model, on="ID", how="inner")) - - # Extrae las columnas que se usarán - df_model2=df_model2[['ID','label', 'Age','Sex','Nasal_Peakedness_Max', 'Nasal_Peakedness_Min', 'Nasal_Peakedness_Median','Nasal_Peakedness_Std', - 'Chest_Peakedness_Max', 'Chest_Peakedness_Min','Abdomen_Peakedness_Max','Abdomen_Peakedness_Min','Abdomen_Peakedness_Std', - 'Flow_Peakedness_Max','Flow_Peakedness_Min','Flow_Peakedness_Median','Flow_Peakedness_Std','SpO2_Max','SpO2_Min','SpO2_Std', - 'CET90','ODI_Mean','ODI_deepness','C3-M2_Hjorth_Complexity','C4-M1_Hjorth_Complexity','F3-M2_Hjorth_Complexity', - 'F4-M1_Hjorth_Complexity','C3-M2_Hjorth_Mobility','F3-M2_Hjorth_Mobility','F4-M1_Hjorth_Mobility','C3-M2_Ratio_Slow_Fast', - 'C4-M1_Ratio_Slow_Fast','F3-M2_Ratio_Slow_Fast','F4-M1_Ratio_Slow_Fast','C3-M2_Rel_Beta','F3-M2_Rel_Beta','F4-M1_Rel_Beta', - 'C4-M1_Rel_Sigma','F3-M2_Rel_Sigma','C3-M2_Relative_Delta_Power','C4-M1_Relative_Delta_Power','F3-M2_Relative_Delta_Power', - 'F4-M1_Relative_Delta_Power','C3-M2_Theta_Alpha_Ratio','C4-M1_Theta_Alpha_Ratio','F3-M2_Theta_Alpha_Ratio','F4-M1_Theta_Alpha_Ratio', - 'C3-M2_Theta_Beta_Ratio','C4-M1_Theta_Beta_Ratio','F3-M2_Theta_Beta_Ratio','F4-M1_Theta_Beta_Ratio','C3-M2_kurtosis_Alpha', - 'C3-M2_kurtosis_Beta','C4-M1_kurtosis_Beta','F3-M2_kurtosis_Beta','F4-M1_kurtosis_Beta','C3-M2_kurtosis_Delta','C4-M1_kurtosis_Delta', - 'F3-M2_kurtosis_Delta','F4-M1_kurtosis_Delta','C3-M2_kurtosis_Sigma','C4-M1_kurtosis_Sigma','F3-M2_kurtosis_Sigma','F4-M1_kurtosis_Sigma', - 'C3-M2_kurtosis_Theta','C4-M1_kurtosis_Theta','F3-M2_kurtosis_Theta','F4-M1_kurtosis_Theta','C3-M2_variability_Delta', - 'C4-M1_variability_Delta','F3-M2_variability_Delta','F4-M1_variability_Delta','PIP_med','PIP_std','PNNSS_med', - 'PNNSS_std','AVNN_med','AVNN_std','SDNN_med','SDNN_std','RMSSD_std','HF_med','HF_std','ECTOPIC_med','ECTOPIC_std']] - - - # Ajusta el tipo de dato, todas deben ser numericas excepto Sex, Label y ID. Sex y label tendría que ser logical? o categorical? - df_model2['Sex'] = df_model2['Sex'].astype('category') - df_model2['label'] = df_model2['label'].astype(int) - - #Unificar los valores faltantes, que sean consistentes - df_model2 = df_model2.replace([-999, "NA", "null", "None", "nan", ""], np.nan) - # Eliminar filas con demasiados NaNs (mínimo 50% de datos válidos) - umbral = int(len(df_model2.columns) * 0.5) - df_model2 = df_model2.dropna(thresh=umbral) - - #Procesa los datos, imputar nans y estandarizar por la media y desviacion estandar (zscore) - y = df_model2['label'] - X = df_model2.drop(columns=[ 'label','ID']) # Excluimos ID, label del entrenamiento - - return {"X": X, - "y": y} - -def prepare_multimodal_data(demo, df_model, testsize=0.1): - - proc_data=preprocess_multimodal_data(demo, df_model) - X = proc_data["X"] - y = proc_data["y"] - X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=testsize, random_state=1999, stratify=y) - - cols_categoricas = ['Sex'] - cols_numericas = [c for c in X_train.columns if c not in cols_categoricas] - - preprocessor = ColumnTransformer([ - ("num", Pipeline([ - ("imputer", KNNImputer(n_neighbors=5)), - ("scaler", StandardScaler()) - ]), cols_numericas), - - ("cat", Pipeline([ - ("imputer", SimpleImputer(strategy="most_frequent")), - ("encoder", OneHotEncoder(handle_unknown="ignore", sparse_output=False)) - ]), cols_categoricas) - ]) - preprocessor.set_output(transform="pandas") - - X_train = preprocessor.fit_transform(X_train) - X_test = preprocessor.transform(X_test) - - feature_names = preprocessor.get_feature_names_out() - resp_mask = np.array([ any(k in col for k in ["Age","Sex","Nasal_Peakedness", "Chest_Peakedness", "Abdomen_Peakedness", "Flow_Peakedness", "SpO2", "CET90", "ODI" ]) - for col in feature_names]) - eeg_mask = np.array([ any(k in col for k in ["Age","Sex","C3-M2", "C4-M1", "F3-M2", "F4-M1"]) - for col in feature_names]) - ecg_mask = np.array([ any(k in col for k in ["Age","Sex","PIP_", "PNNSS_", "AVNN_", "SDNN_", "RMSSD_", "HF_", "ECTOPIC_"]) - for col in feature_names]) - - # Subsets finales - return { - "X_train": X_train, - "X_test": X_test, - "y_train": y_train, - "y_test": y_test, - "resp": (X_train.loc[:, resp_mask], X_test.loc[:, resp_mask]), - "eeg": (X_train.loc[:, eeg_mask], X_test.loc[:, eeg_mask]), - "ecg": (X_train.loc[:, ecg_mask], X_test.loc[:, ecg_mask]), - "preprocessor": preprocessor - } - -def prepare_multimodal_validationdata(demo, df_model, preprocessor): - - #Llamamos al dataset de entrada que contiene las variables calculadas y confirma que no hay IDs duplicados - df_model = df_model.drop_duplicates(subset='ID') - demo=demo.drop_duplicates(subset='ID') - #Haz el merge con las demograficas del paciente, Age, Sex y Label - # Asegurar tipo string en ID - for d in [demo, df_model]: - d['ID'] = d['ID'].astype(str).str.strip() # .strip() por si hay espacios invisibles - - df_model2= (demo.merge(df_model, on="ID", how="inner")) - - # Extrae las columnas que se usarán - columnas_deseadas=['ID', 'Age','Sex','Nasal_Peakedness_Max', 'Nasal_Peakedness_Min', 'Nasal_Peakedness_Median','Nasal_Peakedness_Std', - 'Chest_Peakedness_Max', 'Chest_Peakedness_Min','Abdomen_Peakedness_Max','Abdomen_Peakedness_Min','Abdomen_Peakedness_Std', - 'Flow_Peakedness_Max','Flow_Peakedness_Min','Flow_Peakedness_Median','Flow_Peakedness_Std','SpO2_Max','SpO2_Min','SpO2_Std', - 'CET90','ODI_Mean','ODI_deepness','C3-M2_Hjorth_Complexity','C4-M1_Hjorth_Complexity','F3-M2_Hjorth_Complexity', - 'F4-M1_Hjorth_Complexity','C3-M2_Hjorth_Mobility','F3-M2_Hjorth_Mobility','F4-M1_Hjorth_Mobility','C3-M2_Ratio_Slow_Fast', - 'C4-M1_Ratio_Slow_Fast','F3-M2_Ratio_Slow_Fast','F4-M1_Ratio_Slow_Fast','C3-M2_Rel_Beta','F3-M2_Rel_Beta','F4-M1_Rel_Beta', - 'C4-M1_Rel_Sigma','F3-M2_Rel_Sigma','C3-M2_Relative_Delta_Power','C4-M1_Relative_Delta_Power','F3-M2_Relative_Delta_Power', - 'F4-M1_Relative_Delta_Power','C3-M2_Theta_Alpha_Ratio','C4-M1_Theta_Alpha_Ratio','F3-M2_Theta_Alpha_Ratio','F4-M1_Theta_Alpha_Ratio', - 'C3-M2_Theta_Beta_Ratio','C4-M1_Theta_Beta_Ratio','F3-M2_Theta_Beta_Ratio','F4-M1_Theta_Beta_Ratio','C3-M2_kurtosis_Alpha', - 'C3-M2_kurtosis_Beta','C4-M1_kurtosis_Beta','F3-M2_kurtosis_Beta','F4-M1_kurtosis_Beta','C3-M2_kurtosis_Delta','C4-M1_kurtosis_Delta', - 'F3-M2_kurtosis_Delta','F4-M1_kurtosis_Delta','C3-M2_kurtosis_Sigma','C4-M1_kurtosis_Sigma','F3-M2_kurtosis_Sigma','F4-M1_kurtosis_Sigma', - 'C3-M2_kurtosis_Theta','C4-M1_kurtosis_Theta','F3-M2_kurtosis_Theta','F4-M1_kurtosis_Theta','C3-M2_variability_Delta', - 'C4-M1_variability_Delta','F3-M2_variability_Delta','F4-M1_variability_Delta','PIP_med','PIP_std','PNNSS_med', - 'PNNSS_std','AVNN_med','AVNN_std','SDNN_med','SDNN_std','RMSSD_std','HF_med','HF_std','ECTOPIC_med','ECTOPIC_std'] - #Filtrar extraer solo las que existen en el modelo - columnas_existentes = [col for col in columnas_deseadas if col in df_model2.columns] - df_model2 = df_model2[columnas_existentes] - - # Ajusta el tipo de dato, todas deben ser numericas excepto Sex, Label y ID. Sex y label tendría que ser logical? o categorical? - df_model2['Sex'] = df_model2['Sex'].astype('category') - #df_model2['label'] = df_model2['label'].astype(int) - - #Unificar los valores faltantes, que sean consistentes - df_model2 = df_model2.replace([-999, "NA", "null", "None", "nan", ""], np.nan) - # Eliminar filas con demasiados NaNs (mínimo 50% de datos válidos) - umbral = int(len(df_model2.columns) * 0.5) - df_model2 = df_model2.dropna(thresh=umbral) - - keys_resp = ["Nasal_Peakedness", "Chest_Peakedness", "Abdomen_Peakedness", "Flow_Peakedness", "SpO2", "CET90", "ODI"] - keys_eeg = ["C3-M2", "C4-M1", "F3-M2", "F4-M1"] - keys_ecg = ["PIP_", "PNNSS_", "AVNN_", "SDNN_", "RMSSD_", "HF_", "ECTOPIC_"] - # Identificar qué columnas de 'columnas_deseadas' pertenecen a cada señal - cols_resp_esperadas = [c for c in columnas_deseadas if any(c.startswith(k) for k in keys_resp)] - cols_eeg_esperadas = [c for c in columnas_deseadas if any(c.startswith(k) for k in keys_eeg)] - cols_ecg_esperadas = [c for c in columnas_deseadas if any(c.startswith(k) for k in keys_ecg)] - - # Verificación Estricta: ¿Están TODAS en el dataframe de entrada? - presencia_original = { - "resp": all(c in df_model2.columns for c in cols_resp_esperadas), - "eeg": all(c in df_model2.columns for c in cols_eeg_esperadas), - "ecg": all(c in df_model2.columns for c in cols_ecg_esperadas) - } - - #Procesa los datos, imputar nans y estandarizar por la media y desviacion estandar (zscore) - #y = df_model2['label'] - #X = df_model2.drop(columns=[ 'label','ID']) # Excluimos ID, label del entrenamiento - X = df_model2.drop(columns=[ 'ID']) # Excluimos ID, label del entrenamiento - - # Reconstrucción técnica (solo para que transform no falle, luego lo filtraremos) - columnas_que_espera = preprocessor.feature_names_in_ - for col in columnas_que_espera: - if col not in X.columns: - X[col] = np.nan - - X = X[columnas_que_espera] - X_val_array = preprocessor.transform(X) - feature_names = preprocessor.get_feature_names_out() - X_validation = pd.DataFrame(X_val_array, columns=feature_names, index=X.index) - - # 6. Extracción Final (Si no estaba completa originalmente, devuelve None) - def validar_y_extraer(cols_esperadas, existe_completa): - if not existe_completa: - return None - - # Filtramos en los nombres transformados (ej: num__Nasal...) - cols_senal = [c for c in feature_names if any(k in c for k in cols_esperadas)] - cols_demo = [c for c in feature_names if any(d in c for d in ["Age", "Sex"])] - - orden_final = [c for c in feature_names if c in (cols_demo + cols_senal)] - return X_validation[orden_final] - - return { - "X_valid": X_validation, - "resp": validar_y_extraer(cols_resp_esperadas, presencia_original["resp"]), - "eeg": validar_y_extraer(cols_eeg_esperadas, presencia_original["eeg"]), - "ecg": validar_y_extraer(cols_ecg_esperadas, presencia_original["ecg"]) - } - -def reconstruct_multimodal_data(demo, df_model): - data = prepare_multimodal_data(demo, df_model) - X_train = data["X_train"] - X_test = data["X_test"] - X_train_resp, X_test_resp = data["resp"] - X_train_EEG, X_test_EEG = data["eeg"] - X_train_ECG, X_test_ECG = data["ecg"] - y_train = data["y_train"] - y_test = data["y_test"] - preprocessor=data["preprocessor"] - - # Configuramos Los parámetros base de los modelos - est=300 # Número de árboles - depth=4 # Profundidad (evita overfitting) - lr=0.05 # Paso de aprendizaje - ss=0.8 # Usa el 80% de los datos para cada árbol (evita memorizar) - rs=42 # Para que sea replicable - categ=False - met='auc' # Métrica de error interna - stopi=20 # early stopping si la metrica no mejora en tantas epocas - neg = (y_train == 0).sum() - pos = (y_train == 1).sum() - scale_pos_weight = neg / pos - - all_xgb = XGBClassifier(scale_pos_weight=scale_pos_weight,n_estimators=est, max_depth=depth, learning_rate=lr, subsample=ss, random_state=rs, enable_categorical=categ, eval_metric=met, early_stopping_rounds=stopi) - ECG_xgb = XGBClassifier(scale_pos_weight=scale_pos_weight,n_estimators=est, max_depth=depth, learning_rate=lr, subsample=ss, random_state=rs, enable_categorical=categ, eval_metric=met, early_stopping_rounds=stopi) - resp_xgb = XGBClassifier(scale_pos_weight=scale_pos_weight,n_estimators=est, max_depth=depth, learning_rate=lr, subsample=ss, random_state=rs, enable_categorical=categ, eval_metric=met, early_stopping_rounds=stopi) - EEG_xgb = XGBClassifier(scale_pos_weight=scale_pos_weight,n_estimators=est, max_depth=depth, learning_rate=lr, subsample=ss, random_state=rs, enable_categorical=categ, eval_metric=met, early_stopping_rounds=stopi) - - # Entrenamiento - all_xgb.fit(X_train, y_train,eval_set=[(X_test, y_test)], verbose=False) - ECG_xgb.fit(X_train_ECG, y_train,eval_set=[(X_test_ECG, y_test)], verbose=False) - resp_xgb.fit(X_train_resp, y_train,eval_set=[(X_test_resp, y_test)], verbose=False) - EEG_xgb.fit(X_train_EEG, y_train,eval_set=[(X_test_EEG, y_test)], verbose=False) - - res = prepare_multimodal_validationdata(demo, dfv, preprocessor) - - # Cálculo dinámico de probabilidades - probas = [] - probas_individuales = {} - - if res["resp"] is not None: - vproba_resp = resp_xgb.predict_proba(res["resp"])[:, 1] - probas.append(vproba_resp) - probas_individuales["RESP"] = vproba_resp - - if res["eeg"] is not None: - vproba_eeg = EEG_xgb.predict_proba(res["eeg"])[:, 1] - probas.append(vproba_eeg) - probas_individuales["EEG"] = vproba_eeg - - if res["ecg"] is not None: - vproba_ecg = ECG_xgb.predict_proba(res["ecg"])[:, 1] - probas.append(vproba_ecg) - probas_individuales["ECG"] = vproba_ecg - - # Promedio de lo que sea que hayamos podido ejecutar - vproba_final = np.mean(probas, axis=0) - y_predv = (vproba_final >= best_thr).astype(int) - - #y_v=res["y_valid"] - #evaluar_rendimiento(y_v,y_predv, vproba_final, probas_individuales, nombre_ensemble="Ensemble gboost validation") diff --git a/src/results_analysis.py b/src/results_analysis.py deleted file mode 100644 index b415a4b..0000000 --- a/src/results_analysis.py +++ /dev/null @@ -1,67 +0,0 @@ -import numpy as np -import pandas as pd -import sys -import os -import plotly.express as px - -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -from src.lib import EEG_functions -import seaborn as sns -import matplotlib.pyplot as plt -import plotly.express as px - -hospital = ["I0006","I0002","I0004","I0007", "S0001"] -results = pd.DataFrame() -for h in hospital: - results = pd.concat([results, pd.read_csv(f"results_summaryEEG_{h}.csv")], ignore_index=True) - -demographics = pd.read_csv(os.path.join('C:/BSICoS/CincChallenge2026/CincChallenge_2026/data/training_set', "demographics.csv")) -demographics = pd.concat([demographics, pd.read_csv(os.path.join('C:/BSICoS/CincChallenge2026/CincChallenge_2026/data/supplementary_set', "demographics.csv"))], ignore_index=True) - -for index, row in results.iterrows(): - patient_id = row['Patient_ID'] - hospital_id = row['File'][4:9] # Asumiendo que los primeros 5 caracteres del nombre del archivo indican el hospital - demographics_row = demographics[(demographics['BDSPPatientID'] == patient_id) & (demographics['SiteID'] == hospital_id)] - if not demographics_row.empty: - cognitive_impairment = demographics_row['Cognitive_Impairment'].values[0] - time_to_event = demographics_row['Time_to_Event'].values[0] - results.at[index, 'Hospital'] = hospital_id - results.at[index, 'CognitiveImpairment'] = cognitive_impairment - results.at[index, 'Time_to_Event'] = time_to_event - else: - results.at[index, 'Hospital'] = hospital_id - results.at[index, 'CognitiveImpairment'] = np.nan # O cualquier valor que indique que no se encontró información - results.at[index, 'Time_to_Event'] = np.nan - -df = pd.DataFrame(results) -# Agrupar por electrodo -for elec in results['Channel'].unique(): - subset = results[results['Channel'] == elec] - - print(subset.Hospital.unique()) - # Hacer un boxplot de cada característica que separe entre pacientes con congnitive impairment y sin él - for col in subset.columns[3:-2]: - print(col) - - fig = px.box(subset, - x='CognitiveImpairment', - y=col, - color='CognitiveImpairment', - notched=True, - points="all", - hover_data=['Patient_ID', 'Channel'], - title=f"{elec} - Comparativa de {col} según Estado Cognitivo") - - fig.update_layout(template="plotly_white") - fig.write_html(f"graphs/ComparativaCognitiveImpairment/2segundos/PorHospital/html/{elec}_{col}.html") # Guardar como HTML para visualización interactiva - # fig.delete_traces([0]) # Eliminar la leyenda para que no se repita en cada gráfico - - # Generar el boxplot con Seaborn (Extremadamente rápido) - plt.figure(figsize=(10, 6)) - sns.boxplot(data=df, x='CognitiveImpairment', y=col, hue='CognitiveImpairment', notch=True) - sns.stripplot(data=df, x='CognitiveImpairment', y=col, color="black", alpha=0.3, size=3) # Equivalente a points="all" - - plt.title(f"Comparativa {elec} - {col}") - plt.savefig(f"graphs/ComparativaCognitiveImpairment/2segundos/PorHospital/{hospital_id}/png/{elec}_{col}.png", dpi=100) - # \graphs\ComparativaCognitiveImpairment\2segundos\PorHospital\I0006\png - plt.close() # ¡Importante! Para no saturar la memoria RAM \ No newline at end of file diff --git a/src/segmentation.py b/src/segmentation.py deleted file mode 100644 index 6ae7406..0000000 --- a/src/segmentation.py +++ /dev/null @@ -1,193 +0,0 @@ -from binascii import Error -import numpy as np -import pandas as pd -import sys -import os -import matplotlib.pyplot as plt -import plotly.express as px -import plotly.graph_objects as go -import scipy.signal -from plotly.subplots import make_subplots -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -import lib.helper_code as helper_code -import lib.EEG_functions as EEG_functions - -for hospital in ['I0002','I0006', 'I0004','I0007','S0001']: - print(f"Procesando hospital: {hospital}") - - if hospital == 'I0002' or hospital == 'I0006' or hospital == "S0001": - datapath = 'data/training_set/Physiological_data/'+hospital - else: - datapath = 'data/supplementary_set/Physiological_data/'+hospital - - channels = pd.read_csv("notebooks/channel_table.csv") - selectEEG = channels[channels['Category'].isin(['eeg'])] - demographics = pd.read_csv(os.path.join('data/training_set', "demographics.csv")) - selectresp = channels[channels['Category'].isin(['resp'])] - selectECG = channels[channels['Category'].isin(['ecg'])] - - # Datos = pd.DataFrame(columns=['File', 'Channel', 'Sampling_Frequency', 'Duration_sec']) - lista_dir = os.listdir(datapath) - results = [] - - for file in lista_dir: - # Cargar el archivo (sustituye por tu ruta real) - edf = helper_code.edfio.read_edf(os.path.join(datapath, file)) - - id = file[9:-10] # Asumiendo que el ID es el nombre del archivo sin la extensión - selEEG = [] - selECG = [] - selResp = [] - labels = [] - data = [] - HayECG = False - for i, sig in enumerate(edf.signals): - for index in selectECG.index: - if sig.label.lower() in selectECG['Channel_Names'][index].lower(): - HayECG = True - print(f"Canal seleccionado: {sig.label}") - selECG.append([i,sig]) - labels.append(sig.label) - data.append(sig.data) # Guardar la señal ECG sin filtrar para su posterior procesamiento - break - - HayResp = False - for i, sig in enumerate(edf.signals): - for index in selectresp.index: - if sig.label.lower() in selectresp['Channel_Names'][index].lower(): - HayResp = True - fs = sig.sampling_frequency - if sig.label == "O2": - print(f"Warning: {sig.label} is detected as respiratory signal but has a sampling frequency higher than 100 Hz. Check the data.") - else: - print(f"Canal seleccionado: {sig.label}") - selResp.append([i,sig]) - labels.append(sig.label) - - data.append(sig.data) # Guardar la señal RESP sin filtrar para su posterior procesamiento - break - - # Listar canales para identificar los de interés (ej: C3-M2, O1-M2) - # print("Canales detectados:") - - HayEEG = False - for i, sig in enumerate(edf.signals): - # print(f"[{i}] {sig.label}") - # print length fs and duration - # print(f"Length: {len(sig.data)}, Sampling Frequency: {sig.sampling_frequency} Hz, Duration: {len(sig.data)/sig.sampling_frequency:.2f} seconds") - for index in selectEEG.index: - if sig.label.lower() in selectEEG['Channel_Names'][index].lower(): - print(f"Canal seleccionado: {sig.label}") - selEEG.append([i,sig]) - # labels.append(sig.label) - HayEEG = True - break - # for i in range(len(edf.signals)): - # print(f"Longitud: {edf.signals[i].data.shape}, Canal: {edf.signals[i].label}, Frecuencia de muestreo: {edf.signals[i].sampling_frequency} Hz, Duración: {len(edf.signals[i].data)/edf.signals[i].sampling_frequency:.2f} segundos") - - if HayEEG and HayECG and HayResp: - - Bipolar = pd.DataFrame() - if all(label in labels for label in ["F3", "F4", "M1", "M2"]): - Bipolar['F3-M2'] = edf.signals[edf.labels.index("F3")].data - edf.signals[edf.labels.index("M2")].data - Bipolar['F4-M1'] = edf.signals[edf.labels.index("F4")].data - edf.signals[edf.labels.index("M1")].data - if all(label in labels for label in ["C3", "C4", "M1", "M2"]): - Bipolar['C3-M2'] = edf.signals[edf.labels.index("C3")].data - edf.signals[edf.labels.index("M2")].data - Bipolar['C4-M1'] = edf.signals[edf.labels.index("C4")].data - edf.signals[edf.labels.index("M1")].data - if all(label in labels for label in ["O2", "O1", "M1", "M2"]): - Bipolar['O2-M2'] = edf.signals[edf.labels.index("O1")].data - edf.signals[edf.labels.index("M2")].data - Bipolar['O1-M1'] = edf.signals[edf.labels.index("O2")].data - edf.signals[edf.labels.index("M1")].data - - # print(f"Archivo {file} tiene ECG, RESP y EEG. Se procesará con canales bipolares.") - if not Bipolar.empty: - for col in Bipolar.columns: - # print(f"Archivo: {file}, Canal: {col}, Frecuencia de muestreo: {sig.sampling_frequency} Hz, Duración: {len(Bipolar[col])/sig.sampling_frequency:.2f} segundos") - fs = edf.signals[edf.labels.index("O2")].sampling_frequency # Asumimos que todos los canales tienen la misma frecuencia de muestreo - time = np.linspace(0, len(Bipolar[col]) / fs, len(Bipolar[col])) - fil = EEG_functions.butter_bandpass_filter(Bipolar[col], lowcut=0.3, highcut=35, fs=fs, order=4) - norm = (fil-np.mean(fil))/np.std(fil) - - data.append(norm) # Restar la media para centrar la señal - labels.append(col) - # columns = Bipolar.columns.tolist() - else: - for i, (idx, sig) in enumerate(selEEG): - # print(f"Archivo: {file}, Canal: {sig.label}, Frecuencia de muestreo: {sig.sampling_frequency} Hz, Duración: {len(sig.data)/sig.sampling_frequency:.2f} segundos") - fs = sig.sampling_frequency - time = np.linspace(0, len(sig.data) / fs, len(sig.data)) - fil = EEG_functions.butter_bandpass_filter(sig.data, lowcut=0.3, highcut=35, fs=fs, order=4) - norm = (fil-np.mean(fil))/np.std(fil) - labels.append(sig.label) - data.append(norm) # Restar la media para centrar la señal - - # columns = [selEEG[i][1].label for i in range(len(selEEG))] - - - # for i in range(len(selResp)): - # columns.append(selResp[i][1].label) - demographicsID = demographics[demographics['BDSPPatientID'] == int(id)] - print(demographicsID) - - columnashoras = [] - for elec in labels: - for h in np.floor(np.arange(0, len(sig.data) / fs / 3600, 1)): - columnashoras.append(elec + f"_h{int(h)}") - epochs5min = pd.DataFrame(columns= columnashoras) - - for i, elec in enumerate(labels): - # Check fs of the current channel - if elec == 'O2_resp': - fs = edf.signals[edf.labels.index('O2')].sampling_frequency - else: - fs = edf.signals[edf.labels.index(labels[i])].sampling_frequency - - if fs != 200: - # print(f"Warning: Sampling frequency for channel {elec} in file {file} is {fs} Hz, expected 200 Hz. Check the data.") - duration = len(data[i]) / fs - time_original = np.linspace(0, duration, len(data[i])) - - num_samples_target = int(duration * 200 ) - time_target = np.linspace(0, duration, num_samples_target) - data[i] = np.interp(time_target, time_original, data[i]) - fs = 200 # Update fs to the target sampling frequency after resampling - - # Plot comparison of original and resampled signals - # lim = 50000 - # factor = len(filtered_data[0]) / num_samples_target - # plt.figure(figsize=(12, 6)) - # plt.plot(time_target[:int(lim/factor)], resampled_data[:int(lim/factor)], label='Resampled Signal') - # plt.plot(time_original[:lim], filtered_data[i][:lim], label='Original Signal') - # plt.title(f'Original vs Resampled Signal - {elec} in {file}') - # plt.show() - - - epoch_length = 300 # Duración de cada época en segundos - # epochs = EEG_functions.create_epochs(df[elec].values, fs, epoch_duration=epoch_length) - epochs = EEG_functions.create_epochs(data[i], fs, epoch_duration=epoch_length) - - # Coger los primeros 5min de cada hora - for h in np.floor(np.arange(0, len(epochs)*epoch_length/3600, 1)): - start_epoch = int(h*3600/epoch_length) - end_epoch = int((h*3600 + 5*60)/epoch_length) - if end_epoch > len(epochs): - end_epoch = len(epochs) - c = elec+'_h'+str(int(h)) - epochs5min.loc[:, c] = epochs[start_epoch:end_epoch][0] - - - # del epochs, fs # Liberar memoria - - # Plotly sublot epochs5min.iloc[:,::8].plot() - # h = 6 - # fig = make_subplots(rows=int(epochs5min.shape[1]/8), cols=1, subplot_titles=epochs5min.columns[h::8]) - # for i in range(h, epochs5min.shape[1], 8): - # print(f"Plotting channel: {epochs5min.columns[i]}") - # fig.add_trace(go.Scatter(x=epochs5min.index, y=epochs5min.iloc[:,i], mode='lines', name=epochs5min.columns[i]), row=int(i/8)+1, col=1) - # fig.update_layout(height=3000, width=1200, title_text=f"Epochs de 5 minutos para el archivo {file}") - # fig.show() - if epochs5min.shape[1] > 0 and epochs5min.shape[0] == 60000: - epochs5min.to_parquet(os.path.join('X:/bsicos01/__comun/Physionet/Data5min', f"{id}.parquet")) - else: - print(f"Error: No channels in epochs5min for file {file}") - # stop program if no channels are processed - raise ValueError(f"No channels in epochs5min for file {file}") \ No newline at end of file From b74b6e9a6ef6ddea983d257b8fd1ef2f8060c2aa Mon Sep 17 00:00:00 2001 From: dcajal Date: Mon, 30 Mar 2026 16:46:22 +0200 Subject: [PATCH 54/93] Remove unused imports and commented code --- src/lib/EEG_functions.py | 257 ++----------------------------- src/lib/Resp_features.py | 320 ++++++--------------------------------- 2 files changed, 62 insertions(+), 515 deletions(-) diff --git a/src/lib/EEG_functions.py b/src/lib/EEG_functions.py index 8c7137c..d6e06ea 100644 --- a/src/lib/EEG_functions.py +++ b/src/lib/EEG_functions.py @@ -1,22 +1,11 @@ +"""EEG feature helpers used by the active submission pipeline.""" + from scipy.signal import butter, filtfilt import numpy as np from scipy.signal import welch import pandas as pd -from scipy import signal from scipy.stats import kurtosis, entropy -# try: -# import plotly.graph_objects as go -# from plotly.subplots import make_subplots -# except ModuleNotFoundError: -# go = None -# make_subplots = None - -# try: -# import matplotlib.pyplot as plt -# except ModuleNotFoundError: -# plt = None - def _safe_sqrt_variance_ratio(numerator_signal, denominator_signal): numerator_var = np.var(numerator_signal) denominator_var = np.var(denominator_signal) @@ -28,213 +17,24 @@ def _safe_sqrt_variance_ratio(numerator_signal, denominator_signal): return float(np.sqrt(ratio)) def butter_bandpass_filter(data, lowcut, highcut, fs, order=4): - nyq = 0.5 * fs # Frecuencia de Nyquist + nyq = 0.5 * fs low = lowcut / nyq high = highcut / nyq b, a = butter(order, [low, high], btype='bandpass') - # Usamos filtfilt para que no haya desfase en la señal - # w, h = signal.freqz(b, a, worN=8000) - # frequencies = (w * fs) / (2 * np.pi) - # plt.figure(figsize=(10, 5)) - # plt.plot(frequencies, 20 * np.log10(abs(h))) - # plt.xlim(0, highcut + 20) - # plt.ylim(-40, 5) # Para ver bien la caída - # plt.title('Respuesta Frecuencial Digital (Bandpass)') - # plt.xlabel('Frecuencia [Hz]') - # plt.ylabel('Amplitud [dB]') - # plt.grid(which='both', axis='both') - # plt.axvline(lowcut, color='red', linestyle='--', label='Lowcut') - # plt.axvline(highcut, color='red', linestyle='--', label='Highcut') - # plt.legend() - # plt.show() y = filtfilt(b, a, data) return y -# def plot_EEG(df, columns, fs = 200): -# if go is None or make_subplots is None: -# raise ModuleNotFoundError("plotly is required for plot_EEG") - -# fig = make_subplots(rows=len(columns), cols=1, -# shared_xaxes=True, -# vertical_spacing=0.02, -# subplot_titles=columns) -# limit = int(3000 * fs) -# x = np.arange(df[0].shape[0]) / fs # Asumiendo fs=100Hz, ajusta si es diferente -# downsample = 10 # Factor de downsampling para mejorar rendimiento (ajusta según necesidad) -# for i, col in enumerate(columns): -# fig.add_trace( -# go.Scattergl(x=x[:limit:downsample], y=df[i][:limit:downsample], name=col, mode='lines'), -# row=i+1, col=1 -# ) -# fig.update_layout( -# height=900, -# title_text="Polisomnografía - Canales EEG", -# showlegend=False, -# template="plotly_white" -# ) -# fig.update_xaxes(title_text="Tiempo (segundos)", row=len(columns), col=1) -# fig.show() - -# def plot_EEG_sel(sel, name = "EEG_plot_raw.html"): -# if go is None or make_subplots is None: -# raise ModuleNotFoundError("plotly is required for plot_EEG_sel") - -# fig = make_subplots(rows=len(sel), cols=1, -# shared_xaxes=True, -# vertical_spacing=0.02, -# subplot_titles=[ch[1].label for ch in sel]) - -# for i, (idx, sig) in enumerate(sel): -# # Crear eje de tiempo en segundos -# fs = sig.sampling_frequency -# time = np.linspace(0, len(sig.data) / fs, len(sig.data)) - -# # Añadir traza (solo mostramos los primeros 30s por defecto para no saturar el navegador) -# # Puedes quitar el slice [:int(30*fs)] para ver todo, pero cuidado con el rendimiento -# limit = int(3000 * fs) -# # limit = len(sig.data) if limit > len(sig.data) else limit -# # limit = len(sig.data) -# # downsample = 10 # Factor de downsampling para mejorar rendimiento (ajusta según necesidad) -# fig.add_trace( -# go.Scattergl(x=time[:limit], y=sig.data[:limit], name=sig.label, mode='lines'), -# row=i+1, col=1 -# ) - -# fig.update_layout( -# height=900, -# title_text="Polisomnografía - Canales EEG", -# showlegend=False, -# template="plotly_white" -# ) - -# fig.update_xaxes(title_text="Tiempo (segundos)", row=len(sel), col=1) -# fig.write_html(f"graphs/{name}.html") # Guardar como HTML para visualización interactiva -# # fig.show() - -def filtering_and_normalization(sig, sig_fs): - b, a = signal.butter(4, 0.3, btype='highpass', fs=sig_fs) - sig_filtered = signal.filtfilt(b, a, sig) - b, a = signal.butter(4, 35, btype='lowpass', fs=sig_fs) - sig_filtered = signal.filtfilt(b, a, sig_filtered) - sig_filtered = normalize(sig_filtered) - return sig_filtered - -def normalize(x): - return (x - np.mean(x)) / np.std(x) - -def remove_impulse_artifacts(sig): - # Square of second derivative - aux = np.diff(np.diff(sig)) ** 2 - aux = np.insert(aux, 0, aux[0]) - aux = np.append(aux, aux[-1]) - - # Median filter threshold - wind = 999 - if aux.size < wind: - wind = aux.size - if (wind % 2) != 1: - wind = wind - 1 - mf = signal.medfilt(aux, wind) - - # Find impulses - margin = 20 - impulses = np.asarray(np.where(aux > mf + 0.005)).ravel() - for impulse in impulses: - impulses = np.append(impulses, np.arange(impulse - margin, min(impulse + margin+1, sig.size))) - impulses = np.sort(impulses) - impulses = np.unique(impulses) - impulses = impulses[impulses >= 0] - - # Remove impulses - output = sig - output[impulses] = np.nan - return output - -def clean_movement_artifacts(data, fs, threshold_z=10, window_ms=500): - """ - Identifica y limpia artefactos de gran amplitud. - - Args: - data: Array de la señal. - fs: Frecuencia de muestreo. - threshold_z: Umbral de desviaciones estándar para marcar como artefacto. - window_ms: Tiempo alrededor del artefacto a limpiar para asegurar - que eliminamos la subida y bajada del pico. - """ - cleaned_data = data.copy() - - # 1. Calcular Z-Score de la amplitud - z_scores = np.abs((data - np.mean(data)) / np.std(data)) - - # 2. Encontrar índices que superan el umbral - mask = z_scores > threshold_z - - # 3. Expandir la máscara (el movimiento suele durar un poco más que el pico) - padding = int((window_ms / 1000) * fs) - expanded_mask = np.convolve(mask, np.ones(padding), mode='same') > 0 - - # 4. Reemplazar artefactos con el valor medio (0 si está centrada) - cleaned_data[expanded_mask] = 0 - - artifacts_percentage = (np.sum(expanded_mask) / len(data)) * 100 - print(f"Artefactos eliminados: {artifacts_percentage:.2f}% de la señal.") - - return cleaned_data - -def adaptive_variance_cleaner(data, fs, win_size_ms=500, alpha=0.1, threshold=3.5): - """ - Filtro adaptativo que detecta artefactos cuando la varianza local - excede significativamente la varianza histórica adaptativa. - - Args: - data: Array de la señal (1D). - fs: Frecuencia de muestreo. - win_size_ms: Tamaño de la ventana para calcular la varianza local. - alpha: Factor de adaptación (0 a 1). Cuanto más alto, más rápido olvida el pasado. - threshold: Multiplicador de la varianza adaptativa para marcar artefacto. - """ - win_samples = int((win_size_ms / 1000) * fs) - n_samples = len(data) - cleaned_data = np.copy(data) - - # Inicializamos la varianza adaptativa con la varianza de la primera ventana - first_win = data[:win_samples] - adaptive_var = np.var(first_win) - - # Para guardar dónde detectamos artefactos - artifact_mask = np.zeros(n_samples, dtype=bool) - - # Iteramos por ventanas - for i in range(0, n_samples - win_samples, win_samples): - current_win_idx = slice(i, i + win_samples) - current_var = np.var(data[current_win_idx]) - - # Si la varianza actual es mucho mayor que la adaptativa, es un artefacto - if current_var > threshold * adaptive_var: - artifact_mask[current_win_idx] = True - cleaned_data[current_win_idx] = 0 # O podrías interpolar - # No actualizamos la varianza adaptativa con un artefacto para no "contaminarla" - else: - # Actualización adaptativa (Exponential Moving Average) - adaptive_var = alpha * current_var + (1 - alpha) * adaptive_var - - return cleaned_data, artifact_mask - def create_epochs(data, fs, epoch_duration=30): samples_per_epoch = int(fs * epoch_duration) num_epochs = len(data) // samples_per_epoch - - # Recortamos la señal para que sea divisible exactamente + data_trimmed = data[:num_epochs * samples_per_epoch] - - # Reshape: (Número de épocas, Puntos por época) epochs = data_trimmed.reshape(num_epochs, samples_per_epoch) return epochs def extract_band_powers(epochs, fs, win_len = 2): features = [] complexities = [] - # Definición de las bandas bands = { 'Delta': (0.5, 4), 'Theta': (4, 8), @@ -242,25 +42,18 @@ def extract_band_powers(epochs, fs, win_len = 2): 'Sigma': (11, 16), 'Beta': (12, 30) } - + for epoch in epochs: - # Calcular PSD - freqs, psd = welch(epoch, fs, nperseg=fs*30) # Ventanas de 2 seg para buena resolución - # Plot de PSD para verificar que las bandas se ven bien (opcional) - # plt.semilogy(freqs, psd) - # plt.show() + freqs, psd = welch(epoch, fs, nperseg=fs*30) epoch_features = {} for band_name, (low, high) in bands.items(): - # Encontrar índices de frecuencia para la banda actual idx_band = np.logical_and(freqs >= low, freqs <= high) - # Calcular la potencia media en esa banda epoch_features[band_name] = np.mean(psd[idx_band]) - + features.append(epoch_features) diff = np.diff(epoch) mobility = _safe_sqrt_variance_ratio(diff, epoch) - # 2. Complejidad de Hjorth: Qué tan similar es la señal a una onda senoidal diff2 = np.diff(diff) mobility_diff = _safe_sqrt_variance_ratio(diff2, diff) complexity = mobility_diff / mobility if mobility > 0 else 0 @@ -269,45 +62,26 @@ def extract_band_powers(epochs, fs, win_len = 2): return pd.DataFrame(features), pd.DataFrame(complexities) def get_patient_profile(df_features): - # 1. Calcular Potencia Total por época total_power = df_features.sum(axis=1) avg_p = df_features.mean() total_avg_p = avg_p.sum() - - # 2. Variabilidad (Refleja microdespertares y fragmentación) - # Coeficiente de Variación (CV = std/mean) para normalizar por amplitud + variability = df_features.std() / df_features.mean() variability.index = ['CV_' + col for col in variability.index] - - # 3. Curtosis (Picos súbitos de actividad) + kurt = df_features.apply(kurtosis) kurt.index = ['Kurt_' + col for col in kurt.index] - - # 4. Índices de potencia relativa específicos + rel_delta = avg_p['Delta'] / total_avg_p - - # 5. Ratios de enlentecimiento - tar = avg_p['Theta'] / avg_p['Alpha'] # Theta-Alpha Ratio - tbr = avg_p['Theta'] / avg_p['Beta'] # Theta-Beta Ratio - - # 6. Entropía Espectral (Complejidad del perfil de potencia promedio) - # Cuanto más baja, más "pobre" es la diversidad de frecuencias del cerebro + + tar = avg_p['Theta'] / avg_p['Alpha'] + tbr = avg_p['Theta'] / avg_p['Beta'] + spec_entropy = entropy(df_features) - # 2. Calcular Potencias Relativas (promedio de toda la noche) rel_powers = df_features.div(total_power, axis=0).mean() rel_powers.index = ['Rel_' + col for col in rel_powers.index] - - # Calculate main frecuencies of oscilation on each band (peak frequency) - # Esto puede ser un buen indicador de cambios en la arquitectura del sueño - # peak_freqs = {} - # for band in ['Delta', 'Theta', 'Alpha', 'Sigma', 'Beta']: - # freqs, psd = welch(df_features[band], fs=1/30, nperseg=25, noverlap = 25 // 2, nfft=1024) # fs=1/30 porque cada punto es un promedio de 30s - # idx_peak = np.argmax(psd) - # peak_freqs['PeakFreq_' + band] = freqs[idx_peak] - # 3. Calcular Ratios Críticos - # Usamos la media de las potencias absolutas para el ratio global avg_p = df_features.mean() ratios = { 'Ratio_Theta_Alpha': avg_p['Theta'] / avg_p['Alpha'], @@ -333,7 +107,6 @@ def get_patient_profile(df_features): 'variability_Beta': variability['CV_Beta'], } - - # Combinar todo en una sola fila + profile = pd.concat([rel_powers, pd.Series(ratios)]) return profile \ No newline at end of file diff --git a/src/lib/Resp_features.py b/src/lib/Resp_features.py index 60ff3bf..45b0666 100644 --- a/src/lib/Resp_features.py +++ b/src/lib/Resp_features.py @@ -1,145 +1,87 @@ -import pandas as pd -import numpy as np -from .peakedness import peakednessCost -from scipy.interpolate import interp1d -from scipy.stats import kruskal -from scipy.signal import resample, detrend -import scipy.fft as fft -from scipy.signal import butter, filtfilt - -try: - import plotly.graph_objs as go -except ModuleNotFoundError: - go = None - -try: - import matplotlib.pyplot as plt -except ModuleNotFoundError: - plt = None - -def plot_resp(Data, subjet = 1, DownPrinting = 2): - """ - Plot resp data using Plotly. - """ - if go is None: - raise ModuleNotFoundError("plotly is required for plot_resp") +"""Respiratory feature helpers used by the active submission pipeline.""" - if type(Data) == dict: - Data = pd.DataFrame(Data[str(subjet)]) - Data = Data.iloc[::DownPrinting, :] - end = -1 - elif type(Data) == type(pd.DataFrame()): - Data = Data[Data['Subjet'] == str(subjet)] - Data = Data.iloc[::DownPrinting, :] - end = -2 +import numpy as np +import pandas as pd - # Data.reset_index(drop=True, inplace=True) - print(len(Data.columns)) - fig = go.Figure() - for c in Data.columns[:end]: - fig.add_trace(go.Line(x=Data.Time, y=Data[c], name = c)) - fig.update_layout(title_text='EDA Data', title_x=0.5) +from .peakedness import peakednessCost - fig.show() -def peakedness_application(Data, stage, plotflag = False, subjet = 1): - # print("Compute BR") +def peakedness_application(data, stage, plotflag=False, subjet=1): fs = 25 - Setup = {} - Setup["K"] = 5 - Setup["DT"] = 5 - Setup["Ts"] = 60 #interval length of Welch periodograms (s) - Setup["Tm"] = 20 #interval length of subintervals for Welch periodograms (s) - # Setup["d"] = 0.1 #interval length of subintervals for Welch periodograms (s) - Setup["Omega_r"] = np.array([5, 25])/60 #respiratory rate range in Hz - Setup["plotflag"] = plotflag - Setup["Nfft"] = np.power(2,13) - tsBR = np.arange(0,Data.shape[0]/fs,1/fs) + setup = { + "K": 5, + "DT": 5, + "Ts": 60, + "Tm": 20, + "Omega_r": np.array([5, 25]) / 60, + "plotflag": plotflag, + "Nfft": np.power(2, 13), + } + time_axis = np.arange(0, data.shape[0] / fs, 1 / fs) + + if time_axis.shape[0] != data.shape[0]: + time_axis = np.arange(0, data.shape[0] / fs, 1 / fs)[:data.shape[0]] + + hat_br, sk_br, t_aver, used = peakednessCost( + data, + time_axis, + fs, + setup, + title=stage, + storeGraph=False, + subjet=subjet, + ) + return hat_br, sk_br, t_aver, used - if tsBR.shape[0] != Data.shape[0]: - # print(f"tsBR.shape[0]: {tsBR.shape[0]}, Data.shape[0]: {Data.shape[0]}") - tsBR = np.arange(0,Data.shape[0]/fs,1/fs)[:Data.shape[0]] - - hat_Br, Sk_Br, t_aver, used = peakednessCost(Data, tsBR, fs, Setup, title = stage, storeGraph = False, subjet = subjet) - # print(f"hat_Br: {hat_Br}, Sk_Br: {Sk_Br}, bar_Br: {bar_Br}, t_aver_Br: {t_aver_Br}, f_Br: {f_Br}, used_Br: {used_Br}") - - # print(hat_Br) - return hat_Br, Sk_Br, t_aver, used def ODI_application(data, fs, plotflag=True, subjet=1): - """Detecta desaturaciones de más del 3 % en la señal de saturación de - oxígeno (SpO2) y devuelve estadísticas básicas de los eventos. - - El índice de desaturación de oxígeno (ODI) se define como el número de - episodios en los que la saturación cae al menos un 3 % respecto a una - línea de base móvil, normalizado por hora de grabación. Aquí se calcula - una línea base mediante la mediana móvil de 60 segundos y se agrupan - los índices consecutivos que cumplen el criterio en eventos únicos. - - Args: - data (array-like): valores de SpO2 (0‑100). - fs (float): frecuencia de muestreo en Hz. - plotflag (bool): si True, dibuja la señal y marca los eventos. - subjet (int): identificador de sujeto (utilizado en títulos de gráficas). - - Returns: - tuple: - * odi_mean (float): número de desaturaciones normalizado por hora. - * odi_std (float): desviación estándar de las magnitudes de caída - entre eventos (en porcentaje). - """ - # convertir a serie para comodidad + """Compute oxygen desaturation event rate and average event depth.""" sp = pd.Series(data) if len(sp) == 0 or fs <= 0: return 0.0, 0.0 - # base móvil de 60 segundos (median para ser robusto). ventana en muestras - window = int(fs * 60) - if window < 1: - window = 1 + window = max(1, int(fs * 60)) baseline = sp.rolling(window, min_periods=1, center=True).median() - # diferencia de base menos señal; buscamos caídas >=3 diff = baseline - sp mask = diff >= 3 - # juntar índices contiguos en eventos - events = [] # lista de (start_idx, end_idx) + events = [] in_event = False + prev_idx = 0 for idx, flag in mask.items(): if flag and not in_event: start = idx in_event = True elif not flag and in_event: - end = prev_idx - events.append((start, end)) + events.append((start, prev_idx)) in_event = False prev_idx = idx if in_event: events.append((start, prev_idx)) - num_events = len(events) duration_hours = len(sp) / fs / 3600.0 - odi_mean = num_events / duration_hours if duration_hours > 0 else 0.0 + odi_mean = len(events) / duration_hours if duration_hours > 0 else 0.0 - # calcular magnitudes de caída en cada evento (tomando el valor más bajo) magnitudes = [] for start, end in events: - mag = diff.loc[start:end].max() - magnitudes.append(mag) + magnitudes.append(diff.loc[start:end].max()) odi_deepness = np.mean(magnitudes) if magnitudes else 0.0 if plotflag: - if plt is None: - raise ModuleNotFoundError("matplotlib is required when plotflag=True") - times = np.arange(len(sp)) / fs / 60.0 # minutos + try: + from importlib import import_module + + plt = import_module('matplotlib.pyplot') + except ModuleNotFoundError as exc: + raise ModuleNotFoundError("matplotlib is required when plotflag=True") from exc + + times = np.arange(len(sp)) / fs / 60.0 plt.figure(figsize=(10, 4)) plt.plot(times, sp.values, label='SpO2') plt.plot(times, baseline.values, label='Baseline (60s med)') - for (start, end) in events: - t0 = start / fs / 60.0 - t1 = end / fs / 60.0 - plt.axvspan(t0, t1, color='red', alpha=0.3) + for start, end in events: + plt.axvspan(start / fs / 60.0, end / fs / 60.0, color='red', alpha=0.3) plt.xlabel('Tiempo (min)') plt.ylabel('SpO2 (%)') plt.title(f'Sujeto {subjet} - ODI detectado: {odi_mean:.2f} eventos/h') @@ -147,172 +89,4 @@ def ODI_application(data, fs, plotflag=True, subjet=1): plt.tight_layout() plt.show() - return odi_mean, odi_deepness - -# Butterworth low-pass filter -def lowpass_filter(signal, fs, cutoff=2.0, order=4): - nyq = 0.5 * fs - normal_cutoff = cutoff / nyq - b, a = butter(order, normal_cutoff, btype='low', analog=False) - return filtfilt(b, a, signal) - -def Metrics_per_segment(Data): - """ - Compute peakedness per segment. - """ - - # Results = pd.DataFrame(columns=['Subject', 'Stage', 'Peakedness', 'Slope', 'Intercept', 'Relative Peak', 'Bocanada', 'Contraction', "TidalVolume", "Complexity", "Mobility", "Activity"]) - Results = pd.DataFrame() - - for subjet in Data['Subjet'].unique(): - sel_sujeto = Data[Data['Subjet'] == subjet] - sel_sujeto_ref = sel_sujeto.iloc[:,:-2] - Sol_subject = [] - Sol_interSubject = [] - for secc in sel_sujeto_ref.columns: - if secc == 'Time': - continue - else: - section = sel_sujeto[secc].values - section = section[~np.isnan(section)] - - hat_Br, Sk_Br, t_aver, _ = peakedness_application(section, stage=secc, plotflag = False, subjet= subjet) - # print(f"Subjet: {subjet}, section: {secc} hat_Br: {hat_Br}, Sk_Br: {Sk_Br}") - - # Ajuste lineal - coef = np.polyfit(t_aver, hat_Br, 1) # Grado 1 = línea recta - pendiente, interseccion = coef - # print(f"Pendiente: {pendiente:.6f}, Intersección: {interseccion:.6f}") - - #Picudez relativa - rel_peak_list = [] - for ti in range(len(t_aver)): - f_max = np.argmax(Sk_Br[:,ti]) - rel_peak_list.append(np.sum(Sk_Br[f_max-1:f_max+1,ti]) / np.sum(Sk_Br[:,ti])) - real_peak = np.mean(rel_peak_list) - - # Derivada - diff = np.diff(section) - bocanada = max(np.percentile(diff, 90), np.abs(np.percentile(diff, 10))) - - Contraction = np.percentile(np.abs(diff), 10) - - #Tidal Volume - TidalVolume = max(np.percentile(section, 99), np.abs(np.percentile(section, 1))) - - # Calculate derivatives - dx = np.diff(section) - ddx = np.diff(dx) - - # Calculate variance and its derivatives - x_var = np.var(section) # = activity - dx_var = np.var(dx) - ddx_var = np.var(ddx) - - # Mobility and complexity - mobility = np.sqrt(dx_var / x_var) - complexity = np.sqrt(ddx_var / dx_var) / mobility - - - filtered_signal = lowpass_filter(section, 100, cutoff=2.0, order=4) - segment4Hz = resample(filtered_signal, int(filtered_signal.size/100*4)) # Resample to 4Hz - - fft_signal = fft.fft(detrend(segment4Hz), n=2**12) - power = np.abs(fft_signal)**2 - freqs = fft.fftfreq(2**12, d = 1/4) - max_freq_index = np.argmax(power) - max_freq = freqs[max_freq_index] - power_at_max_freq = power[max_freq_index-51:max_freq_index+51].sum() - power_ratio = power_at_max_freq / np.sum(power[:len(power)//2]) - - Sol = [subjet, secc[:secc.find('_')], np.mean(hat_Br), pendiente, interseccion, real_peak, bocanada, Contraction, TidalVolume, complexity, mobility, x_var, max_freq, power_ratio] - - Sol_subject.append(Sol) - - - Sol_subject = pd.DataFrame(Sol_subject, columns=['Subject', 'Stage', 'Peakedness', 'Slope', - 'Intercept', 'Relative Peak', 'Bocanada', - "Contraction","TidalVolume", "Complexity", - "Mobility", "Activity", "Max_freq", "Power_ratio"]) - - peakmean = Sol_subject['Peakedness'].mean() - peakmin = Sol_subject['Peakedness'].min() - peakmax = Sol_subject['Peakedness'].max() - - slopemean = Sol_subject['Slope'].mean() - slopemin = Sol_subject['Slope'].min() - slopemax = Sol_subject['Slope'].max() - - Rel_peak_mean = Sol_subject['Relative Peak'].mean() - - Bocanada_max = Sol_subject['Bocanada'].max() - Contraction_max = Sol_subject['Contraction'].max() - TidalVolume_max = Sol_subject['TidalVolume'].max() - - Rel_metrics = ['Subject', 'Stage',"Peakmean", "Peakmin", "Peakmax", "Slopemean", "Slopemin", "Slopemax", "Rel_peak_mean", "Bocanada_max", "Contraction_max", "TidalVolume_max"] - # Rel_metrics = ["Peakmean", "Peakmin", "Peakmax", "Slopemean", "Slopemin", "Slopemax", "Rel_peak_mean", "Bocanada_max", "Contraction_max", "TidalVolume_max"] - - Sol_interSubject_DF = pd.DataFrame(Sol_interSubject, columns=Rel_metrics) - Sol_interSubject_DF = pd.DataFrame(Sol_interSubject, columns=Rel_metrics[2:]) - for i in Sol_subject.index: - # Sol_interSubject_DF.at[i,Rel_metrics[0]] = Sol_subject.iloc[i,0] - # Sol_interSubject_DF.at[i,Rel_metrics[1]] = Sol_subject.at[i,'Stage'] - Sol_interSubject_DF.at[i,Rel_metrics[2]] = Sol_subject.at[i,'Peakedness']/peakmean - Sol_interSubject_DF.at[i,Rel_metrics[3]] = Sol_subject.at[i,'Peakedness']/peakmin - Sol_interSubject_DF.at[i,Rel_metrics[4]] = Sol_subject.at[i,'Peakedness']/peakmax - Sol_interSubject_DF.at[i,Rel_metrics[5]] = Sol_subject.at[i,'Slope']/slopemean - Sol_interSubject_DF.at[i,Rel_metrics[6]] = Sol_subject.at[i,'Slope']/slopemin - Sol_interSubject_DF.at[i,Rel_metrics[7]] = Sol_subject.at[i,'Slope']/slopemax - Sol_interSubject_DF.at[i,Rel_metrics[8]] = Sol_subject.at[i,'Relative Peak']/Rel_peak_mean - Sol_interSubject_DF.at[i,Rel_metrics[9]] = Sol_subject.at[i,'Bocanada']/Bocanada_max - Sol_interSubject_DF.at[i,Rel_metrics[10]] = Sol_subject.at[i,"Contraction"]/Contraction_max - Sol_interSubject_DF.at[i,Rel_metrics[11]] = Sol_subject.at[i,"TidalVolume"]/TidalVolume_max - - Sol = pd.concat([Sol_subject, Sol_interSubject_DF], axis=1) - Results = pd.concat([Results, Sol], ignore_index=True) - - - - return Results - -def Significance_tests(RespData): - """ - Compute significance tests for the features. - """ - results = {} - for metrica in RespData.columns[2:]: - # print(f"Realizando prueba de Kruskal-Wallis para la métrica: {metrica}") - # Realizar la prueba de Kruskal-Wallis - estadistico, p_valor = kruskal( - np.array(RespData[RespData.Stage == "Baseline"][metrica].reset_index(drop=True)), - np.array(RespData[RespData.Stage == "LOW"][metrica].reset_index(drop=True)), - np.array(RespData[RespData.Stage == "HIGH"][metrica].reset_index(drop=True)), - np.array(RespData[RespData.Stage == "REST"][metrica].reset_index(drop=True)) - ) - - # Imprimir resultados - - # print(f"Estadístico de Kruskal-Wallis: {estadistico}") - print(f"Metrica: "+metrica+" tiene un valor p: {p_valor}") - results[metrica] = p_valor - # if p_valor < 0.05: - # print("Se rechaza la hipótesis nula: hay diferencias significativas entre los grupos.") - # else: - # print("No se rechaza la hipótesis nula: no hay diferencias significativas entre los grupos.") - - results = pd.DataFrame.from_dict(results, orient='index', columns=['p_value']) - results = results.reset_index() - results.to_excel('./Graphs/kruskal_results.xlsx', index=False) - - if plt is None: - raise ModuleNotFoundError("matplotlib is required for Significance_tests plotting") - - plt.plot(results['index'], results['p_value']) - plt.axhline(y=0.05, color='r', linestyle='--') - plt.xlabel('Métrica') - plt.ylabel('Valor p') - plt.title('Resultados de la prueba de Kruskal-Wallis') - plt.xticks(rotation=90) - plt.tight_layout() - plt.savefig('./Graphs/kruskal_results.png') - plt.show() \ No newline at end of file + return odi_mean, odi_deepness \ No newline at end of file From 16c600ceb68d336c97994b3b6cb347bf89feaa61 Mon Sep 17 00:00:00 2001 From: dcajal Date: Mon, 30 Mar 2026 17:20:56 +0200 Subject: [PATCH 55/93] Refactor and renaming --- src/ecg_processing.py | 4 +- src/eeg_processing.py | 10 +- src/lib/ECG_processing.py | 143 ---------------- src/lib/Resp_features.py | 30 +--- src/lib/compute_hrv_hrf.py | 155 ------------------ src/lib/ecg_features.py | 108 ++++++++++++ src/lib/ecg_hrv_features.py | 123 ++++++++++++++ src/lib/ecg_nn_interpolation.py | 34 ++++ ...{pan_tompkins.py => ecg_peak_detection.py} | 2 +- src/lib/ecg_rr_cleaning.py | 38 +++++ src/lib/{EEG_functions.py => eeg_features.py} | 0 src/lib/interpolate_NN.py | 40 ----- src/lib/remove_ectopic_beat.py | 41 ----- src/lib/{peakedness.py => resp_peakedness.py} | 33 +--- src/resp_processing.py | 9 +- 15 files changed, 324 insertions(+), 446 deletions(-) delete mode 100644 src/lib/ECG_processing.py delete mode 100644 src/lib/compute_hrv_hrf.py create mode 100644 src/lib/ecg_features.py create mode 100644 src/lib/ecg_hrv_features.py create mode 100644 src/lib/ecg_nn_interpolation.py rename src/lib/{pan_tompkins.py => ecg_peak_detection.py} (99%) create mode 100644 src/lib/ecg_rr_cleaning.py rename src/lib/{EEG_functions.py => eeg_features.py} (100%) delete mode 100644 src/lib/interpolate_NN.py delete mode 100644 src/lib/remove_ectopic_beat.py rename src/lib/{peakedness.py => resp_peakedness.py} (96%) diff --git a/src/ecg_processing.py b/src/ecg_processing.py index 134a3e7..e4d9a96 100644 --- a/src/ecg_processing.py +++ b/src/ecg_processing.py @@ -1,4 +1,4 @@ -from .lib.ECG_processing import ECGprocessing +from .lib.ecg_features import compute_ecg_features import numpy as np from src.common.channel_utils import normalize_channel_label @@ -50,7 +50,7 @@ def processECG(physiological_data, physiological_fs, csv_path): return results try: - values = ECGprocessing(ecg_signal, fs, ECG_FEATURE_LENGTH) + values = compute_ecg_features(ecg_signal, fs, ECG_FEATURE_LENGTH) if values is None or len(values) == 0: return results diff --git a/src/eeg_processing.py b/src/eeg_processing.py index 112855b..ab4d00d 100644 --- a/src/eeg_processing.py +++ b/src/eeg_processing.py @@ -2,7 +2,7 @@ import numpy as np from src.common.channel_utils import find_matching_label, get_cached_channel_table, normalize_channel_label, split_channel_aliases from src.common.signal_utils import resample_signal -from .lib import EEG_functions +from .lib import eeg_features EEG_CHANNEL_SPECS = { 'C3-M2': {'direct': 'c3-m2', 'positive': 'c3', 'reference': 'm2'}, @@ -122,24 +122,24 @@ def _extract_channel_metrics(signal, fs): if fs != 200: signal, fs = resample_signal(signal, fs, 200) - filtered = EEG_functions.butter_bandpass_filter(signal, lowcut=0.3, highcut=35, fs=fs, order=4) + filtered = eeg_features.butter_bandpass_filter(signal, lowcut=0.3, highcut=35, fs=fs, order=4) signal_std = np.std(filtered) if signal_std == 0 or not np.isfinite(signal_std): return None normalized = (filtered - np.mean(filtered)) / signal_std - epochs = EEG_functions.create_epochs(normalized, fs, epoch_duration=30) + epochs = eeg_features.create_epochs(normalized, fs, epoch_duration=30) if epochs.size == 0: return None - band_powers, complexities = EEG_functions.extract_band_powers(epochs, fs, win_len=15) + band_powers, complexities = eeg_features.extract_band_powers(epochs, fs, win_len=15) if len(band_powers) > 60: band_powers = band_powers.iloc[60:] complexities = complexities.iloc[60:] if band_powers.empty: return None - patient_profile = EEG_functions.get_patient_profile(band_powers) + patient_profile = eeg_features.get_patient_profile(band_powers) metrics = { str(name): float(value) for name, value in patient_profile.replace([np.inf, -np.inf], np.nan).fillna(0.0).items() diff --git a/src/lib/ECG_processing.py b/src/lib/ECG_processing.py deleted file mode 100644 index 37e2013..0000000 --- a/src/lib/ECG_processing.py +++ /dev/null @@ -1,143 +0,0 @@ -import numpy as np -import pandas as pd -from scipy.signal import butter, filtfilt, resample -from .pan_tompkins import pan_tompkin -from .compute_hrv_hrf import compute_HRV_HRF -from .interpolate_NN import interpolate_NN_pchip -from .remove_ectopic_beat import remove_ectopic_beats - - -def ECGprocessing(ecg_signal, fs, ECG_FEATURE_LENGTH): - - ecg_signal = ecg_signal - np.mean(ecg_signal) - - # =============================== - # RESAMPLE TO 200 Hz IF NEEDED - # =============================== - target_fs = 200 - - if fs != target_fs: - num_samples = int(len(ecg_signal) * target_fs / fs) - ecg_signal = resample(ecg_signal, num_samples) - fs = target_fs - - # =============================== - # SEGMENT INTO 5-MIN WINDOWS - # =============================== - win_sec = 300 - win_samples = int(win_sec * fs) - - N = len(ecg_signal) - n_windows = N // win_samples - - if n_windows == 0: - print("Signal too short.") - return None - - # =============================== - # FIND VALID WINDOWS - # =============================== - valid_windows = [] - - for w in range(n_windows): - - idx_start = w * win_samples - idx_end = (w + 1) * win_samples - - ecg_win = ecg_signal[idx_start:idx_end] - - # Quality check - if np.sum(np.isnan(ecg_win)) != 0 or np.sum(ecg_win == 0) > 0.2 * len(ecg_win): - continue - - valid_windows.append(w) - - # =============================== - # PROCESS WINDOWS - # =============================== - HRV_all = [] - - for w in valid_windows: - - idx_start = w * win_samples - idx_end = (w + 1) * win_samples - - ecg_win = ecg_signal[idx_start:idx_end] - - ecg_win = ecg_win - np.mean(ecg_win) - - # --- Filtering --- - # Notch - b, a = butter(3, [59.5/(fs/2), 60.5/(fs/2)], btype='bandstop') - ecg_win = filtfilt(b, a, ecg_win) - - # High-pass - b, a = butter(3, 0.5/(fs/2), btype='high') - ecg_win = filtfilt(b, a, ecg_win) - - # Low-pass - b, a = butter(3, 45/(fs/2), btype='low') - ecg_win = filtfilt(b, a, ecg_win) - - # =============================== - # QRS DETECTION - # =============================== - qrs_amp_raw, R_locs, delay = pan_tompkin(ecg_win, fs, 0) - - if len(R_locs) < 150: - continue - - # =============================== - # NN INTERVALS - # =============================== - NN = np.diff(R_locs) / fs - - # =============================== - # HRV PREPROCESSING - # =============================== - NN, ectopic_perc = remove_ectopic_beats(NN, 40, 0.10) - - NN = interpolate_NN_pchip(NN, 2) - - valid_ratio = np.sum(~np.isnan(NN)) / len(NN) - NN = NN[~np.isnan(NN)] - - if valid_ratio < 0.75: - continue - - # =============================== - # HRV + HRF METRICS - # =============================== - res = compute_HRV_HRF(NN, fs) - - HRV_all.append([ - res["PIP"], res["PNNLS"], res["PNNSS"], - res["AVNN"], res["SDNN"], res["RMSSD"], res["HF"], - ectopic_perc - ]) -# =============================== -# SUBJECT-LEVEL METRICS -# =============================== - if len(HRV_all) == 0: - return np.zeros(ECG_FEATURE_LENGTH, dtype=np.float32) - - HRV_all = np.array(HRV_all) - - median_vals = np.nanmedian(HRV_all, axis=0) - std_vals = np.nanstd(HRV_all, axis=0) - - # =============================== - # BUILD FIXED FEATURE VECTOR - # =============================== - features = np.array([ - median_vals[0], std_vals[0], # PIP - median_vals[1], std_vals[1], # PNNLS - median_vals[2], std_vals[2], # PNNSS - median_vals[3], std_vals[3], # AVNN - median_vals[4], std_vals[4], # SDNN - median_vals[5], std_vals[5], # RMSSD - median_vals[6], std_vals[6], # HF - median_vals[7], std_vals[7], # ECTOPIC - ], dtype=np.float32) - - return features \ No newline at end of file diff --git a/src/lib/Resp_features.py b/src/lib/Resp_features.py index 45b0666..3afcdbb 100644 --- a/src/lib/Resp_features.py +++ b/src/lib/Resp_features.py @@ -3,10 +3,10 @@ import numpy as np import pandas as pd -from .peakedness import peakednessCost +from .resp_peakedness import peakednessCost -def peakedness_application(data, stage, plotflag=False, subjet=1): +def peakedness_application(data, stage, subject_id=1): fs = 25 setup = { "K": 5, @@ -14,7 +14,6 @@ def peakedness_application(data, stage, plotflag=False, subjet=1): "Ts": 60, "Tm": 20, "Omega_r": np.array([5, 25]) / 60, - "plotflag": plotflag, "Nfft": np.power(2, 13), } time_axis = np.arange(0, data.shape[0] / fs, 1 / fs) @@ -29,12 +28,12 @@ def peakedness_application(data, stage, plotflag=False, subjet=1): setup, title=stage, storeGraph=False, - subjet=subjet, + subjet=subject_id, ) return hat_br, sk_br, t_aver, used -def ODI_application(data, fs, plotflag=True, subjet=1): +def odi_application(data, fs): """Compute oxygen desaturation event rate and average event depth.""" sp = pd.Series(data) if len(sp) == 0 or fs <= 0: @@ -68,25 +67,4 @@ def ODI_application(data, fs, plotflag=True, subjet=1): magnitudes.append(diff.loc[start:end].max()) odi_deepness = np.mean(magnitudes) if magnitudes else 0.0 - if plotflag: - try: - from importlib import import_module - - plt = import_module('matplotlib.pyplot') - except ModuleNotFoundError as exc: - raise ModuleNotFoundError("matplotlib is required when plotflag=True") from exc - - times = np.arange(len(sp)) / fs / 60.0 - plt.figure(figsize=(10, 4)) - plt.plot(times, sp.values, label='SpO2') - plt.plot(times, baseline.values, label='Baseline (60s med)') - for start, end in events: - plt.axvspan(start / fs / 60.0, end / fs / 60.0, color='red', alpha=0.3) - plt.xlabel('Tiempo (min)') - plt.ylabel('SpO2 (%)') - plt.title(f'Sujeto {subjet} - ODI detectado: {odi_mean:.2f} eventos/h') - plt.legend() - plt.tight_layout() - plt.show() - return odi_mean, odi_deepness \ No newline at end of file diff --git a/src/lib/compute_hrv_hrf.py b/src/lib/compute_hrv_hrf.py deleted file mode 100644 index 15b0422..0000000 --- a/src/lib/compute_hrv_hrf.py +++ /dev/null @@ -1,155 +0,0 @@ -import numpy as np -from scipy.signal import lombscargle - -def compute_HRV_HRF(NN, SF): - """ - NN: array of NN intervals in seconds - SF: sampling frequency (Hz) - """ - - NN = np.asarray(NN).flatten() - - # =============================== - # ΔNN - # =============================== - dNN = np.diff(NN) - - n = 1 - thr = n / SF - - # Classification - acc = dNN <= -thr - dec = dNN >= thr - noch = (dNN > -thr) & (dNN < thr) - - # Sign representation - sign_dNN = np.zeros_like(dNN) - sign_dNN[acc] = -1 - sign_dNN[dec] = 1 - - N = len(dNN) - - # =============================== - # PIP (Inflection Points) - # =============================== - inflection = 0 - - for i in range(N - 1): - if (dNN[i+1] * dNN[i] <= 0) and (dNN[i+1] != dNN[i]): - inflection += 1 - - PIP = (inflection / (N - 1)) * 100 if N > 1 else np.nan - - # =============================== - # Segment Detection - # =============================== - segments = [] - if N > 0: - current_seg = sign_dNN[0] - length_seg = 1 - - for i in range(1, N): - if sign_dNN[i] == current_seg and sign_dNN[i] != 0: - length_seg += 1 - else: - if current_seg != 0: - segments.append(length_seg) - current_seg = sign_dNN[i] - length_seg = 1 - - # Add last segment - if current_seg != 0: - segments.append(length_seg) - - segments = np.array(segments) - - # =============================== - # PNNLS & PNNSS - # =============================== - if len(segments) > 0: - long_segments = segments[segments >= 3] - short_segments = segments[segments < 3] - - PNNLS = np.sum(long_segments) / N * 100 - PNNSS = np.sum(short_segments) / np.sum(segments) * 100 - else: - PNNLS = np.nan - PNNSS = np.nan - - # =============================== - # Time-domain HRV - # =============================== - win_length = 300 # seconds - - time = np.cumsum(NN) - - AVNN_all = [] - SDNN_all = [] - RMSSD_all = [] - - i = 0 - while i < len(NN): - t_start = time[i] - t_end = t_start + win_length - - idx = np.where((time >= t_start) & (time < t_end))[0] - - if len(idx) >= 150: - NN_win = NN[idx] - - AVNN_all.append(np.nanmean(NN_win)) - SDNN_all.append(np.nanstd(NN_win, ddof=1)) - - diffNN = np.diff(NN_win) - RMSSD_all.append(np.sqrt(np.nanmean(diffNN**2))) - - next_i = np.where(time >= t_end)[0] - if len(next_i) == 0: - break - i = next_i[0] - - AVNN = np.nanmean(AVNN_all) if len(AVNN_all) > 0 else np.nan - SDNN = np.nanmean(SDNN_all) if len(SDNN_all) > 0 else np.nan - RMSSD = np.nanmean(RMSSD_all) if len(RMSSD_all) > 0 else np.nan - - # =============================== - # Frequency-domain (HF) - # =============================== - HF_all = [] - - for _ in range(len(AVNN_all)): - # NOTE: simplified like MATLAB version - NN_win = NN.copy() - t_win = np.cumsum(NN_win) - - # Convert to angular frequency - f = np.linspace(0.01, 0.5, 1000) - angular_f = 2 * np.pi * f - - # Remove mean (important for Lomb) - NN_detrended = NN_win - np.mean(NN_win) - - Pxx = lombscargle(t_win, NN_detrended, angular_f, normalize=True) - - HF_band = (f >= 0.15) & (f <= 0.4) - - HF_power = np.trapezoid(Pxx[HF_band], f[HF_band]) - - HF_all.append(HF_power) - - HF = np.nanmean(HF_all) if len(HF_all) > 0 else np.nan - - # =============================== - # OUTPUT - # =============================== - results = { - "PIP": PIP, - "PNNLS": PNNLS, - "PNNSS": PNNSS, - "AVNN": AVNN, - "SDNN": SDNN, - "RMSSD": RMSSD, - "HF": HF - } - - return results \ No newline at end of file diff --git a/src/lib/ecg_features.py b/src/lib/ecg_features.py new file mode 100644 index 0000000..eb964b1 --- /dev/null +++ b/src/lib/ecg_features.py @@ -0,0 +1,108 @@ +import numpy as np +from scipy.signal import butter, filtfilt, resample +from .ecg_peak_detection import pan_tompkins +from .ecg_hrv_features import compute_hrv_hrf +from .ecg_nn_interpolation import interpolate_nn_pchip +from .ecg_rr_cleaning import remove_ectopic_beats + + +def compute_ecg_features(ecg_signal, fs, ecg_feature_length): + + ecg_signal = ecg_signal - np.mean(ecg_signal) + + target_fs = 200 + + if fs != target_fs: + num_samples = int(len(ecg_signal) * target_fs / fs) + ecg_signal = resample(ecg_signal, num_samples) + fs = target_fs + + win_sec = 300 + win_samples = int(win_sec * fs) + + total_samples = len(ecg_signal) + n_windows = total_samples // win_samples + + if n_windows == 0: + print("Signal too short.") + return None + + valid_windows = [] + + for window_index in range(n_windows): + + idx_start = window_index * win_samples + idx_end = (window_index + 1) * win_samples + + ecg_win = ecg_signal[idx_start:idx_end] + + if np.sum(np.isnan(ecg_win)) != 0 or np.sum(ecg_win == 0) > 0.2 * len(ecg_win): + continue + + valid_windows.append(window_index) + + hrv_all = [] + + for window_index in valid_windows: + + idx_start = window_index * win_samples + idx_end = (window_index + 1) * win_samples + + ecg_win = ecg_signal[idx_start:idx_end] + + ecg_win = ecg_win - np.mean(ecg_win) + + b, a = butter(3, [59.5/(fs/2), 60.5/(fs/2)], btype='bandstop') + ecg_win = filtfilt(b, a, ecg_win) + + b, a = butter(3, 0.5/(fs/2), btype='high') + ecg_win = filtfilt(b, a, ecg_win) + + b, a = butter(3, 45/(fs/2), btype='low') + ecg_win = filtfilt(b, a, ecg_win) + + _, r_locs, _ = pan_tompkins(ecg_win, fs, 0) + + if len(r_locs) < 150: + continue + + nn_intervals = np.diff(r_locs) / fs + + nn_intervals, ectopic_perc = remove_ectopic_beats(nn_intervals, 40, 0.10) + + nn_intervals = interpolate_nn_pchip(nn_intervals, 2) + + valid_ratio = np.sum(~np.isnan(nn_intervals)) / len(nn_intervals) + nn_intervals = nn_intervals[~np.isnan(nn_intervals)] + + if valid_ratio < 0.75: + continue + + metrics = compute_hrv_hrf(nn_intervals, fs) + + hrv_all.append([ + metrics["PIP"], metrics["PNNLS"], metrics["PNNSS"], + metrics["AVNN"], metrics["SDNN"], metrics["RMSSD"], metrics["HF"], + ectopic_perc + ]) + + if len(hrv_all) == 0: + return np.zeros(ecg_feature_length, dtype=np.float32) + + hrv_all = np.array(hrv_all) + + median_vals = np.nanmedian(hrv_all, axis=0) + std_vals = np.nanstd(hrv_all, axis=0) + + features = np.array([ + median_vals[0], std_vals[0], + median_vals[1], std_vals[1], + median_vals[2], std_vals[2], + median_vals[3], std_vals[3], + median_vals[4], std_vals[4], + median_vals[5], std_vals[5], + median_vals[6], std_vals[6], + median_vals[7], std_vals[7], + ], dtype=np.float32) + + return features \ No newline at end of file diff --git a/src/lib/ecg_hrv_features.py b/src/lib/ecg_hrv_features.py new file mode 100644 index 0000000..e8ede73 --- /dev/null +++ b/src/lib/ecg_hrv_features.py @@ -0,0 +1,123 @@ +import numpy as np +from scipy.signal import lombscargle + +def compute_hrv_hrf(nn_intervals, sampling_frequency): + """Compute time-domain and HF HRV metrics from NN intervals.""" + + nn_intervals = np.asarray(nn_intervals).flatten() + + delta_nn = np.diff(nn_intervals) + + n = 1 + threshold = n / sampling_frequency + + acceleration = delta_nn <= -threshold + deceleration = delta_nn >= threshold + + sign_delta_nn = np.zeros_like(delta_nn) + sign_delta_nn[acceleration] = -1 + sign_delta_nn[deceleration] = 1 + + num_deltas = len(delta_nn) + + inflection = 0 + + for index in range(num_deltas - 1): + if (delta_nn[index + 1] * delta_nn[index] <= 0) and (delta_nn[index + 1] != delta_nn[index]): + inflection += 1 + + pip = (inflection / (num_deltas - 1)) * 100 if num_deltas > 1 else np.nan + + segments = [] + if num_deltas > 0: + current_segment = sign_delta_nn[0] + segment_length = 1 + + for index in range(1, num_deltas): + if sign_delta_nn[index] == current_segment and sign_delta_nn[index] != 0: + segment_length += 1 + else: + if current_segment != 0: + segments.append(segment_length) + current_segment = sign_delta_nn[index] + segment_length = 1 + + if current_segment != 0: + segments.append(segment_length) + + segments = np.array(segments) + + if len(segments) > 0: + long_segments = segments[segments >= 3] + short_segments = segments[segments < 3] + + pnnls = np.sum(long_segments) / num_deltas * 100 + pnnss = np.sum(short_segments) / np.sum(segments) * 100 + else: + pnnls = np.nan + pnnss = np.nan + + window_length = 300 + + elapsed_time = np.cumsum(nn_intervals) + + avnn_values = [] + sdnn_values = [] + rmssd_values = [] + + index = 0 + while index < len(nn_intervals): + window_start = elapsed_time[index] + window_end = window_start + window_length + + window_indices = np.where((elapsed_time >= window_start) & (elapsed_time < window_end))[0] + + if len(window_indices) >= 150: + window_nn = nn_intervals[window_indices] + + avnn_values.append(np.nanmean(window_nn)) + sdnn_values.append(np.nanstd(window_nn, ddof=1)) + + diff_nn = np.diff(window_nn) + rmssd_values.append(np.sqrt(np.nanmean(diff_nn**2))) + + next_indices = np.where(elapsed_time >= window_end)[0] + if len(next_indices) == 0: + break + index = next_indices[0] + + avnn = np.nanmean(avnn_values) if len(avnn_values) > 0 else np.nan + sdnn = np.nanmean(sdnn_values) if len(sdnn_values) > 0 else np.nan + rmssd = np.nanmean(rmssd_values) if len(rmssd_values) > 0 else np.nan + + hf_values = [] + + for _ in range(len(avnn_values)): + window_nn = nn_intervals.copy() + window_time = np.cumsum(window_nn) + + frequencies = np.linspace(0.01, 0.5, 1000) + angular_frequencies = 2 * np.pi * frequencies + detrended_nn = window_nn - np.mean(window_nn) + + power_spectrum = lombscargle(window_time, detrended_nn, angular_frequencies, normalize=True) + + hf_band = (frequencies >= 0.15) & (frequencies <= 0.4) + + hf_power = np.trapezoid(power_spectrum[hf_band], frequencies[hf_band]) + + hf_values.append(hf_power) + + hf = np.nanmean(hf_values) if len(hf_values) > 0 else np.nan + + results = { + "PIP": pip, + "PNNLS": pnnls, + "PNNSS": pnnss, + "AVNN": avnn, + "SDNN": sdnn, + "RMSSD": rmssd, + "HF": hf, + } + + return results \ No newline at end of file diff --git a/src/lib/ecg_nn_interpolation.py b/src/lib/ecg_nn_interpolation.py new file mode 100644 index 0000000..dc61a42 --- /dev/null +++ b/src/lib/ecg_nn_interpolation.py @@ -0,0 +1,34 @@ +import numpy as np +from scipy.interpolate import PchipInterpolator + +def interpolate_nn_pchip(nn_intervals, max_gap): + """Interpolate short NaN gaps in NN intervals using PCHIP.""" + + nn_intervals = np.asarray(nn_intervals).flatten() + nn_interpolated = nn_intervals.copy() + + nan_indices = np.isnan(nn_intervals) + + boundaries = np.diff(np.concatenate(([0], nan_indices.astype(int), [0]))) + start_indices = np.where(boundaries == 1)[0] + end_indices = np.where(boundaries == -1)[0] - 1 + + for index in range(len(start_indices)): + segment_length = end_indices[index] - start_indices[index] + 1 + + if segment_length <= max_gap: + left = start_indices[index] - 1 + right = end_indices[index] + 1 + + if (left >= 0 and right < len(nn_intervals) and + not np.isnan(nn_intervals[left]) and not np.isnan(nn_intervals[right])): + + x = np.array([left, right]) + y = np.array([nn_intervals[left], nn_intervals[right]]) + + xi = np.arange(start_indices[index], end_indices[index] + 1) + + interpolator = PchipInterpolator(x, y) + nn_interpolated[xi] = interpolator(xi) + + return nn_interpolated \ No newline at end of file diff --git a/src/lib/pan_tompkins.py b/src/lib/ecg_peak_detection.py similarity index 99% rename from src/lib/pan_tompkins.py rename to src/lib/ecg_peak_detection.py index 3f37214..6801f8c 100644 --- a/src/lib/pan_tompkins.py +++ b/src/lib/ecg_peak_detection.py @@ -2,7 +2,7 @@ from scipy.signal import butter, filtfilt, find_peaks -def pan_tompkin(ecg, fs, gr=0): +def pan_tompkins(ecg, fs, gr=0): ecg = np.asarray(ecg).flatten() delay = 0 diff --git a/src/lib/ecg_rr_cleaning.py b/src/lib/ecg_rr_cleaning.py new file mode 100644 index 0000000..e56597a --- /dev/null +++ b/src/lib/ecg_rr_cleaning.py @@ -0,0 +1,38 @@ +import numpy as np + +def remove_ectopic_beats(nn_intervals, window_size, threshold): + nn_intervals = np.asarray(nn_intervals).flatten() + nn_corrected = nn_intervals.copy() + + half_win = window_size // 2 + ectopic_count = 0 + valid_count = 0 + + for index in range(len(nn_intervals)): + + if np.isnan(nn_intervals[index]): + continue + + valid_count += 1 + + left = max(0, index - half_win) + right = min(len(nn_intervals), index + half_win + 1) + + local_segment = nn_intervals[left:right] + local_segment = local_segment[~np.isnan(local_segment)] + + if local_segment.size == 0: + continue + + med_val = np.median(local_segment) + + if abs(nn_intervals[index] - med_val) > threshold * med_val: + nn_corrected[index] = med_val + ectopic_count += 1 + + if valid_count > 0: + ectopic_perc = (ectopic_count / valid_count) * 100 + else: + ectopic_perc = np.nan + + return nn_corrected, ectopic_perc \ No newline at end of file diff --git a/src/lib/EEG_functions.py b/src/lib/eeg_features.py similarity index 100% rename from src/lib/EEG_functions.py rename to src/lib/eeg_features.py diff --git a/src/lib/interpolate_NN.py b/src/lib/interpolate_NN.py deleted file mode 100644 index 987019f..0000000 --- a/src/lib/interpolate_NN.py +++ /dev/null @@ -1,40 +0,0 @@ -import numpy as np -from scipy.interpolate import PchipInterpolator - -def interpolate_NN_pchip(NN, maxGap): - """ - NN: array of NN intervals (seconds) - maxGap: max number of consecutive NaNs allowed for interpolation - """ - - NN = np.asarray(NN).flatten() - NN_interp = NN.copy() - - nan_idx = np.isnan(NN) - - # Find NaN segments - d = np.diff(np.concatenate(([0], nan_idx.astype(int), [0]))) - start_idx = np.where(d == 1)[0] - end_idx = np.where(d == -1)[0] - 1 - - for k in range(len(start_idx)): - seg_len = end_idx[k] - start_idx[k] + 1 - - if seg_len <= maxGap: - left = start_idx[k] - 1 - right = end_idx[k] + 1 - - # Check bounds - if (left >= 0 and right < len(NN) and - not np.isnan(NN[left]) and not np.isnan(NN[right])): - - x = np.array([left, right]) - y = np.array([NN[left], NN[right]]) - - xi = np.arange(start_idx[k], end_idx[k] + 1) - - # PCHIP interpolation - interpolator = PchipInterpolator(x, y) - NN_interp[xi] = interpolator(xi) - - return NN_interp \ No newline at end of file diff --git a/src/lib/remove_ectopic_beat.py b/src/lib/remove_ectopic_beat.py deleted file mode 100644 index de60caa..0000000 --- a/src/lib/remove_ectopic_beat.py +++ /dev/null @@ -1,41 +0,0 @@ -import numpy as np - -def remove_ectopic_beats(NN, window_size, threshold): - NN = np.asarray(NN).flatten() - NN_corrected = NN.copy() - - half_win = window_size // 2 - ectopic_count = 0 - valid_count = 0 - - for i in range(len(NN)): - - if np.isnan(NN[i]): - continue - - valid_count += 1 - - # Define local window - left = max(0, i - half_win) - right = min(len(NN), i + half_win + 1) # Python slice is exclusive - - local_segment = NN[left:right] - local_segment = local_segment[~np.isnan(local_segment)] - - if local_segment.size == 0: - continue - - med_val = np.median(local_segment) - - # Detect ectopic - if abs(NN[i] - med_val) > threshold * med_val: - NN_corrected[i] = med_val - ectopic_count += 1 - - # Percentage over valid NN - if valid_count > 0: - ectopic_perc = (ectopic_count / valid_count) * 100 - else: - ectopic_perc = np.nan - - return NN_corrected, ectopic_perc \ No newline at end of file diff --git a/src/lib/peakedness.py b/src/lib/resp_peakedness.py similarity index 96% rename from src/lib/peakedness.py rename to src/lib/resp_peakedness.py index 772c907..f14d41e 100644 --- a/src/lib/peakedness.py +++ b/src/lib/resp_peakedness.py @@ -4,7 +4,6 @@ from numpy.fft import fft from scipy.signal import detrend, find_peaks from time import time -import os def _safe_ratio(numerator, denominator, default=0.0): if denominator is None or not np.isfinite(denominator) or denominator == 0: @@ -274,21 +273,7 @@ def init_module(kk,vars,param, plotflag): vars["bar_fr"][kk] = f[fj] else: vars["bar_fr"][kk] = 0 - # Save in vars - # vars["bar_fr"][kk] = f[fj] - - # if plotflag: - # if plt is None: - # raise ModuleNotFoundError("matplotlib is required when plotflag=True") - # plt.plot(f, averS) - # plt.plot(f[fj], averS[fj], '-') - # plt.title('Initialization - Averaged Spectrum') - # plt.show() - return vars - # # No spectra fulfill the initialization - # if plotflag: - # keyboard def compute_Xkl( Skl, f, bar_fr, O, ksi_p, ksi_a, d): # function [ Xkl ] = compute_Xkl( Skl, f, bar_fr, O, ksi_p, ksi_a, d) @@ -371,20 +356,15 @@ def compute_fJmin( S, f, bar_fr, d): # Put the location in the correct perspective lm = peaks + (Omega[:] ==1).argmax() - # Select the frequency that corresponds to the location - fJ = f[lm] - - # print(len(lm)) - if len(lm) > 0: - # Compute the cost function for deviation from previous fr and maximum power + if lm.shape[0] == 1: + fJmin = f[lm[0]] + elif lm.shape[0] > 1: + fJ = f[lm] C_f = abs(fJ-bar_fr)/(2*d) C_a = 1-S[lm]/max(S[Omega]) - - # Select the minimum cost + C = C_f+C_a Jmin = C.argmin() - - # Store the frequency with the minimum cost fJmin = fJ[Jmin] return fJmin @@ -392,16 +372,13 @@ def compute_fJmin( S, f, bar_fr, d): def peakednessCost(signals, ts, fs, Setup = {}, title = "", storeGraph = False, subjet =1): vars = {} - # Set parameters / Arrange inputs [ DT, Ts, Tm, Nfft, K, Omega_r, ksi_p, ksi_a, d, b, a,N_k, plotflag , Setup] = setParamFr(Setup) - # Start the time stamps at zero ts1 = ts[0] ts = ts-ts1 if type(signals) == type(pd.DataFrame()): signals = signals.to_numpy() - # Get the number of signals if len(signals.shape) == 1: signals = np.reshape(signals, (signals.shape[0],1)) if signals.shape[0] Date: Tue, 31 Mar 2026 11:22:29 +0200 Subject: [PATCH 56/93] Add feature cache directory support in run scripts --- run.ps1 | 13 +++++++++++++ run.sh | 29 +++++++++++++++++++++++++---- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/run.ps1 b/run.ps1 index 8d2141b..e33cae4 100644 --- a/run.ps1 +++ b/run.ps1 @@ -34,6 +34,7 @@ $IMAGE_NAME = "cinc2026" $MODEL_FULL_REL = "model" $MODEL_SMOKE_REL = "model_smoke" +$FEATURE_CACHE_REL = ".feature_cache" $OUT_FULL_REL = "outputs" $OUT_SMOKE_REL = "outputs_smoke" @@ -98,12 +99,15 @@ function Train-Full { $FULL_DATA = Get-AbsolutePath $TRAIN_DATA_REL $MODEL_FULL = Join-Path (Get-AbsolutePath ".") $MODEL_FULL_REL + $FEATURE_CACHE = Join-Path (Get-AbsolutePath ".") $FEATURE_CACHE_REL Ensure-Directory $MODEL_FULL + Ensure-Directory $FEATURE_CACHE docker run --rm ` -v "${FULL_DATA}:/challenge/training_data:ro" ` -v "${MODEL_FULL}:/challenge/model" ` + -v "${FEATURE_CACHE}:/challenge/.feature_cache" ` $IMAGE_NAME ` python train_model.py -d training_data -m model -v } @@ -112,12 +116,15 @@ function Train-Smoke { $SMOKE_DATA = Get-AbsolutePath $SMOKE_DATA_REL $MODEL_SMOKE = Join-Path (Get-AbsolutePath ".") $MODEL_SMOKE_REL + $FEATURE_CACHE = Join-Path (Get-AbsolutePath ".") $FEATURE_CACHE_REL Ensure-Directory $MODEL_SMOKE + Ensure-Directory $FEATURE_CACHE docker run --rm ` -v "${SMOKE_DATA}:/challenge/training_data:ro" ` -v "${MODEL_SMOKE}:/challenge/model" ` + -v "${FEATURE_CACHE}:/challenge/.feature_cache" ` $IMAGE_NAME ` python train_model.py -d training_data -m model -v } @@ -127,13 +134,16 @@ function Run-Full { $RUN_DATA = Get-AbsolutePath $RUN_DATA_REL $MODEL_FULL = Get-AbsolutePath $MODEL_FULL_REL $OUT_FULL = Join-Path (Get-AbsolutePath ".") $OUT_FULL_REL + $FEATURE_CACHE = Join-Path (Get-AbsolutePath ".") $FEATURE_CACHE_REL Ensure-Directory $OUT_FULL + Ensure-Directory $FEATURE_CACHE docker run --rm ` -v "${RUN_DATA}:/challenge/holdout_data:ro" ` -v "${MODEL_FULL}:/challenge/model:ro" ` -v "${OUT_FULL}:/challenge/holdout_outputs" ` + -v "${FEATURE_CACHE}:/challenge/.feature_cache" ` $IMAGE_NAME ` python run_model.py -d holdout_data -m model -o holdout_outputs -v @@ -149,13 +159,16 @@ function Run-Smoke { $SMOKE_DATA = Get-AbsolutePath $SMOKE_DATA_REL $MODEL_SMOKE = Get-AbsolutePath $MODEL_SMOKE_REL $OUT_SMOKE = Join-Path (Get-AbsolutePath ".") $OUT_SMOKE_REL + $FEATURE_CACHE = Join-Path (Get-AbsolutePath ".") $FEATURE_CACHE_REL Ensure-Directory $OUT_SMOKE + Ensure-Directory $FEATURE_CACHE docker run --rm ` -v "${SMOKE_DATA}:/challenge/holdout_data:ro" ` -v "${MODEL_SMOKE}:/challenge/model:ro" ` -v "${OUT_SMOKE}:/challenge/holdout_outputs" ` + -v "${FEATURE_CACHE}:/challenge/.feature_cache" ` $IMAGE_NAME ` python run_model.py -d holdout_data -m model -o holdout_outputs -v diff --git a/run.sh b/run.sh index dcabdbf..7569760 100644 --- a/run.sh +++ b/run.sh @@ -20,6 +20,7 @@ IMAGE_NAME="cinc2026" MODEL_FULL_REL="model" MODEL_SMOKE_REL="model_smoke" +FEATURE_CACHE_REL=".feature_cache" OUT_FULL_REL="outputs" OUT_SMOKE_REL="outputs_smoke" @@ -110,57 +111,72 @@ create_smoke() { train_full() { local full_data model_full - local full_data_docker model_full_docker + local feature_cache + local full_data_docker model_full_docker feature_cache_docker full_data="$(get_absolute_path "$TRAIN_DATA_REL")" model_full="$(get_absolute_path ".")/${MODEL_FULL_REL}" + feature_cache="$(get_absolute_path ".")/${FEATURE_CACHE_REL}" full_data_docker="$(to_docker_path "$full_data")" model_full_docker="$(to_docker_path "$model_full")" + feature_cache_docker="$(to_docker_path "$feature_cache")" ensure_directory "$model_full" + ensure_directory "$feature_cache" docker_cli run --rm \ -v "${full_data_docker}:/challenge/training_data:ro" \ -v "${model_full_docker}:/challenge/model" \ + -v "${feature_cache_docker}:/challenge/.feature_cache" \ "$IMAGE_NAME" \ python train_model.py -d training_data -m model -v } train_smoke() { local smoke_data model_smoke - local smoke_data_docker model_smoke_docker + local feature_cache + local smoke_data_docker model_smoke_docker feature_cache_docker smoke_data="$(get_absolute_path "$SMOKE_DATA_REL")" model_smoke="$(get_absolute_path ".")/${MODEL_SMOKE_REL}" + feature_cache="$(get_absolute_path ".")/${FEATURE_CACHE_REL}" smoke_data_docker="$(to_docker_path "$smoke_data")" model_smoke_docker="$(to_docker_path "$model_smoke")" + feature_cache_docker="$(to_docker_path "$feature_cache")" ensure_directory "$model_smoke" + ensure_directory "$feature_cache" docker_cli run --rm \ -v "${smoke_data_docker}:/challenge/training_data:ro" \ -v "${model_smoke_docker}:/challenge/model" \ + -v "${feature_cache_docker}:/challenge/.feature_cache" \ "$IMAGE_NAME" \ python train_model.py -d training_data -m model -v } run_full() { local run_data model_full out_full - local run_data_docker model_full_docker out_full_docker + local feature_cache + local run_data_docker model_full_docker out_full_docker feature_cache_docker run_data="$(get_absolute_path "$RUN_DATA_REL")" model_full="$(get_absolute_path "$MODEL_FULL_REL")" out_full="$(get_absolute_path ".")/${OUT_FULL_REL}" + feature_cache="$(get_absolute_path ".")/${FEATURE_CACHE_REL}" run_data_docker="$(to_docker_path "$run_data")" model_full_docker="$(to_docker_path "$model_full")" out_full_docker="$(to_docker_path "$out_full")" + feature_cache_docker="$(to_docker_path "$feature_cache")" ensure_directory "$out_full" + ensure_directory "$feature_cache" docker_cli run --rm \ -v "${run_data_docker}:/challenge/holdout_data:ro" \ -v "${model_full_docker}:/challenge/model:ro" \ -v "${out_full_docker}:/challenge/holdout_outputs" \ + -v "${feature_cache_docker}:/challenge/.feature_cache" \ "$IMAGE_NAME" \ python run_model.py -d holdout_data -m model -o holdout_outputs -v @@ -173,21 +189,26 @@ run_full() { run_smoke() { local smoke_data model_smoke out_smoke - local smoke_data_docker model_smoke_docker out_smoke_docker + local feature_cache + local smoke_data_docker model_smoke_docker out_smoke_docker feature_cache_docker smoke_data="$(get_absolute_path "$SMOKE_DATA_REL")" model_smoke="$(get_absolute_path "$MODEL_SMOKE_REL")" out_smoke="$(get_absolute_path ".")/${OUT_SMOKE_REL}" + feature_cache="$(get_absolute_path ".")/${FEATURE_CACHE_REL}" smoke_data_docker="$(to_docker_path "$smoke_data")" model_smoke_docker="$(to_docker_path "$model_smoke")" out_smoke_docker="$(to_docker_path "$out_smoke")" + feature_cache_docker="$(to_docker_path "$feature_cache")" ensure_directory "$out_smoke" + ensure_directory "$feature_cache" docker_cli run --rm \ -v "${smoke_data_docker}:/challenge/holdout_data:ro" \ -v "${model_smoke_docker}:/challenge/model:ro" \ -v "${out_smoke_docker}:/challenge/holdout_outputs" \ + -v "${feature_cache_docker}:/challenge/.feature_cache" \ "$IMAGE_NAME" \ python run_model.py -d holdout_data -m model -o holdout_outputs -v From cee9467541788d624e959acf1f2e344233f08e5b Mon Sep 17 00:00:00 2001 From: dcajal Date: Tue, 31 Mar 2026 12:02:46 +0200 Subject: [PATCH 57/93] Multimodal ensemble model integration --- src/pipeline/features.py | 50 +++++++++++- src/pipeline/training.py | 169 ++++++++++++++++++++++++++++++++++----- team_code.py | 11 +-- 3 files changed, 204 insertions(+), 26 deletions(-) diff --git a/src/pipeline/features.py b/src/pipeline/features.py index 4f03ef1..d29aae8 100644 --- a/src/pipeline/features.py +++ b/src/pipeline/features.py @@ -7,14 +7,32 @@ from src.common.channel_utils import normalize_channel_label from helper_code import HEADERS, PHYSIOLOGICAL_DATA_SUBFOLDER, load_age, load_sex, load_signal_data -from src.ecg_processing import ECG_FEATURE_LENGTH, ECG_KEYWORDS, processECG -from src.eeg_processing import EEG_CHANNEL_SPECS, EEG_FEATURE_LENGTH, processEEG, _get_eeg_aliases -from src.resp_processing import RESP_FEATURE_LENGTH, processResp, _get_resp_alias_groups +from src.ecg_processing import ECG_FEATURE_LENGTH, ECG_FEATURE_NAMES, ECG_KEYWORDS, processECG +from src.eeg_processing import EEG_CHANNEL_SPECS, EEG_FEATURE_LENGTH, EEG_FEATURE_NAMES, processEEG, _get_eeg_aliases +from src.resp_processing import RESP_FEATURE_LENGTH, RESP_FEATURE_NAMES, processResp, _get_resp_alias_groups from .config import DEFAULT_CSV_PATH, FEATURE_CACHE_FOLDER_NAME, SCRIPT_DIR, TOTAL_PHYSIOLOGICAL_FEATURE_LENGTH REQUIRED_SIGNAL_ALIASES_CACHE = {} +DEMOGRAPHIC_FEATURE_NAMES = ( + 'Age', + 'Sex_Female', + 'Sex_Male', + 'Sex_Unknown', +) +FEATURE_NAME_GROUPS = { + 'demographics': DEMOGRAPHIC_FEATURE_NAMES, + 'resp': tuple(RESP_FEATURE_NAMES), + 'eeg': tuple(EEG_FEATURE_NAMES), + 'ecg': tuple(ECG_FEATURE_NAMES), +} +FEATURE_NAMES = ( + *FEATURE_NAME_GROUPS['demographics'], + *FEATURE_NAME_GROUPS['resp'], + *FEATURE_NAME_GROUPS['eeg'], + *FEATURE_NAME_GROUPS['ecg'], +) def _coerce_feature_vector(features): @@ -145,6 +163,32 @@ def extract_demographic_features(data): return np.concatenate([age, sex_vec]) +def get_feature_names(): + return FEATURE_NAMES + + +def get_feature_group_indices(include_demographics=False): + groups = {} + start = len(FEATURE_NAME_GROUPS['demographics']) + + if include_demographics: + demo_indices = np.arange(start, dtype=np.int32) + else: + demo_indices = np.array([], dtype=np.int32) + + for group_name in ('resp', 'eeg', 'ecg'): + group_length = len(FEATURE_NAME_GROUPS[group_name]) + group_indices = np.arange(start, start + group_length, dtype=np.int32) + if include_demographics: + groups[group_name] = np.concatenate([demo_indices, group_indices]) + else: + groups[group_name] = group_indices + start += group_length + + groups['all'] = np.arange(len(FEATURE_NAMES), dtype=np.int32) + return groups + + def extract_extended_physiological_features(physiological_data, physiological_fs, csv_path=DEFAULT_CSV_PATH): try: resp_features = _extract_optional_features( diff --git a/src/pipeline/training.py b/src/pipeline/training.py index ae2c34a..550ad62 100644 --- a/src/pipeline/training.py +++ b/src/pipeline/training.py @@ -3,13 +3,19 @@ import numpy as np import pandas as pd +from sklearn.metrics import f1_score +from sklearn.model_selection import train_test_split from tqdm import tqdm from xgboost import XGBClassifier from helper_code import DEMOGRAPHICS_FILE, HEADERS, find_patients, load_label from .config import MAX_TRAIN_WORKERS -from .features import get_or_create_record_feature_vector +from .features import get_feature_group_indices, get_feature_names, get_or_create_record_feature_vector + + +DEFAULT_ENSEMBLE_THRESHOLD = 0.5 +ENSEMBLE_MODALITIES = ('resp', 'eeg', 'ecg') def build_training_metadata_cache(patient_data_file): @@ -53,7 +59,130 @@ def process_training_record(record, data_folder, demographics_cache, diagnosis_c return patient_id, None, None, f"Error processing {patient_id}: {exc}" -def train_xgb_model(data_folder, verbose, csv_path): +def best_threshold(probabilities, labels): + thresholds = np.linspace(0, 1, 101) + best_score = -1.0 + best_value = DEFAULT_ENSEMBLE_THRESHOLD + + for threshold in thresholds: + predictions = (probabilities >= threshold).astype(np.int32) + score = f1_score(labels, predictions, zero_division=0) + if score > best_score: + best_score = score + best_value = float(threshold) + + return best_value + + +def _build_xgb_model(labels): + neg = int(np.sum(labels == 0)) + pos = int(np.sum(labels == 1)) + scale_pos_weight = (neg / pos) if pos > 0 else 1.0 + + return XGBClassifier( + scale_pos_weight=scale_pos_weight, + n_estimators=300, + max_depth=4, + learning_rate=0.05, + subsample=0.8, + random_state=42, + eval_metric='auc', + tree_method='hist', + ) + + +def _fit_model(feature_matrix, labels): + model = _build_xgb_model(labels) + model.fit(feature_matrix, labels) + return model + + +def _fit_ensemble(feature_matrix, labels, feature_indices): + models = { + 'all': _fit_model(feature_matrix[:, feature_indices['all']], labels), + } + + for modality in ENSEMBLE_MODALITIES: + models[modality] = _fit_model(feature_matrix[:, feature_indices[modality]], labels) + + return models + + +def _has_modality_signal(feature_vector, modality_presence_indices): + modality_values = feature_vector[modality_presence_indices] + return bool(np.any(np.abs(modality_values) > 0.0)) + + +def predict_ensemble_probabilities(model_bundle, feature_matrix): + feature_matrix = np.asarray(feature_matrix, dtype=np.float32) + if feature_matrix.ndim == 1: + feature_matrix = feature_matrix.reshape(1, -1) + + models = model_bundle['models'] + feature_indices = { + name: np.asarray(indices, dtype=np.int32) + for name, indices in model_bundle['feature_indices'].items() + } + modality_presence_indices = { + name: np.asarray(indices, dtype=np.int32) + for name, indices in model_bundle['modality_presence_indices'].items() + } + + probabilities = np.zeros(feature_matrix.shape[0], dtype=np.float32) + for row_index, feature_vector in enumerate(feature_matrix): + modality_probabilities = [] + for modality in ENSEMBLE_MODALITIES: + if _has_modality_signal(feature_vector, modality_presence_indices[modality]): + modality_vector = feature_vector[feature_indices[modality]].reshape(1, -1) + modality_probability = models[modality].predict_proba(modality_vector)[0][1] + modality_probabilities.append(float(modality_probability)) + + if modality_probabilities: + probabilities[row_index] = float(np.mean(modality_probabilities)) + else: + all_features = feature_vector[feature_indices['all']].reshape(1, -1) + probabilities[row_index] = float(models['all'].predict_proba(all_features)[0][1]) + + return probabilities + + +def predict_ensemble_labels(model_bundle, feature_matrix): + threshold = float(model_bundle.get('threshold', DEFAULT_ENSEMBLE_THRESHOLD)) + probabilities = predict_ensemble_probabilities(model_bundle, feature_matrix) + labels = (probabilities >= threshold).astype(np.int32) + return labels, probabilities + + +def _calibrate_threshold(feature_matrix, labels, feature_indices): + classes, class_counts = np.unique(labels, return_counts=True) + if len(classes) != 2 or np.min(class_counts) < 2 or len(labels) < 10: + return DEFAULT_ENSEMBLE_THRESHOLD + + try: + train_features, validation_features, train_labels, validation_labels = train_test_split( + feature_matrix, + labels, + test_size=0.2, + random_state=42, + stratify=labels, + ) + except ValueError: + return DEFAULT_ENSEMBLE_THRESHOLD + + calibration_bundle = { + 'models': _fit_ensemble(train_features, train_labels, feature_indices), + 'feature_indices': feature_indices, + 'modality_presence_indices': { + modality: feature_indices[f'{modality}_only'] + for modality in ENSEMBLE_MODALITIES + }, + 'threshold': DEFAULT_ENSEMBLE_THRESHOLD, + } + validation_probabilities = predict_ensemble_probabilities(calibration_bundle, validation_features) + return best_threshold(validation_probabilities, validation_labels) + + +def train_multimodal_ensemble(data_folder, verbose, csv_path): patient_data_file = os.path.join(data_folder, DEMOGRAPHICS_FILE) patient_metadata_list = find_patients(patient_data_file) demographics_cache, diagnosis_cache = build_training_metadata_cache(patient_data_file) @@ -97,19 +226,23 @@ def train_xgb_model(data_folder, verbose, csv_path): if features.size == 0 or features.ndim != 2 or features.shape[0] == 0: raise ValueError('No valid training samples were extracted. Review feature extraction logs for the skipped records.') - neg = int(np.sum(labels == 0)) - pos = int(np.sum(labels == 1)) - scale_pos_weight = (neg / pos) if pos > 0 else 1.0 - - model = XGBClassifier( - scale_pos_weight=scale_pos_weight, - n_estimators=300, - max_depth=4, - learning_rate=0.05, - subsample=0.8, - random_state=42, - eval_metric='auc', - tree_method='hist', - ) - model.fit(features, labels) - return model \ No newline at end of file + feature_indices = get_feature_group_indices(include_demographics=True) + threshold = _calibrate_threshold(features, labels, feature_indices) + models = _fit_ensemble(features, labels, feature_indices) + + return { + 'type': 'multimodal_xgb_ensemble', + 'threshold': threshold, + 'feature_names': list(get_feature_names()), + 'feature_indices': { + name: indices.tolist() + for name, indices in feature_indices.items() + if name in {'all', 'resp', 'eeg', 'ecg'} + }, + 'modality_presence_indices': { + modality: get_feature_group_indices(include_demographics=False)[modality].tolist() + for modality in ENSEMBLE_MODALITIES + }, + 'models': models, + 'preprocessor': None, + } \ No newline at end of file diff --git a/team_code.py b/team_code.py index 35b1c11..53926be 100644 --- a/team_code.py +++ b/team_code.py @@ -19,8 +19,8 @@ from helper_code import * from src.pipeline.config import DEFAULT_CSV_PATH -from src.pipeline.features import _get_feature_cache_file, get_or_create_record_feature_vector -from src.pipeline.training import train_xgb_model +from src.pipeline.features import get_or_create_record_feature_vector +from src.pipeline.training import predict_ensemble_labels, train_multimodal_ensemble ################################################################################ # Path & Constant Configuration (Added for Robustness) ################################################################################ @@ -82,7 +82,7 @@ def train_model(data_folder, model_folder, verbose, csv_path=DEFAULT_CSV_PATH): if verbose: print('Training the model on the data...') - model = train_xgb_model(data_folder, verbose, csv_path) + model = train_multimodal_ensemble(data_folder, verbose, csv_path) # Create a folder for the model if it does not already exist. os.makedirs(model_folder, exist_ok=True) @@ -150,8 +150,9 @@ def run_model(model, record, data_folder, verbose): ).reshape(1, -1) # Get the model outputs. - binary_output = model.predict(features)[0] - probability_output = model.predict_proba(features)[0][1] + labels, probabilities = predict_ensemble_labels(model, features) + binary_output = bool(int(labels[0])) + probability_output = float(probabilities[0]) if verbose and RUN_MODEL_PBAR is not None: RUN_MODEL_PBAR.update(1) From e902673363c13c06bcec9c2bca4c30810b237525 Mon Sep 17 00:00:00 2001 From: dcajal Date: Tue, 31 Mar 2026 13:02:13 +0200 Subject: [PATCH 58/93] Update processing functions to handle NaN values. Add imputation with knn --- src/ecg_processing.py | 5 ++-- src/eeg_processing.py | 14 ++++----- src/pipeline/features.py | 24 ++++++++++----- src/pipeline/training.py | 64 +++++++++++++++++++++++++++------------- src/resp_processing.py | 2 +- 5 files changed, 71 insertions(+), 38 deletions(-) diff --git a/src/ecg_processing.py b/src/ecg_processing.py index e4d9a96..044f54c 100644 --- a/src/ecg_processing.py +++ b/src/ecg_processing.py @@ -33,7 +33,7 @@ def _find_ecg_channel(physiological_data): def processECG(physiological_data, physiological_fs, csv_path): - results = np.zeros(ECG_FEATURE_LENGTH, dtype=np.float32) + results = np.full(ECG_FEATURE_LENGTH, np.nan, dtype=np.float32) ecg_label = _find_ecg_channel(physiological_data) @@ -55,7 +55,8 @@ def processECG(physiological_data, physiological_fs, csv_path): if values is None or len(values) == 0: return results - values = values.astype(np.float32) + values = np.asarray(values, dtype=np.float32) + values[~np.isfinite(values)] = np.nan if len(values) >= ECG_FEATURE_LENGTH: results[:] = values[:ECG_FEATURE_LENGTH] diff --git a/src/eeg_processing.py b/src/eeg_processing.py index ab4d00d..37b74f5 100644 --- a/src/eeg_processing.py +++ b/src/eeg_processing.py @@ -142,14 +142,14 @@ def _extract_channel_metrics(signal, fs): patient_profile = eeg_features.get_patient_profile(band_powers) metrics = { str(name): float(value) - for name, value in patient_profile.replace([np.inf, -np.inf], np.nan).fillna(0.0).items() + for name, value in patient_profile.replace([np.inf, -np.inf], np.nan).items() } for complexity_name in ('Hjorth_Mobility', 'Hjorth_Complexity'): if complexity_name in complexities: - value = complexities[complexity_name].replace([np.inf, -np.inf], np.nan).fillna(0.0).std() - metrics[complexity_name] = float(0.0 if pd.isna(value) else value) + value = complexities[complexity_name].replace([np.inf, -np.inf], np.nan).std() + metrics[complexity_name] = float(np.nan if pd.isna(value) else value) else: - metrics[complexity_name] = 0.0 + metrics[complexity_name] = np.nan return metrics @@ -167,15 +167,15 @@ def processEEG(physiological_data, physiological_fs, csv_path): channel_profiles[channel_name] = metrics if not channel_profiles: - return np.zeros(EEG_FEATURE_LENGTH, dtype=np.float32) + return np.full(EEG_FEATURE_LENGTH, np.nan, dtype=np.float32) values = [] for channel_name, metric_name in EEG_FEATURE_SPECS: channel_metrics = channel_profiles.get(channel_name) if channel_metrics is None: - values.append(0.0) + values.append(np.nan) continue - values.append(float(channel_metrics.get(metric_name, 0.0))) + values.append(float(channel_metrics.get(metric_name, np.nan))) return np.asarray(values, dtype=np.float32) diff --git a/src/pipeline/features.py b/src/pipeline/features.py index d29aae8..6b092be 100644 --- a/src/pipeline/features.py +++ b/src/pipeline/features.py @@ -37,7 +37,8 @@ def _coerce_feature_vector(features): vector = np.asarray(features, dtype=np.float32).reshape(-1) - return np.nan_to_num(vector, nan=0.0, posinf=0.0, neginf=0.0) + vector[~np.isfinite(vector)] = np.nan + return vector def _extract_optional_features(extractor, expected_length, *args, **kwargs): @@ -149,10 +150,17 @@ def _save_cached_feature_vector(cache_file, feature_vector): def extract_demographic_features(data): - age = np.array([load_age(data)]) + age_value = data.get(HEADERS['age']) + try: + age = float(age_value) + except (TypeError, ValueError): + age = np.nan + if not np.isfinite(age): + age = np.nan + age = np.array([age], dtype=np.float32) sex = load_sex(data) - sex_vec = np.zeros(3) + sex_vec = np.zeros(3, dtype=np.float32) if sex == 'Female': sex_vec[0] = 1 elif sex == 'Male': @@ -160,7 +168,7 @@ def extract_demographic_features(data): else: sex_vec[2] = 1 - return np.concatenate([age, sex_vec]) + return np.concatenate([age, sex_vec]).astype(np.float32) def get_feature_names(): @@ -199,7 +207,7 @@ def extract_extended_physiological_features(physiological_data, physiological_fs csv_path=csv_path, ) except Exception: - resp_features = np.zeros(RESP_FEATURE_LENGTH, dtype=np.float32) + resp_features = np.full(RESP_FEATURE_LENGTH, np.nan, dtype=np.float32) try: eeg_features = _extract_optional_features( @@ -210,7 +218,7 @@ def extract_extended_physiological_features(physiological_data, physiological_fs csv_path=csv_path, ) except Exception: - eeg_features = np.zeros(EEG_FEATURE_LENGTH, dtype=np.float32) + eeg_features = np.full(EEG_FEATURE_LENGTH, np.nan, dtype=np.float32) try: ecg_features = _extract_optional_features( @@ -221,7 +229,7 @@ def extract_extended_physiological_features(physiological_data, physiological_fs csv_path=csv_path, ) except Exception: - ecg_features = np.zeros(ECG_FEATURE_LENGTH, dtype=np.float32) + ecg_features = np.full(ECG_FEATURE_LENGTH, np.nan, dtype=np.float32) return np.hstack([resp_features, eeg_features, ecg_features]).astype(np.float32) @@ -245,7 +253,7 @@ def _compute_record_feature_vector(patient_data, data_folder, site_id, patient_i elif require_physiological_data: raise FileNotFoundError(f"Missing physiological data for {patient_id}.") else: - physiological_features = np.zeros(TOTAL_PHYSIOLOGICAL_FEATURE_LENGTH, dtype=np.float32) + physiological_features = np.full(TOTAL_PHYSIOLOGICAL_FEATURE_LENGTH, np.nan, dtype=np.float32) return np.hstack([demographic_features, physiological_features]).astype(np.float32) diff --git a/src/pipeline/training.py b/src/pipeline/training.py index 550ad62..7d18c3b 100644 --- a/src/pipeline/training.py +++ b/src/pipeline/training.py @@ -3,8 +3,11 @@ import numpy as np import pandas as pd +from sklearn.impute import KNNImputer from sklearn.metrics import f1_score from sklearn.model_selection import train_test_split +from sklearn.pipeline import Pipeline +from sklearn.preprocessing import StandardScaler from tqdm import tqdm from xgboost import XGBClassifier @@ -16,6 +19,7 @@ DEFAULT_ENSEMBLE_THRESHOLD = 0.5 ENSEMBLE_MODALITIES = ('resp', 'eeg', 'ecg') +DEFAULT_KNN_NEIGHBORS = 5 def build_training_metadata_cache(patient_data_file): @@ -97,6 +101,14 @@ def _fit_model(feature_matrix, labels): return model +def _build_preprocessor(num_samples): + neighbors = min(DEFAULT_KNN_NEIGHBORS, max(1, num_samples - 1)) if num_samples > 1 else 1 + return Pipeline([ + ('imputer', KNNImputer(n_neighbors=neighbors, keep_empty_features=True)), + ('scaler', StandardScaler()), + ]) + + def _fit_ensemble(feature_matrix, labels, feature_indices): models = { 'all': _fit_model(feature_matrix[:, feature_indices['all']], labels), @@ -110,13 +122,15 @@ def _fit_ensemble(feature_matrix, labels, feature_indices): def _has_modality_signal(feature_vector, modality_presence_indices): modality_values = feature_vector[modality_presence_indices] - return bool(np.any(np.abs(modality_values) > 0.0)) + return bool(np.any(np.isfinite(modality_values))) def predict_ensemble_probabilities(model_bundle, feature_matrix): - feature_matrix = np.asarray(feature_matrix, dtype=np.float32) - if feature_matrix.ndim == 1: - feature_matrix = feature_matrix.reshape(1, -1) + raw_feature_matrix = np.asarray(feature_matrix, dtype=np.float32) + if raw_feature_matrix.ndim == 1: + raw_feature_matrix = raw_feature_matrix.reshape(1, -1) + raw_feature_matrix = raw_feature_matrix.copy() + raw_feature_matrix[~np.isfinite(raw_feature_matrix)] = np.nan models = model_bundle['models'] feature_indices = { @@ -127,20 +141,26 @@ def predict_ensemble_probabilities(model_bundle, feature_matrix): name: np.asarray(indices, dtype=np.int32) for name, indices in model_bundle['modality_presence_indices'].items() } - - probabilities = np.zeros(feature_matrix.shape[0], dtype=np.float32) - for row_index, feature_vector in enumerate(feature_matrix): + preprocessor = model_bundle.get('preprocessor') + if preprocessor is not None: + processed_feature_matrix = np.asarray(preprocessor.transform(raw_feature_matrix), dtype=np.float32) + else: + processed_feature_matrix = raw_feature_matrix + + probabilities = np.zeros(raw_feature_matrix.shape[0], dtype=np.float32) + for row_index, raw_feature_vector in enumerate(raw_feature_matrix): + processed_feature_vector = processed_feature_matrix[row_index] modality_probabilities = [] for modality in ENSEMBLE_MODALITIES: - if _has_modality_signal(feature_vector, modality_presence_indices[modality]): - modality_vector = feature_vector[feature_indices[modality]].reshape(1, -1) + if _has_modality_signal(raw_feature_vector, modality_presence_indices[modality]): + modality_vector = processed_feature_vector[feature_indices[modality]].reshape(1, -1) modality_probability = models[modality].predict_proba(modality_vector)[0][1] modality_probabilities.append(float(modality_probability)) if modality_probabilities: probabilities[row_index] = float(np.mean(modality_probabilities)) else: - all_features = feature_vector[feature_indices['all']].reshape(1, -1) + all_features = processed_feature_vector[feature_indices['all']].reshape(1, -1) probabilities[row_index] = float(models['all'].predict_proba(all_features)[0][1]) return probabilities @@ -153,7 +173,7 @@ def predict_ensemble_labels(model_bundle, feature_matrix): return labels, probabilities -def _calibrate_threshold(feature_matrix, labels, feature_indices): +def _calibrate_threshold(feature_matrix, labels, feature_indices, modality_presence_indices): classes, class_counts = np.unique(labels, return_counts=True) if len(classes) != 2 or np.min(class_counts) < 2 or len(labels) < 10: return DEFAULT_ENSEMBLE_THRESHOLD @@ -169,14 +189,15 @@ def _calibrate_threshold(feature_matrix, labels, feature_indices): except ValueError: return DEFAULT_ENSEMBLE_THRESHOLD + preprocessor = _build_preprocessor(len(train_labels)) + processed_train_features = np.asarray(preprocessor.fit_transform(train_features), dtype=np.float32) + calibration_bundle = { - 'models': _fit_ensemble(train_features, train_labels, feature_indices), + 'models': _fit_ensemble(processed_train_features, train_labels, feature_indices), 'feature_indices': feature_indices, - 'modality_presence_indices': { - modality: feature_indices[f'{modality}_only'] - for modality in ENSEMBLE_MODALITIES - }, + 'modality_presence_indices': modality_presence_indices, 'threshold': DEFAULT_ENSEMBLE_THRESHOLD, + 'preprocessor': preprocessor, } validation_probabilities = predict_ensemble_probabilities(calibration_bundle, validation_features) return best_threshold(validation_probabilities, validation_labels) @@ -227,8 +248,11 @@ def train_multimodal_ensemble(data_folder, verbose, csv_path): raise ValueError('No valid training samples were extracted. Review feature extraction logs for the skipped records.') feature_indices = get_feature_group_indices(include_demographics=True) - threshold = _calibrate_threshold(features, labels, feature_indices) - models = _fit_ensemble(features, labels, feature_indices) + modality_presence_indices = get_feature_group_indices(include_demographics=False) + preprocessor = _build_preprocessor(len(labels)) + processed_features = np.asarray(preprocessor.fit_transform(features), dtype=np.float32) + threshold = _calibrate_threshold(features, labels, feature_indices, modality_presence_indices) + models = _fit_ensemble(processed_features, labels, feature_indices) return { 'type': 'multimodal_xgb_ensemble', @@ -240,9 +264,9 @@ def train_multimodal_ensemble(data_folder, verbose, csv_path): if name in {'all', 'resp', 'eeg', 'ecg'} }, 'modality_presence_indices': { - modality: get_feature_group_indices(include_demographics=False)[modality].tolist() + modality: modality_presence_indices[modality].tolist() for modality in ENSEMBLE_MODALITIES }, 'models': models, - 'preprocessor': None, + 'preprocessor': preprocessor, } \ No newline at end of file diff --git a/src/resp_processing.py b/src/resp_processing.py index fe0f8e3..bf419b8 100644 --- a/src/resp_processing.py +++ b/src/resp_processing.py @@ -111,7 +111,7 @@ def _summarize_spo2(data, fs): def processResp(physiological_data, physiological_fs, csv_path): alias_groups = _get_resp_alias_groups(csv_path) - results = {feature_name: 0.0 for feature_name in RESP_FEATURE_NAMES} + results = {feature_name: np.nan for feature_name in RESP_FEATURE_NAMES} best_quality = {group_name: -np.inf for group_name in RESP_CHANNEL_GROUPS} for label, signal in physiological_data.items(): From fea70646af29cc29a288741d615143cb1d035544 Mon Sep 17 00:00:00 2001 From: dcajal Date: Tue, 31 Mar 2026 13:13:19 +0200 Subject: [PATCH 59/93] Export cache also in csv --- src/pipeline/features.py | 31 +++++++++++++++++------- src/pipeline/training.py | 52 +++++++++++++++++++++++++++++++++------- team_code.py | 16 ++++++++++--- 3 files changed, 80 insertions(+), 19 deletions(-) diff --git a/src/pipeline/features.py b/src/pipeline/features.py index 6b092be..fc39d88 100644 --- a/src/pipeline/features.py +++ b/src/pipeline/features.py @@ -1,9 +1,9 @@ -import hashlib import os import edfio import joblib import numpy as np +import pandas as pd from src.common.channel_utils import normalize_channel_label from helper_code import HEADERS, PHYSIOLOGICAL_DATA_SUBFOLDER, load_age, load_sex, load_signal_data @@ -35,6 +35,10 @@ ) +def _get_feature_cache_root(data_folder): + return os.path.join(SCRIPT_DIR, FEATURE_CACHE_FOLDER_NAME) + + def _coerce_feature_vector(features): vector = np.asarray(features, dtype=np.float32).reshape(-1) vector[~np.isfinite(vector)] = np.nan @@ -108,16 +112,18 @@ def _load_required_signal_data(edf_path, csv_path): def _get_feature_cache_file(data_folder, site_id, patient_id, session_id): - folder_hash = hashlib.sha1(os.path.abspath(data_folder).encode('utf-8')).hexdigest()[:12] - cache_dir = os.path.join( - SCRIPT_DIR, - FEATURE_CACHE_FOLDER_NAME, - folder_hash, - site_id, - ) + cache_dir = os.path.join(_get_feature_cache_root(data_folder), site_id) return os.path.join(cache_dir, f"{patient_id}_ses-{session_id}.sav") +def get_feature_export_dir(data_folder): + return os.path.join(_get_feature_cache_root(data_folder), 'exports') + + +def _get_feature_cache_csv_file(cache_file): + return f"{os.path.splitext(cache_file)[0]}.csv" + + def _load_cached_feature_vector(cache_file): if not os.path.exists(cache_file): return None @@ -148,6 +154,15 @@ def _save_cached_feature_vector(cache_file, feature_vector): if os.path.exists(temp_cache_file): os.remove(temp_cache_file) + csv_file = _get_feature_cache_csv_file(cache_file) + temp_csv_file = f"{csv_file}.tmp" + try: + pd.DataFrame([payload], columns=get_feature_names()).to_csv(temp_csv_file, index=False) + os.replace(temp_csv_file, csv_file) + finally: + if os.path.exists(temp_csv_file): + os.remove(temp_csv_file) + def extract_demographic_features(data): age_value = data.get(HEADERS['age']) diff --git a/src/pipeline/training.py b/src/pipeline/training.py index 7d18c3b..af28417 100644 --- a/src/pipeline/training.py +++ b/src/pipeline/training.py @@ -39,6 +39,7 @@ def build_training_metadata_cache(patient_data_file): def process_training_record(record, data_folder, demographics_cache, diagnosis_cache, csv_path): patient_id = record[HEADERS['bids_folder']] session_id = record[HEADERS['session_id']] + site_id = record[HEADERS['site_id']] try: patient_data = demographics_cache.get((patient_id, session_id), {}) @@ -51,16 +52,38 @@ def process_training_record(record, data_folder, demographics_cache, diagnosis_c ) label = diagnosis_cache.get(patient_id) + metadata = { + 'patient_id': patient_id, + 'site_id': site_id, + 'session_id': session_id, + } if label == 0 or label == 1: - return patient_id, feature_vector, label, None + return metadata, feature_vector, label, None - return patient_id, None, None, f"Invalid label for {patient_id}. Skipping..." + return metadata, None, None, f"Invalid label for {patient_id}. Skipping..." except FileNotFoundError as exc: - return patient_id, None, None, f"{exc} Skipping..." + return { + 'patient_id': patient_id, + 'site_id': site_id, + 'session_id': session_id, + }, None, None, f"{exc} Skipping..." except Exception as exc: - return patient_id, None, None, f"Error processing {patient_id}: {exc}" + return { + 'patient_id': patient_id, + 'site_id': site_id, + 'session_id': session_id, + }, None, None, f"Error processing {patient_id}: {exc}" + + +def _export_feature_matrix_csv(output_path, metadata_rows, labels, feature_matrix, feature_names): + dataframe = pd.DataFrame(metadata_rows) + dataframe['label'] = labels + feature_frame = pd.DataFrame(feature_matrix, columns=feature_names) + dataframe = pd.concat([dataframe.reset_index(drop=True), feature_frame.reset_index(drop=True)], axis=1) + os.makedirs(os.path.dirname(output_path), exist_ok=True) + dataframe.to_csv(output_path, index=False) def best_threshold(probabilities, labels): @@ -203,7 +226,7 @@ def _calibrate_threshold(feature_matrix, labels, feature_indices, modality_prese return best_threshold(validation_probabilities, validation_labels) -def train_multimodal_ensemble(data_folder, verbose, csv_path): +def train_multimodal_ensemble(data_folder, verbose, csv_path, export_folder=None): patient_data_file = os.path.join(data_folder, DEMOGRAPHICS_FILE) patient_metadata_list = find_patients(patient_data_file) demographics_cache, diagnosis_cache = build_training_metadata_cache(patient_data_file) @@ -214,6 +237,7 @@ def train_multimodal_ensemble(data_folder, verbose, csv_path): features = [] labels = [] + metadata_rows = [] with ThreadPoolExecutor(max_workers=MAX_TRAIN_WORKERS) as executor: results = executor.map( @@ -228,9 +252,9 @@ def train_multimodal_ensemble(data_folder, verbose, csv_path): ) pbar = tqdm(results, total=num_records, desc='Extracting Features', unit='record', disable=not verbose) - for patient_id, feature_vector, label, message in pbar: + for metadata, feature_vector, label, message in pbar: if verbose: - pbar.set_postfix({'patient': patient_id}) + pbar.set_postfix({'patient': metadata['patient_id']}) if message is not None: tqdm.write(f" ! {message}") @@ -238,6 +262,7 @@ def train_multimodal_ensemble(data_folder, verbose, csv_path): features.append(feature_vector) labels.append(label) + metadata_rows.append(metadata) pbar.close() @@ -254,10 +279,17 @@ def train_multimodal_ensemble(data_folder, verbose, csv_path): threshold = _calibrate_threshold(features, labels, feature_indices, modality_presence_indices) models = _fit_ensemble(processed_features, labels, feature_indices) + export_root = export_folder or os.path.join(os.getcwd(), 'feature_exports') + raw_feature_export_path = os.path.join(export_root, 'training_features_raw.csv') + preprocessed_feature_export_path = os.path.join(export_root, 'training_features_preprocessed.csv') + feature_names = list(get_feature_names()) + _export_feature_matrix_csv(raw_feature_export_path, metadata_rows, labels, features, feature_names) + _export_feature_matrix_csv(preprocessed_feature_export_path, metadata_rows, labels, processed_features, feature_names) + return { 'type': 'multimodal_xgb_ensemble', 'threshold': threshold, - 'feature_names': list(get_feature_names()), + 'feature_names': feature_names, 'feature_indices': { name: indices.tolist() for name, indices in feature_indices.items() @@ -269,4 +301,8 @@ def train_multimodal_ensemble(data_folder, verbose, csv_path): }, 'models': models, 'preprocessor': preprocessor, + 'feature_exports': { + 'raw': raw_feature_export_path, + 'preprocessed': preprocessed_feature_export_path, + }, } \ No newline at end of file diff --git a/team_code.py b/team_code.py index 53926be..fb32d0e 100644 --- a/team_code.py +++ b/team_code.py @@ -19,7 +19,7 @@ from helper_code import * from src.pipeline.config import DEFAULT_CSV_PATH -from src.pipeline.features import get_or_create_record_feature_vector +from src.pipeline.features import get_feature_export_dir, get_or_create_record_feature_vector from src.pipeline.training import predict_ensemble_labels, train_multimodal_ensemble ################################################################################ # Path & Constant Configuration (Added for Robustness) @@ -82,14 +82,24 @@ def train_model(data_folder, model_folder, verbose, csv_path=DEFAULT_CSV_PATH): if verbose: print('Training the model on the data...') - model = train_multimodal_ensemble(data_folder, verbose, csv_path) - # Create a folder for the model if it does not already exist. os.makedirs(model_folder, exist_ok=True) + model = train_multimodal_ensemble( + data_folder, + verbose, + csv_path, + export_folder=get_feature_export_dir(data_folder), + ) + # Save the model. save_model(model_folder, model) + feature_exports = model.get('feature_exports', {}) + if verbose and feature_exports: + print(f"Raw features CSV: {feature_exports.get('raw')}") + print(f"Preprocessed features CSV: {feature_exports.get('preprocessed')}") + if verbose: print('Done.') print() From eb28e119d777f1e93c3664de470321a039288403 Mon Sep 17 00:00:00 2001 From: dcajal Date: Tue, 31 Mar 2026 13:13:26 +0200 Subject: [PATCH 60/93] Add data_smoke directory to .gitignore --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 7cc07cb..7a3bfcc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,9 @@ # Dataset data/ +data_smoke/ + +# Cache +.feature_cache/ # Model artifacts model/ From 1bc434e3742213a811080bb6f74d4a4c0df884f5 Mon Sep 17 00:00:00 2001 From: dcajal Date: Wed, 1 Apr 2026 09:31:33 +0200 Subject: [PATCH 61/93] Validate sampling frequency in compute_ecg_features and pan_tompkins functions --- src/lib/ecg_features.py | 3 +++ src/lib/ecg_peak_detection.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/src/lib/ecg_features.py b/src/lib/ecg_features.py index eb964b1..01751e0 100644 --- a/src/lib/ecg_features.py +++ b/src/lib/ecg_features.py @@ -7,6 +7,9 @@ def compute_ecg_features(ecg_signal, fs, ecg_feature_length): + fs = int(round(float(fs))) + if fs <= 0: + return None ecg_signal = ecg_signal - np.mean(ecg_signal) diff --git a/src/lib/ecg_peak_detection.py b/src/lib/ecg_peak_detection.py index 6801f8c..89e4df3 100644 --- a/src/lib/ecg_peak_detection.py +++ b/src/lib/ecg_peak_detection.py @@ -3,6 +3,9 @@ def pan_tompkins(ecg, fs, gr=0): + fs = int(round(float(fs))) + if fs <= 0: + raise ValueError('Sampling frequency must be positive.') ecg = np.asarray(ecg).flatten() delay = 0 From b9b078eaf64bf45b12daf385a14d1dc68cdcef34 Mon Sep 17 00:00:00 2001 From: dcajal Date: Wed, 1 Apr 2026 09:45:51 +0200 Subject: [PATCH 62/93] Add segment duration and stride constants to config and update features module --- src/pipeline/config.py | 2 + src/pipeline/features.py | 144 ++++++++++++++++++++++++++++++--------- 2 files changed, 114 insertions(+), 32 deletions(-) diff --git a/src/pipeline/config.py b/src/pipeline/config.py index 9845da8..4513a56 100644 --- a/src/pipeline/config.py +++ b/src/pipeline/config.py @@ -9,6 +9,8 @@ DEFAULT_CSV_PATH = os.path.join(SCRIPT_DIR, 'channel_table.csv') FEATURE_CACHE_FOLDER_NAME = '.feature_cache' MAX_TRAIN_WORKERS = max(1, min(4, os.cpu_count() or 1)) +SEGMENT_DURATION_SECONDS = 5 * 60 +SEGMENT_STRIDE_SECONDS = 60 * 60 TOTAL_PHYSIOLOGICAL_FEATURE_LENGTH = ( RESP_FEATURE_LENGTH + EEG_FEATURE_LENGTH diff --git a/src/pipeline/features.py b/src/pipeline/features.py index fc39d88..6e74820 100644 --- a/src/pipeline/features.py +++ b/src/pipeline/features.py @@ -11,7 +11,14 @@ from src.eeg_processing import EEG_CHANNEL_SPECS, EEG_FEATURE_LENGTH, EEG_FEATURE_NAMES, processEEG, _get_eeg_aliases from src.resp_processing import RESP_FEATURE_LENGTH, RESP_FEATURE_NAMES, processResp, _get_resp_alias_groups -from .config import DEFAULT_CSV_PATH, FEATURE_CACHE_FOLDER_NAME, SCRIPT_DIR, TOTAL_PHYSIOLOGICAL_FEATURE_LENGTH +from .config import ( + DEFAULT_CSV_PATH, + FEATURE_CACHE_FOLDER_NAME, + SCRIPT_DIR, + SEGMENT_DURATION_SECONDS, + SEGMENT_STRIDE_SECONDS, + TOTAL_PHYSIOLOGICAL_FEATURE_LENGTH, +) REQUIRED_SIGNAL_ALIASES_CACHE = {} @@ -212,39 +219,112 @@ def get_feature_group_indices(include_demographics=False): return groups -def extract_extended_physiological_features(physiological_data, physiological_fs, csv_path=DEFAULT_CSV_PATH): - try: - resp_features = _extract_optional_features( - processResp, - RESP_FEATURE_LENGTH, - physiological_data, - physiological_fs, - csv_path=csv_path, - ) - except Exception: - resp_features = np.full(RESP_FEATURE_LENGTH, np.nan, dtype=np.float32) +def _iter_signal_segments(physiological_data, physiological_fs): + if not physiological_data: + return [] - try: - eeg_features = _extract_optional_features( - processEEG, - EEG_FEATURE_LENGTH, - physiological_data, - physiological_fs, - csv_path=csv_path, - ) - except Exception: - eeg_features = np.full(EEG_FEATURE_LENGTH, np.nan, dtype=np.float32) + durations = [] + for label, signal in physiological_data.items(): + fs = physiological_fs.get(label) + if fs is None or fs <= 0: + continue + durations.append(len(signal) / float(fs)) + + if not durations: + return [] + + max_duration_seconds = max(durations) + segment_starts = np.arange(0.0, max_duration_seconds, SEGMENT_STRIDE_SECONDS, dtype=float) + segments = [] + + for start_seconds in segment_starts: + end_seconds = start_seconds + SEGMENT_DURATION_SECONDS + segment_data = {} + segment_fs = {} + + for label, signal in physiological_data.items(): + fs = physiological_fs.get(label) + if fs is None or fs <= 0: + continue + + start_index = int(round(start_seconds * fs)) + end_index = int(round(end_seconds * fs)) + if start_index >= len(signal): + continue + + sliced_signal = np.asarray(signal[start_index:min(end_index, len(signal))], dtype=float) + if sliced_signal.size == 0: + continue + + segment_data[label] = sliced_signal + segment_fs[label] = fs + + if segment_data: + segments.append((segment_data, segment_fs)) + + return segments + + +def _aggregate_segment_feature_vectors(feature_vectors, expected_length): + if not feature_vectors: + return np.full(expected_length, np.nan, dtype=np.float32) + + matrix = np.asarray(feature_vectors, dtype=np.float32) + with np.errstate(invalid='ignore'): + aggregated = np.nanmean(matrix, axis=0) + aggregated = np.asarray(aggregated, dtype=np.float32) + aggregated[~np.isfinite(aggregated)] = np.nan + return aggregated + + +def _extract_segmented_features(extractor, expected_length, physiological_data, physiological_fs, csv_path): + segments = _iter_signal_segments(physiological_data, physiological_fs) + if not segments: + return np.full(expected_length, np.nan, dtype=np.float32) + + segment_feature_vectors = [] + for segment_data, segment_fs in segments: + try: + vector = _extract_optional_features( + extractor, + expected_length, + segment_data, + segment_fs, + csv_path=csv_path, + ) + except Exception: + continue - try: - ecg_features = _extract_optional_features( - processECG, - ECG_FEATURE_LENGTH, - physiological_data, - physiological_fs, - csv_path=csv_path, - ) - except Exception: - ecg_features = np.full(ECG_FEATURE_LENGTH, np.nan, dtype=np.float32) + if np.all(np.isnan(vector)): + continue + + segment_feature_vectors.append(vector) + + return _aggregate_segment_feature_vectors(segment_feature_vectors, expected_length) + + +def extract_extended_physiological_features(physiological_data, physiological_fs, csv_path=DEFAULT_CSV_PATH): + resp_features = _extract_segmented_features( + processResp, + RESP_FEATURE_LENGTH, + physiological_data, + physiological_fs, + csv_path, + ) + eeg_features = _extract_segmented_features( + processEEG, + EEG_FEATURE_LENGTH, + physiological_data, + physiological_fs, + csv_path, + ) + ecg_features = _extract_segmented_features( + processECG, + ECG_FEATURE_LENGTH, + physiological_data, + physiological_fs, + csv_path, + ) return np.hstack([resp_features, eeg_features, ecg_features]).astype(np.float32) From 3f7343036b78fcb92f6d1e3a8a1bdbe77d82fe23 Mon Sep 17 00:00:00 2001 From: sromagnoli-10 Date: Wed, 1 Apr 2026 11:11:28 +0200 Subject: [PATCH 63/93] Update mean to nanmean in ecg_hrv_features.py --- src/lib/ecg_hrv_features.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/ecg_hrv_features.py b/src/lib/ecg_hrv_features.py index e8ede73..a1f886d 100644 --- a/src/lib/ecg_hrv_features.py +++ b/src/lib/ecg_hrv_features.py @@ -98,7 +98,7 @@ def compute_hrv_hrf(nn_intervals, sampling_frequency): frequencies = np.linspace(0.01, 0.5, 1000) angular_frequencies = 2 * np.pi * frequencies - detrended_nn = window_nn - np.mean(window_nn) + detrended_nn = window_nn - np.nanmean(window_nn) power_spectrum = lombscargle(window_time, detrended_nn, angular_frequencies, normalize=True) @@ -120,4 +120,4 @@ def compute_hrv_hrf(nn_intervals, sampling_frequency): "HF": hf, } - return results \ No newline at end of file + return results From 66bbd24695a53670526eae32c1a393785be6bf96 Mon Sep 17 00:00:00 2001 From: dcajal Date: Wed, 1 Apr 2026 11:31:19 +0200 Subject: [PATCH 64/93] Segment feature aggregation --- src/ecg_processing.py | 38 ++++++--------- src/eeg_processing.py | 8 +-- src/lib/ecg_features.py | 102 +++++++++++---------------------------- src/pipeline/features.py | 71 ++++++++++++++++++++------- src/resp_processing.py | 50 ++++++++----------- 5 files changed, 121 insertions(+), 148 deletions(-) diff --git a/src/ecg_processing.py b/src/ecg_processing.py index 044f54c..c0f8243 100644 --- a/src/ecg_processing.py +++ b/src/ecg_processing.py @@ -4,25 +4,19 @@ ECG_KEYWORDS = ['ecg', 'ekg'] -ECG_FEATURE_NAMES = [ - "PIP_med", - "PIP_std", - "PNNLS_med", - "PNNLS_std", - "PNNSS_med", - "PNNSS_std", - "AVNN_med", - "AVNN_std", - "SDNN_med", - "SDNN_std", - "RMSSD_med", - "RMSSD_std", - "HF_med", - "HF_std", - "ECTOPIC_med", - "ECTOPIC_std" +ECG_SEGMENT_FEATURE_NAMES = [ + "PIP", + "PNNLS", + "PNNSS", + "AVNN", + "SDNN", + "RMSSD", + "HF", + "ECTOPIC" ] -ECG_FEATURE_LENGTH = len(ECG_FEATURE_NAMES) +ECG_SEGMENT_FEATURE_LENGTH = len(ECG_SEGMENT_FEATURE_NAMES) +ECG_FEATURE_NAMES = ECG_SEGMENT_FEATURE_NAMES +ECG_FEATURE_LENGTH = ECG_SEGMENT_FEATURE_LENGTH def _find_ecg_channel(physiological_data): for label in physiological_data.keys(): @@ -33,7 +27,7 @@ def _find_ecg_channel(physiological_data): def processECG(physiological_data, physiological_fs, csv_path): - results = np.full(ECG_FEATURE_LENGTH, np.nan, dtype=np.float32) + results = np.full(ECG_SEGMENT_FEATURE_LENGTH, np.nan, dtype=np.float32) ecg_label = _find_ecg_channel(physiological_data) @@ -50,7 +44,7 @@ def processECG(physiological_data, physiological_fs, csv_path): return results try: - values = compute_ecg_features(ecg_signal, fs, ECG_FEATURE_LENGTH) + values = compute_ecg_features(ecg_signal, fs, ECG_SEGMENT_FEATURE_LENGTH) if values is None or len(values) == 0: return results @@ -58,8 +52,8 @@ def processECG(physiological_data, physiological_fs, csv_path): values = np.asarray(values, dtype=np.float32) values[~np.isfinite(values)] = np.nan - if len(values) >= ECG_FEATURE_LENGTH: - results[:] = values[:ECG_FEATURE_LENGTH] + if len(values) >= ECG_SEGMENT_FEATURE_LENGTH: + results[:] = values[:ECG_SEGMENT_FEATURE_LENGTH] else: results[:len(values)] = values diff --git a/src/eeg_processing.py b/src/eeg_processing.py index 37b74f5..6568412 100644 --- a/src/eeg_processing.py +++ b/src/eeg_processing.py @@ -61,8 +61,10 @@ ('F3-M2', 'variability_Delta'), ('F4-M1', 'variability_Delta'), ] -EEG_FEATURE_NAMES = [f'{channel}_{metric}' for channel, metric in EEG_FEATURE_SPECS] -EEG_FEATURE_LENGTH = len(EEG_FEATURE_NAMES) +EEG_SEGMENT_FEATURE_NAMES = [f'{channel}_{metric}' for channel, metric in EEG_FEATURE_SPECS] +EEG_SEGMENT_FEATURE_LENGTH = len(EEG_SEGMENT_FEATURE_NAMES) +EEG_FEATURE_NAMES = EEG_SEGMENT_FEATURE_NAMES +EEG_FEATURE_LENGTH = EEG_SEGMENT_FEATURE_LENGTH EEG_ALIASES_CACHE = {} @@ -167,7 +169,7 @@ def processEEG(physiological_data, physiological_fs, csv_path): channel_profiles[channel_name] = metrics if not channel_profiles: - return np.full(EEG_FEATURE_LENGTH, np.nan, dtype=np.float32) + return np.full(EEG_SEGMENT_FEATURE_LENGTH, np.nan, dtype=np.float32) values = [] for channel_name, metric_name in EEG_FEATURE_SPECS: diff --git a/src/lib/ecg_features.py b/src/lib/ecg_features.py index 01751e0..722d28b 100644 --- a/src/lib/ecg_features.py +++ b/src/lib/ecg_features.py @@ -1,4 +1,5 @@ import numpy as np +from typing import cast from scipy.signal import butter, filtfilt, resample from .ecg_peak_detection import pan_tompkins from .ecg_hrv_features import compute_hrv_hrf @@ -20,92 +21,45 @@ def compute_ecg_features(ecg_signal, fs, ecg_feature_length): ecg_signal = resample(ecg_signal, num_samples) fs = target_fs - win_sec = 300 - win_samples = int(win_sec * fs) + if np.sum(np.isnan(ecg_signal)) != 0 or np.sum(ecg_signal == 0) > 0.2 * len(ecg_signal): + return np.full(ecg_feature_length, np.nan, dtype=np.float32) - total_samples = len(ecg_signal) - n_windows = total_samples // win_samples + b, a = cast(tuple[np.ndarray, np.ndarray], butter(3, [59.5/(fs/2), 60.5/(fs/2)], btype='bandstop', output='ba')) + ecg_signal = filtfilt(b, a, ecg_signal) - if n_windows == 0: - print("Signal too short.") - return None - - valid_windows = [] - - for window_index in range(n_windows): - - idx_start = window_index * win_samples - idx_end = (window_index + 1) * win_samples - - ecg_win = ecg_signal[idx_start:idx_end] - - if np.sum(np.isnan(ecg_win)) != 0 or np.sum(ecg_win == 0) > 0.2 * len(ecg_win): - continue - - valid_windows.append(window_index) - - hrv_all = [] - - for window_index in valid_windows: - - idx_start = window_index * win_samples - idx_end = (window_index + 1) * win_samples - - ecg_win = ecg_signal[idx_start:idx_end] - - ecg_win = ecg_win - np.mean(ecg_win) - - b, a = butter(3, [59.5/(fs/2), 60.5/(fs/2)], btype='bandstop') - ecg_win = filtfilt(b, a, ecg_win) - - b, a = butter(3, 0.5/(fs/2), btype='high') - ecg_win = filtfilt(b, a, ecg_win) - - b, a = butter(3, 45/(fs/2), btype='low') - ecg_win = filtfilt(b, a, ecg_win) - - _, r_locs, _ = pan_tompkins(ecg_win, fs, 0) - - if len(r_locs) < 150: - continue - - nn_intervals = np.diff(r_locs) / fs - - nn_intervals, ectopic_perc = remove_ectopic_beats(nn_intervals, 40, 0.10) + b, a = cast(tuple[np.ndarray, np.ndarray], butter(3, 0.5/(fs/2), btype='high', output='ba')) + ecg_signal = filtfilt(b, a, ecg_signal) - nn_intervals = interpolate_nn_pchip(nn_intervals, 2) + b, a = cast(tuple[np.ndarray, np.ndarray], butter(3, 45/(fs/2), btype='low', output='ba')) + ecg_signal = filtfilt(b, a, ecg_signal) - valid_ratio = np.sum(~np.isnan(nn_intervals)) / len(nn_intervals) - nn_intervals = nn_intervals[~np.isnan(nn_intervals)] + _, r_locs, _ = pan_tompkins(ecg_signal, fs, 0) - if valid_ratio < 0.75: - continue + if len(r_locs) < 150: + return np.full(ecg_feature_length, np.nan, dtype=np.float32) - metrics = compute_hrv_hrf(nn_intervals, fs) + nn_intervals = np.diff(r_locs) / fs - hrv_all.append([ - metrics["PIP"], metrics["PNNLS"], metrics["PNNSS"], - metrics["AVNN"], metrics["SDNN"], metrics["RMSSD"], metrics["HF"], - ectopic_perc - ]) + nn_intervals, ectopic_perc = remove_ectopic_beats(nn_intervals, 40, 0.10) + nn_intervals = interpolate_nn_pchip(nn_intervals, 2) - if len(hrv_all) == 0: - return np.zeros(ecg_feature_length, dtype=np.float32) + valid_ratio = np.sum(~np.isnan(nn_intervals)) / len(nn_intervals) + nn_intervals = nn_intervals[~np.isnan(nn_intervals)] - hrv_all = np.array(hrv_all) + if valid_ratio < 0.75 or len(nn_intervals) == 0: + return np.full(ecg_feature_length, np.nan, dtype=np.float32) - median_vals = np.nanmedian(hrv_all, axis=0) - std_vals = np.nanstd(hrv_all, axis=0) + metrics = compute_hrv_hrf(nn_intervals, fs) features = np.array([ - median_vals[0], std_vals[0], - median_vals[1], std_vals[1], - median_vals[2], std_vals[2], - median_vals[3], std_vals[3], - median_vals[4], std_vals[4], - median_vals[5], std_vals[5], - median_vals[6], std_vals[6], - median_vals[7], std_vals[7], + metrics["PIP"], + metrics["PNNLS"], + metrics["PNNSS"], + metrics["AVNN"], + metrics["SDNN"], + metrics["RMSSD"], + metrics["HF"], + ectopic_perc, ], dtype=np.float32) return features \ No newline at end of file diff --git a/src/pipeline/features.py b/src/pipeline/features.py index 6e74820..e9aae0f 100644 --- a/src/pipeline/features.py +++ b/src/pipeline/features.py @@ -17,22 +17,32 @@ SCRIPT_DIR, SEGMENT_DURATION_SECONDS, SEGMENT_STRIDE_SECONDS, - TOTAL_PHYSIOLOGICAL_FEATURE_LENGTH, ) REQUIRED_SIGNAL_ALIASES_CACHE = {} +SEGMENT_AGGREGATION_NAMES = ('Max', 'Min', 'Mean', 'Median', 'Std') DEMOGRAPHIC_FEATURE_NAMES = ( 'Age', 'Sex_Female', 'Sex_Male', 'Sex_Unknown', ) + + +def _build_aggregated_feature_names(segment_feature_names): + return tuple( + f'{feature_name}_{aggregation_name}' + for feature_name in segment_feature_names + for aggregation_name in SEGMENT_AGGREGATION_NAMES + ) + + FEATURE_NAME_GROUPS = { 'demographics': DEMOGRAPHIC_FEATURE_NAMES, - 'resp': tuple(RESP_FEATURE_NAMES), - 'eeg': tuple(EEG_FEATURE_NAMES), - 'ecg': tuple(ECG_FEATURE_NAMES), + 'resp': _build_aggregated_feature_names(RESP_FEATURE_NAMES), + 'eeg': _build_aggregated_feature_names(EEG_FEATURE_NAMES), + 'ecg': _build_aggregated_feature_names(ECG_FEATURE_NAMES), } FEATURE_NAMES = ( *FEATURE_NAME_GROUPS['demographics'], @@ -40,6 +50,11 @@ *FEATURE_NAME_GROUPS['eeg'], *FEATURE_NAME_GROUPS['ecg'], ) +TOTAL_PHYSIOLOGICAL_FEATURE_LENGTH = ( + len(FEATURE_NAME_GROUPS['resp']) + + len(FEATURE_NAME_GROUPS['eeg']) + + len(FEATURE_NAME_GROUPS['ecg']) +) def _get_feature_cache_root(data_folder): @@ -146,7 +161,10 @@ def _load_cached_feature_vector(cache_file): if payload is None: return None - return _coerce_feature_vector(payload) + vector = _coerce_feature_vector(payload) + if vector.size != len(FEATURE_NAMES): + return None + return vector def _save_cached_feature_vector(cache_file, feature_vector): @@ -265,22 +283,39 @@ def _iter_signal_segments(physiological_data, physiological_fs): return segments -def _aggregate_segment_feature_vectors(feature_vectors, expected_length): +def _aggregate_segment_feature_vectors(feature_vectors, segment_feature_names): + aggregated_length = len(segment_feature_names) * len(SEGMENT_AGGREGATION_NAMES) if not feature_vectors: - return np.full(expected_length, np.nan, dtype=np.float32) + return np.full(aggregated_length, np.nan, dtype=np.float32) matrix = np.asarray(feature_vectors, dtype=np.float32) - with np.errstate(invalid='ignore'): - aggregated = np.nanmean(matrix, axis=0) - aggregated = np.asarray(aggregated, dtype=np.float32) - aggregated[~np.isfinite(aggregated)] = np.nan - return aggregated + aggregated_values = [] + + for column_index in range(matrix.shape[1]): + column_values = matrix[:, column_index] + finite_values = column_values[np.isfinite(column_values)] + + if finite_values.size == 0: + aggregated_values.extend([np.nan] * len(SEGMENT_AGGREGATION_NAMES)) + continue + + aggregated_values.extend([ + float(np.max(finite_values)), + float(np.min(finite_values)), + float(np.mean(finite_values)), + float(np.median(finite_values)), + float(np.std(finite_values)), + ]) + + return np.asarray(aggregated_values, dtype=np.float32) -def _extract_segmented_features(extractor, expected_length, physiological_data, physiological_fs, csv_path): +def _extract_segmented_features(extractor, segment_feature_names, physiological_data, physiological_fs, csv_path): + expected_length = len(segment_feature_names) segments = _iter_signal_segments(physiological_data, physiological_fs) if not segments: - return np.full(expected_length, np.nan, dtype=np.float32) + aggregated_length = expected_length * len(SEGMENT_AGGREGATION_NAMES) + return np.full(aggregated_length, np.nan, dtype=np.float32) segment_feature_vectors = [] for segment_data, segment_fs in segments: @@ -300,27 +335,27 @@ def _extract_segmented_features(extractor, expected_length, physiological_data, segment_feature_vectors.append(vector) - return _aggregate_segment_feature_vectors(segment_feature_vectors, expected_length) + return _aggregate_segment_feature_vectors(segment_feature_vectors, segment_feature_names) def extract_extended_physiological_features(physiological_data, physiological_fs, csv_path=DEFAULT_CSV_PATH): resp_features = _extract_segmented_features( processResp, - RESP_FEATURE_LENGTH, + RESP_FEATURE_NAMES, physiological_data, physiological_fs, csv_path, ) eeg_features = _extract_segmented_features( processEEG, - EEG_FEATURE_LENGTH, + EEG_FEATURE_NAMES, physiological_data, physiological_fs, csv_path, ) ecg_features = _extract_segmented_features( processECG, - ECG_FEATURE_LENGTH, + ECG_FEATURE_NAMES, physiological_data, physiological_fs, csv_path, diff --git a/src/resp_processing.py b/src/resp_processing.py index bf419b8..11b474f 100644 --- a/src/resp_processing.py +++ b/src/resp_processing.py @@ -4,20 +4,18 @@ from src.common.signal_utils import resample_signal RESP_CHANNEL_GROUPS = ("Abdomen", "Chest", "Nasal", "Flow") -RESP_FEATURE_NAMES = [ - f"{group}_Peakedness_{metric}" +RESP_SEGMENT_FEATURE_NAMES = [ + f"{group}_Peakedness" for group in RESP_CHANNEL_GROUPS - for metric in ("Max", "Min", "Mean", "Median", "Std") ] + [ - "SpO2_Max", - "SpO2_Min", - "SpO2_Mean", - "SpO2_Std", + "SpO2", "CET90", - "ODI_Mean", + "ODI", "ODI_deepness", ] -RESP_FEATURE_LENGTH = len(RESP_FEATURE_NAMES) +RESP_SEGMENT_FEATURE_LENGTH = len(RESP_SEGMENT_FEATURE_NAMES) +RESP_FEATURE_NAMES = RESP_SEGMENT_FEATURE_NAMES +RESP_FEATURE_LENGTH = RESP_SEGMENT_FEATURE_LENGTH RESP_ALIAS_GROUPS_CACHE = {} @@ -63,21 +61,15 @@ def _compute_resp_quality(used, hat_br): return float(np.mean(np.isfinite(hat_br))) -def _summarize_peakedness(hat_br): +def _compute_peakedness_metric(hat_br): finite_values = np.asarray(hat_br, dtype=float) finite_values = finite_values[np.isfinite(finite_values)] if finite_values.size == 0: - return None - return { - 'Max': float(np.max(finite_values)), - 'Min': float(np.min(finite_values)), - 'Mean': float(np.mean(finite_values)), - 'Median': float(np.median(finite_values)), - 'Std': float(np.std(finite_values)), - } + return np.nan + return float(np.mean(finite_values)) -def _summarize_spo2(data, fs): +def _compute_spo2_segment_metrics(data, fs): if data.size == 0: return {} working = np.asarray(data, dtype=float).copy() @@ -99,19 +91,16 @@ def _summarize_spo2(data, fs): odi_mean, odi_deepness = resp_features.odi_application(desaturation_mask, fs) return { - 'SpO2_Max': float(np.max(valid)), - 'SpO2_Min': float(np.min(valid)), - 'SpO2_Mean': float(np.mean(valid)), - 'SpO2_Std': float(np.std(valid)), + 'SpO2': float(np.mean(valid)), 'CET90': cet90, - 'ODI_Mean': float(odi_mean), + 'ODI': float(odi_mean), 'ODI_deepness': float(odi_deepness), } def processResp(physiological_data, physiological_fs, csv_path): alias_groups = _get_resp_alias_groups(csv_path) - results = {feature_name: np.nan for feature_name in RESP_FEATURE_NAMES} + results = {feature_name: np.nan for feature_name in RESP_SEGMENT_FEATURE_NAMES} best_quality = {group_name: -np.inf for group_name in RESP_CHANNEL_GROUPS} for label, signal in physiological_data.items(): @@ -126,7 +115,7 @@ def processResp(physiological_data, physiological_fs, csv_path): resampled = np.nan_to_num(resampled, nan=0.0, posinf=0.0, neginf=0.0) if group_name == 'SpO2': - results.update(_summarize_spo2(resampled, fs)) + results.update(_compute_spo2_segment_metrics(resampled, fs)) continue try: @@ -138,8 +127,8 @@ def processResp(physiological_data, physiological_fs, csv_path): except Exception: continue - summary = _summarize_peakedness(hat_br) - if summary is None: + peakedness_metric = _compute_peakedness_metric(hat_br) + if not np.isfinite(peakedness_metric): continue quality = _compute_resp_quality(used, hat_br) @@ -147,7 +136,6 @@ def processResp(physiological_data, physiological_fs, csv_path): continue best_quality[group_name] = quality - for metric_name, metric_value in summary.items(): - results[f'{group_name}_Peakedness_{metric_name}'] = metric_value + results[f'{group_name}_Peakedness'] = peakedness_metric - return np.array([results[name] for name in RESP_FEATURE_NAMES], dtype=np.float32) + return np.array([results[name] for name in RESP_SEGMENT_FEATURE_NAMES], dtype=np.float32) From 6a1927d93e3f8ea72bc1633e0d0c9e812843511b Mon Sep 17 00:00:00 2001 From: dcajal Date: Wed, 1 Apr 2026 11:48:22 +0200 Subject: [PATCH 65/93] Use np.nanmean for detrending in compute_hrv_hrf function --- src/lib/ecg_hrv_features.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/ecg_hrv_features.py b/src/lib/ecg_hrv_features.py index e8ede73..da1b0b1 100644 --- a/src/lib/ecg_hrv_features.py +++ b/src/lib/ecg_hrv_features.py @@ -98,7 +98,7 @@ def compute_hrv_hrf(nn_intervals, sampling_frequency): frequencies = np.linspace(0.01, 0.5, 1000) angular_frequencies = 2 * np.pi * frequencies - detrended_nn = window_nn - np.mean(window_nn) + detrended_nn = window_nn - np.nanmean(window_nn) power_spectrum = lombscargle(window_time, detrended_nn, angular_frequencies, normalize=True) From c9344a3df41a226bfe4aae8fd66f5eceafcf8c2b Mon Sep 17 00:00:00 2001 From: dcajal Date: Wed, 1 Apr 2026 11:54:15 +0200 Subject: [PATCH 66/93] Update minimum intervals check in compute_hrv_hrf function --- src/lib/ecg_hrv_features.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lib/ecg_hrv_features.py b/src/lib/ecg_hrv_features.py index da1b0b1..7b6e7d9 100644 --- a/src/lib/ecg_hrv_features.py +++ b/src/lib/ecg_hrv_features.py @@ -58,6 +58,7 @@ def compute_hrv_hrf(nn_intervals, sampling_frequency): pnnss = np.nan window_length = 300 + minimum_intervals_per_window = window_length / 2 elapsed_time = np.cumsum(nn_intervals) @@ -72,7 +73,7 @@ def compute_hrv_hrf(nn_intervals, sampling_frequency): window_indices = np.where((elapsed_time >= window_start) & (elapsed_time < window_end))[0] - if len(window_indices) >= 150: + if len(window_indices) >= minimum_intervals_per_window: window_nn = nn_intervals[window_indices] avnn_values.append(np.nanmean(window_nn)) From 4c2259bad229f660802fd0367cd78b14f8d097ed Mon Sep 17 00:00:00 2001 From: sromagnoli-10 Date: Wed, 1 Apr 2026 12:30:29 +0200 Subject: [PATCH 67/93] Update length ecg as variable to use it in ecg_ecg_hrv_features and do not compute it 3 times ecg_features.py --- src/lib/ecg_features.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/lib/ecg_features.py b/src/lib/ecg_features.py index 722d28b..a18dedf 100644 --- a/src/lib/ecg_features.py +++ b/src/lib/ecg_features.py @@ -15,13 +15,14 @@ def compute_ecg_features(ecg_signal, fs, ecg_feature_length): ecg_signal = ecg_signal - np.mean(ecg_signal) target_fs = 200 - + length_ecg=len(ecg_signal) + if fs != target_fs: - num_samples = int(len(ecg_signal) * target_fs / fs) + num_samples = int(length_ecg * target_fs / fs) ecg_signal = resample(ecg_signal, num_samples) fs = target_fs - if np.sum(np.isnan(ecg_signal)) != 0 or np.sum(ecg_signal == 0) > 0.2 * len(ecg_signal): + if np.sum(np.isnan(ecg_signal)) != 0 or np.sum(ecg_signal == 0) > 0.2 * length_ecg: return np.full(ecg_feature_length, np.nan, dtype=np.float32) b, a = cast(tuple[np.ndarray, np.ndarray], butter(3, [59.5/(fs/2), 60.5/(fs/2)], btype='bandstop', output='ba')) @@ -49,7 +50,7 @@ def compute_ecg_features(ecg_signal, fs, ecg_feature_length): if valid_ratio < 0.75 or len(nn_intervals) == 0: return np.full(ecg_feature_length, np.nan, dtype=np.float32) - metrics = compute_hrv_hrf(nn_intervals, fs) + metrics = compute_hrv_hrf(nn_intervals, fs, length_ecg) features = np.array([ metrics["PIP"], @@ -62,4 +63,4 @@ def compute_ecg_features(ecg_signal, fs, ecg_feature_length): ectopic_perc, ], dtype=np.float32) - return features \ No newline at end of file + return features From 259ee8b347ac6b4f8ee9caa3930e347eccfba39e Mon Sep 17 00:00:00 2001 From: sromagnoli-10 Date: Wed, 1 Apr 2026 12:33:09 +0200 Subject: [PATCH 68/93] Update length window ecg_hrv_features.py --- src/lib/ecg_hrv_features.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lib/ecg_hrv_features.py b/src/lib/ecg_hrv_features.py index 7b6e7d9..2193b55 100644 --- a/src/lib/ecg_hrv_features.py +++ b/src/lib/ecg_hrv_features.py @@ -1,7 +1,7 @@ import numpy as np from scipy.signal import lombscargle -def compute_hrv_hrf(nn_intervals, sampling_frequency): +def compute_hrv_hrf(nn_intervals, sampling_frequency, length_ecg): """Compute time-domain and HF HRV metrics from NN intervals.""" nn_intervals = np.asarray(nn_intervals).flatten() @@ -57,7 +57,7 @@ def compute_hrv_hrf(nn_intervals, sampling_frequency): pnnls = np.nan pnnss = np.nan - window_length = 300 + window_length = int(length_ecg / fs) minimum_intervals_per_window = window_length / 2 elapsed_time = np.cumsum(nn_intervals) @@ -121,4 +121,4 @@ def compute_hrv_hrf(nn_intervals, sampling_frequency): "HF": hf, } - return results \ No newline at end of file + return results From 31180c24d255a2af73c622199ad49f32956a9799 Mon Sep 17 00:00:00 2001 From: sromagnoli-10 Date: Wed, 1 Apr 2026 12:37:48 +0200 Subject: [PATCH 69/93] Update sampling frequency instead of fs ecg_hrv_features.py --- src/lib/ecg_hrv_features.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/ecg_hrv_features.py b/src/lib/ecg_hrv_features.py index 2193b55..034e6b9 100644 --- a/src/lib/ecg_hrv_features.py +++ b/src/lib/ecg_hrv_features.py @@ -57,7 +57,7 @@ def compute_hrv_hrf(nn_intervals, sampling_frequency, length_ecg): pnnls = np.nan pnnss = np.nan - window_length = int(length_ecg / fs) + window_length = int(length_ecg / sampling_frequency) minimum_intervals_per_window = window_length / 2 elapsed_time = np.cumsum(nn_intervals) From 35e37a784b4ff463e24a67437f4014659a3e35c1 Mon Sep 17 00:00:00 2001 From: dcajal Date: Wed, 1 Apr 2026 13:05:27 +0200 Subject: [PATCH 70/93] Refactor compute_ecg_features and compute_hrv_hrf functions for improved clarity and performance --- src/lib/ecg_features.py | 17 +++++++--- src/lib/ecg_hrv_features.py | 62 ++++++++----------------------------- 2 files changed, 26 insertions(+), 53 deletions(-) diff --git a/src/lib/ecg_features.py b/src/lib/ecg_features.py index a18dedf..b07582a 100644 --- a/src/lib/ecg_features.py +++ b/src/lib/ecg_features.py @@ -12,16 +12,22 @@ def compute_ecg_features(ecg_signal, fs, ecg_feature_length): if fs <= 0: return None + signal_duration_seconds = len(ecg_signal) / fs + ecg_signal = ecg_signal - np.mean(ecg_signal) target_fs = 200 - length_ecg=len(ecg_signal) + length_ecg = len(ecg_signal) if fs != target_fs: num_samples = int(length_ecg * target_fs / fs) ecg_signal = resample(ecg_signal, num_samples) fs = target_fs + length_ecg = len(ecg_signal) + window_length_seconds = max(1, int(signal_duration_seconds)) + minimum_intervals_per_window = max(1, int(np.ceil(window_length_seconds / 2))) + if np.sum(np.isnan(ecg_signal)) != 0 or np.sum(ecg_signal == 0) > 0.2 * length_ecg: return np.full(ecg_feature_length, np.nan, dtype=np.float32) @@ -36,7 +42,7 @@ def compute_ecg_features(ecg_signal, fs, ecg_feature_length): _, r_locs, _ = pan_tompkins(ecg_signal, fs, 0) - if len(r_locs) < 150: + if len(r_locs) - 1 < minimum_intervals_per_window: return np.full(ecg_feature_length, np.nan, dtype=np.float32) nn_intervals = np.diff(r_locs) / fs @@ -44,13 +50,16 @@ def compute_ecg_features(ecg_signal, fs, ecg_feature_length): nn_intervals, ectopic_perc = remove_ectopic_beats(nn_intervals, 40, 0.10) nn_intervals = interpolate_nn_pchip(nn_intervals, 2) + if len(nn_intervals) == 0: + return np.full(ecg_feature_length, np.nan, dtype=np.float32) + valid_ratio = np.sum(~np.isnan(nn_intervals)) / len(nn_intervals) nn_intervals = nn_intervals[~np.isnan(nn_intervals)] - if valid_ratio < 0.75 or len(nn_intervals) == 0: + if valid_ratio < 0.75 or len(nn_intervals) < minimum_intervals_per_window: return np.full(ecg_feature_length, np.nan, dtype=np.float32) - metrics = compute_hrv_hrf(nn_intervals, fs, length_ecg) + metrics = compute_hrv_hrf(nn_intervals, fs) features = np.array([ metrics["PIP"], diff --git a/src/lib/ecg_hrv_features.py b/src/lib/ecg_hrv_features.py index 034e6b9..f3c2c4b 100644 --- a/src/lib/ecg_hrv_features.py +++ b/src/lib/ecg_hrv_features.py @@ -1,7 +1,8 @@ import numpy as np +from scipy.integrate import trapezoid from scipy.signal import lombscargle -def compute_hrv_hrf(nn_intervals, sampling_frequency, length_ecg): +def compute_hrv_hrf(nn_intervals, sampling_frequency): """Compute time-domain and HF HRV metrics from NN intervals.""" nn_intervals = np.asarray(nn_intervals).flatten() @@ -57,59 +58,22 @@ def compute_hrv_hrf(nn_intervals, sampling_frequency, length_ecg): pnnls = np.nan pnnss = np.nan - window_length = int(length_ecg / sampling_frequency) - minimum_intervals_per_window = window_length / 2 + avnn = np.nanmean(nn_intervals) if len(nn_intervals) > 0 else np.nan + sdnn = np.nanstd(nn_intervals, ddof=1) if len(nn_intervals) > 1 else np.nan - elapsed_time = np.cumsum(nn_intervals) - - avnn_values = [] - sdnn_values = [] - rmssd_values = [] - - index = 0 - while index < len(nn_intervals): - window_start = elapsed_time[index] - window_end = window_start + window_length - - window_indices = np.where((elapsed_time >= window_start) & (elapsed_time < window_end))[0] - - if len(window_indices) >= minimum_intervals_per_window: - window_nn = nn_intervals[window_indices] - - avnn_values.append(np.nanmean(window_nn)) - sdnn_values.append(np.nanstd(window_nn, ddof=1)) - - diff_nn = np.diff(window_nn) - rmssd_values.append(np.sqrt(np.nanmean(diff_nn**2))) - - next_indices = np.where(elapsed_time >= window_end)[0] - if len(next_indices) == 0: - break - index = next_indices[0] - - avnn = np.nanmean(avnn_values) if len(avnn_values) > 0 else np.nan - sdnn = np.nanmean(sdnn_values) if len(sdnn_values) > 0 else np.nan - rmssd = np.nanmean(rmssd_values) if len(rmssd_values) > 0 else np.nan - - hf_values = [] - - for _ in range(len(avnn_values)): - window_nn = nn_intervals.copy() - window_time = np.cumsum(window_nn) + diff_nn = np.diff(nn_intervals) + rmssd = np.sqrt(np.nanmean(diff_nn**2)) if len(diff_nn) > 0 else np.nan + if len(nn_intervals) > 1: + elapsed_time = np.cumsum(nn_intervals) frequencies = np.linspace(0.01, 0.5, 1000) angular_frequencies = 2 * np.pi * frequencies - detrended_nn = window_nn - np.nanmean(window_nn) - - power_spectrum = lombscargle(window_time, detrended_nn, angular_frequencies, normalize=True) - + detrended_nn = nn_intervals - np.nanmean(nn_intervals) + power_spectrum = lombscargle(elapsed_time, detrended_nn, angular_frequencies, normalize=True) hf_band = (frequencies >= 0.15) & (frequencies <= 0.4) - - hf_power = np.trapezoid(power_spectrum[hf_band], frequencies[hf_band]) - - hf_values.append(hf_power) - - hf = np.nanmean(hf_values) if len(hf_values) > 0 else np.nan + hf = trapezoid(power_spectrum[hf_band], frequencies[hf_band]) + else: + hf = np.nan results = { "PIP": pip, From 28aa94bc58329eca32111cc2148e9502096eebba Mon Sep 17 00:00:00 2001 From: dcajal Date: Wed, 1 Apr 2026 13:43:29 +0200 Subject: [PATCH 71/93] Refactor demographic feature extraction and segment iteration to avoid last segment if it is incomplete --- src/pipeline/features.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/src/pipeline/features.py b/src/pipeline/features.py index e9aae0f..08b736f 100644 --- a/src/pipeline/features.py +++ b/src/pipeline/features.py @@ -24,9 +24,7 @@ SEGMENT_AGGREGATION_NAMES = ('Max', 'Min', 'Mean', 'Median', 'Std') DEMOGRAPHIC_FEATURE_NAMES = ( 'Age', - 'Sex_Female', - 'Sex_Male', - 'Sex_Unknown', + 'Sex', ) @@ -200,13 +198,13 @@ def extract_demographic_features(data): age = np.array([age], dtype=np.float32) sex = load_sex(data) - sex_vec = np.zeros(3, dtype=np.float32) - if sex == 'Female': - sex_vec[0] = 1 - elif sex == 'Male': - sex_vec[1] = 1 + if sex == 'Male': + sex_value = 1.0 + elif sex == 'Female': + sex_value = 0.0 else: - sex_vec[2] = 1 + sex_value = np.nan + sex_vec = np.array([sex_value], dtype=np.float32) return np.concatenate([age, sex_vec]).astype(np.float32) @@ -252,7 +250,11 @@ def _iter_signal_segments(physiological_data, physiological_fs): return [] max_duration_seconds = max(durations) - segment_starts = np.arange(0.0, max_duration_seconds, SEGMENT_STRIDE_SECONDS, dtype=float) + last_full_segment_start = max_duration_seconds - SEGMENT_DURATION_SECONDS + if last_full_segment_start < 0: + return [] + + segment_starts = np.arange(0.0, last_full_segment_start + 1e-9, SEGMENT_STRIDE_SECONDS, dtype=float) segments = [] for start_seconds in segment_starts: @@ -267,10 +269,10 @@ def _iter_signal_segments(physiological_data, physiological_fs): start_index = int(round(start_seconds * fs)) end_index = int(round(end_seconds * fs)) - if start_index >= len(signal): + if start_index >= len(signal) or end_index > len(signal): continue - sliced_signal = np.asarray(signal[start_index:min(end_index, len(signal))], dtype=float) + sliced_signal = np.asarray(signal[start_index:end_index], dtype=float) if sliced_signal.size == 0: continue From 39863089c211cd37e1b5016b58a33b4ee60e39a1 Mon Sep 17 00:00:00 2001 From: sromagnoli-10 Date: Wed, 1 Apr 2026 18:30:55 +0200 Subject: [PATCH 72/93] Update resp_features into Resp_features during importresp_processing.py --- src/resp_processing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/resp_processing.py b/src/resp_processing.py index 11b474f..b5a89f9 100644 --- a/src/resp_processing.py +++ b/src/resp_processing.py @@ -1,4 +1,4 @@ -from .lib import resp_features +from .lib import Resp_features import numpy as np from src.common.channel_utils import get_cached_channel_table, normalize_channel_label, split_channel_aliases from src.common.signal_utils import resample_signal From 5179cb3f996ad71d3bea1f0c819eef1c699cad97 Mon Sep 17 00:00:00 2001 From: sromagnoli-10 Date: Thu, 2 Apr 2026 16:43:52 +0200 Subject: [PATCH 73/93] Rename Resp_features.py to resp_features.py --- src/lib/{Resp_features.py => resp_features.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename src/lib/{Resp_features.py => resp_features.py} (98%) diff --git a/src/lib/Resp_features.py b/src/lib/resp_features.py similarity index 98% rename from src/lib/Resp_features.py rename to src/lib/resp_features.py index 3afcdbb..c9f43a2 100644 --- a/src/lib/Resp_features.py +++ b/src/lib/resp_features.py @@ -67,4 +67,4 @@ def odi_application(data, fs): magnitudes.append(diff.loc[start:end].max()) odi_deepness = np.mean(magnitudes) if magnitudes else 0.0 - return odi_mean, odi_deepness \ No newline at end of file + return odi_mean, odi_deepness From 856393a282f69340a60bacf9085c393903c37066 Mon Sep 17 00:00:00 2001 From: sromagnoli-10 Date: Thu, 2 Apr 2026 16:44:28 +0200 Subject: [PATCH 74/93] Update import resp_features in resp_processing.py --- src/resp_processing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/resp_processing.py b/src/resp_processing.py index b5a89f9..11b474f 100644 --- a/src/resp_processing.py +++ b/src/resp_processing.py @@ -1,4 +1,4 @@ -from .lib import Resp_features +from .lib import resp_features import numpy as np from src.common.channel_utils import get_cached_channel_table, normalize_channel_label, split_channel_aliases from src.common.signal_utils import resample_signal From b7f338373348a873e119e7d82d59354b7a121ef2 Mon Sep 17 00:00:00 2001 From: sromagnoli-10 Date: Sun, 5 Apr 2026 22:57:12 +0200 Subject: [PATCH 75/93] Update training - threshold = 0.5.py --- src/pipeline/training.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/pipeline/training.py b/src/pipeline/training.py index af28417..db718b3 100644 --- a/src/pipeline/training.py +++ b/src/pipeline/training.py @@ -276,7 +276,8 @@ def train_multimodal_ensemble(data_folder, verbose, csv_path, export_folder=None modality_presence_indices = get_feature_group_indices(include_demographics=False) preprocessor = _build_preprocessor(len(labels)) processed_features = np.asarray(preprocessor.fit_transform(features), dtype=np.float32) - threshold = _calibrate_threshold(features, labels, feature_indices, modality_presence_indices) + # threshold = _calibrate_threshold(features, labels, feature_indices, modality_presence_indices) + threshold = 0.5 models = _fit_ensemble(processed_features, labels, feature_indices) export_root = export_folder or os.path.join(os.getcwd(), 'feature_exports') @@ -305,4 +306,4 @@ def train_multimodal_ensemble(data_folder, verbose, csv_path, export_folder=None 'raw': raw_feature_export_path, 'preprocessed': preprocessed_feature_export_path, }, - } \ No newline at end of file + } From 64a2023cbeb186c6dec060b909911fd58ff94c8f Mon Sep 17 00:00:00 2001 From: rolopu1 Date: Mon, 6 Apr 2026 14:38:47 +0200 Subject: [PATCH 76/93] Menor coste computacional resp y eeg --- src/eeg_processing.py | 2 +- src/lib/resp_features.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/eeg_processing.py b/src/eeg_processing.py index 6568412..b5cf505 100644 --- a/src/eeg_processing.py +++ b/src/eeg_processing.py @@ -134,7 +134,7 @@ def _extract_channel_metrics(signal, fs): if epochs.size == 0: return None - band_powers, complexities = eeg_features.extract_band_powers(epochs, fs, win_len=15) + band_powers, complexities = eeg_features.extract_band_powers(epochs, fs, win_len=30) if len(band_powers) > 60: band_powers = band_powers.iloc[60:] complexities = complexities.iloc[60:] diff --git a/src/lib/resp_features.py b/src/lib/resp_features.py index c9f43a2..378e51e 100644 --- a/src/lib/resp_features.py +++ b/src/lib/resp_features.py @@ -9,12 +9,12 @@ def peakedness_application(data, stage, subject_id=1): fs = 25 setup = { - "K": 5, - "DT": 5, + "K": 15, + "DT": 15, "Ts": 60, - "Tm": 20, + "Tm": 30, "Omega_r": np.array([5, 25]) / 60, - "Nfft": np.power(2, 13), + "Nfft": np.power(2, 12), } time_axis = np.arange(0, data.shape[0] / fs, 1 / fs) From 8510355fa4aa065cbf8b8a94df592dbc65d26988 Mon Sep 17 00:00:00 2001 From: dcajal Date: Tue, 7 Apr 2026 15:04:02 +0200 Subject: [PATCH 77/93] Run model feature export handling --- src/pipeline/training.py | 85 ++++++++++++++++++++++++++++++---------- team_code.py | 59 ++++++++++++++++++++++++++-- 2 files changed, 120 insertions(+), 24 deletions(-) diff --git a/src/pipeline/training.py b/src/pipeline/training.py index db718b3..0d46aa6 100644 --- a/src/pipeline/training.py +++ b/src/pipeline/training.py @@ -77,15 +77,62 @@ def process_training_record(record, data_folder, demographics_cache, diagnosis_c }, None, None, f"Error processing {patient_id}: {exc}" -def _export_feature_matrix_csv(output_path, metadata_rows, labels, feature_matrix, feature_names): +def prepare_feature_matrix(feature_matrix, preprocessor=None): + raw_feature_matrix = np.asarray(feature_matrix, dtype=np.float32) + if raw_feature_matrix.ndim == 1: + raw_feature_matrix = raw_feature_matrix.reshape(1, -1) + raw_feature_matrix = raw_feature_matrix.copy() + raw_feature_matrix[~np.isfinite(raw_feature_matrix)] = np.nan + + if preprocessor is not None: + processed_feature_matrix = np.asarray(preprocessor.transform(raw_feature_matrix), dtype=np.float32) + else: + processed_feature_matrix = raw_feature_matrix + + return raw_feature_matrix, processed_feature_matrix + + +def export_feature_matrix_csv(output_path, metadata_rows, feature_matrix, feature_names, labels=None): dataframe = pd.DataFrame(metadata_rows) - dataframe['label'] = labels + if labels is not None: + dataframe['label'] = labels + feature_frame = pd.DataFrame(feature_matrix, columns=feature_names) dataframe = pd.concat([dataframe.reset_index(drop=True), feature_frame.reset_index(drop=True)], axis=1) os.makedirs(os.path.dirname(output_path), exist_ok=True) dataframe.to_csv(output_path, index=False) +def get_feature_export_paths(export_root, prefix): + return { + 'raw': os.path.join(export_root, f'{prefix}_features_raw.csv'), + 'preprocessed': os.path.join(export_root, f'{prefix}_features_preprocessed.csv'), + } + + +def export_feature_views(export_root, prefix, metadata_rows, feature_matrix, feature_names, preprocessor=None, labels=None): + raw_feature_matrix, processed_feature_matrix = prepare_feature_matrix( + feature_matrix, + preprocessor=preprocessor, + ) + export_paths = get_feature_export_paths(export_root, prefix) + export_feature_matrix_csv( + export_paths['raw'], + metadata_rows, + raw_feature_matrix, + feature_names, + labels=labels, + ) + export_feature_matrix_csv( + export_paths['preprocessed'], + metadata_rows, + processed_feature_matrix, + feature_names, + labels=labels, + ) + return export_paths + + def best_threshold(probabilities, labels): thresholds = np.linspace(0, 1, 101) best_score = -1.0 @@ -149,11 +196,10 @@ def _has_modality_signal(feature_vector, modality_presence_indices): def predict_ensemble_probabilities(model_bundle, feature_matrix): - raw_feature_matrix = np.asarray(feature_matrix, dtype=np.float32) - if raw_feature_matrix.ndim == 1: - raw_feature_matrix = raw_feature_matrix.reshape(1, -1) - raw_feature_matrix = raw_feature_matrix.copy() - raw_feature_matrix[~np.isfinite(raw_feature_matrix)] = np.nan + raw_feature_matrix, processed_feature_matrix = prepare_feature_matrix( + feature_matrix, + preprocessor=model_bundle.get('preprocessor'), + ) models = model_bundle['models'] feature_indices = { @@ -164,11 +210,6 @@ def predict_ensemble_probabilities(model_bundle, feature_matrix): name: np.asarray(indices, dtype=np.int32) for name, indices in model_bundle['modality_presence_indices'].items() } - preprocessor = model_bundle.get('preprocessor') - if preprocessor is not None: - processed_feature_matrix = np.asarray(preprocessor.transform(raw_feature_matrix), dtype=np.float32) - else: - processed_feature_matrix = raw_feature_matrix probabilities = np.zeros(raw_feature_matrix.shape[0], dtype=np.float32) for row_index, raw_feature_vector in enumerate(raw_feature_matrix): @@ -280,12 +321,17 @@ def train_multimodal_ensemble(data_folder, verbose, csv_path, export_folder=None threshold = 0.5 models = _fit_ensemble(processed_features, labels, feature_indices) - export_root = export_folder or os.path.join(os.getcwd(), 'feature_exports') - raw_feature_export_path = os.path.join(export_root, 'training_features_raw.csv') - preprocessed_feature_export_path = os.path.join(export_root, 'training_features_preprocessed.csv') feature_names = list(get_feature_names()) - _export_feature_matrix_csv(raw_feature_export_path, metadata_rows, labels, features, feature_names) - _export_feature_matrix_csv(preprocessed_feature_export_path, metadata_rows, labels, processed_features, feature_names) + export_root = export_folder or os.path.join(os.getcwd(), 'feature_exports') + feature_exports = export_feature_views( + export_root, + 'training', + metadata_rows, + features, + feature_names, + preprocessor=preprocessor, + labels=labels, + ) return { 'type': 'multimodal_xgb_ensemble', @@ -302,8 +348,5 @@ def train_multimodal_ensemble(data_folder, verbose, csv_path, export_folder=None }, 'models': models, 'preprocessor': preprocessor, - 'feature_exports': { - 'raw': raw_feature_export_path, - 'preprocessed': preprocessed_feature_export_path, - }, + 'feature_exports': feature_exports, } diff --git a/team_code.py b/team_code.py index fb32d0e..9cd788a 100644 --- a/team_code.py +++ b/team_code.py @@ -15,12 +15,18 @@ import builtins import re import sys + +import numpy as np from tqdm import tqdm from helper_code import * from src.pipeline.config import DEFAULT_CSV_PATH from src.pipeline.features import get_feature_export_dir, get_or_create_record_feature_vector -from src.pipeline.training import predict_ensemble_labels, train_multimodal_ensemble +from src.pipeline.training import ( + export_feature_views, + predict_ensemble_labels, + train_multimodal_ensemble, +) ################################################################################ # Path & Constant Configuration (Added for Robustness) ################################################################################ @@ -28,6 +34,7 @@ # Progress bar state for run_model (initialized lazily) RUN_MODEL_PBAR = None RUN_MODEL_PBAR_TOTAL = None +RUN_MODEL_EXPORT_STATE = None ORIGINAL_PRINT = builtins.print PRINT_FILTER_ACTIVE = False RUN_PROGRESS_LINE_RE = re.compile(r'^-\s+\d+/\d+:\s') @@ -38,6 +45,28 @@ def _close_run_model_pbar(): RUN_MODEL_PBAR = None +def _flush_run_feature_exports(): + global RUN_MODEL_EXPORT_STATE + + if RUN_MODEL_EXPORT_STATE is None or not RUN_MODEL_EXPORT_STATE['metadata_rows']: + return + + feature_exports = export_feature_views( + RUN_MODEL_EXPORT_STATE['export_root'], + 'test', + RUN_MODEL_EXPORT_STATE['metadata_rows'], + np.asarray(RUN_MODEL_EXPORT_STATE['raw_features'], dtype=np.float32), + RUN_MODEL_EXPORT_STATE['feature_names'], + preprocessor=RUN_MODEL_EXPORT_STATE['preprocessor'], + ) + + if RUN_MODEL_EXPORT_STATE['verbose']: + print(f"Raw features CSV: {feature_exports.get('raw')}") + print(f"Preprocessed features CSV: {feature_exports.get('preprocessed')}") + + RUN_MODEL_EXPORT_STATE = None + + def _install_run_print_filter(): global PRINT_FILTER_ACTIVE if PRINT_FILTER_ACTIVE: @@ -61,6 +90,7 @@ def _restore_print(): atexit.register(_close_run_model_pbar) +atexit.register(_flush_run_feature_exports) atexit.register(_restore_print) @@ -117,7 +147,7 @@ def load_model(model_folder, verbose): # Run your trained model. This function is *required*. You should edit this function to add your code, but do *not* change the # arguments of this function. def run_model(model, record, data_folder, verbose): - global RUN_MODEL_PBAR, RUN_MODEL_PBAR_TOTAL + global RUN_MODEL_PBAR, RUN_MODEL_PBAR_TOTAL, RUN_MODEL_EXPORT_STATE # Load the model. model = model['model'] @@ -145,6 +175,17 @@ def run_model(model, record, data_folder, verbose): disable=not verbose ) + if RUN_MODEL_EXPORT_STATE is None: + export_root = get_feature_export_dir(data_folder) + RUN_MODEL_EXPORT_STATE = { + 'export_root': export_root, + 'metadata_rows': [], + 'raw_features': [], + 'feature_names': list(model.get('feature_names', [])), + 'preprocessor': model.get('preprocessor'), + 'verbose': verbose, + } + if verbose and RUN_MODEL_PBAR is not None: RUN_MODEL_PBAR.set_postfix({"patient": patient_id}) @@ -157,7 +198,15 @@ def run_model(model, record, data_folder, verbose): patient_data, csv_path=DEFAULT_CSV_PATH, require_physiological_data=False, - ).reshape(1, -1) + ) + features = np.asarray(features, dtype=np.float32).reshape(1, -1) + + RUN_MODEL_EXPORT_STATE['metadata_rows'].append({ + 'patient_id': patient_id, + 'site_id': site_id, + 'session_id': session_id, + }) + RUN_MODEL_EXPORT_STATE['raw_features'].append(features[0]) # Get the model outputs. labels, probabilities = predict_ensemble_labels(model, features) @@ -170,6 +219,10 @@ def run_model(model, record, data_folder, verbose): RUN_MODEL_PBAR.close() RUN_MODEL_PBAR = None + if RUN_MODEL_PBAR_TOTAL is not None and RUN_MODEL_EXPORT_STATE is not None: + if len(RUN_MODEL_EXPORT_STATE['metadata_rows']) >= RUN_MODEL_PBAR_TOTAL: + _flush_run_feature_exports() + return binary_output, probability_output # Save your trained model. From 7e5a5f9e30121720ca7101211cda829e96b16fbd Mon Sep 17 00:00:00 2001 From: sromagnoli-10 Date: Tue, 7 Apr 2026 23:04:10 +0200 Subject: [PATCH 78/93] Update training - CV of model hyperparameters and threshold.py --- src/pipeline/training.py | 288 ++++++++++++++++++++++++++++++--------- 1 file changed, 220 insertions(+), 68 deletions(-) diff --git a/src/pipeline/training.py b/src/pipeline/training.py index 0d46aa6..4fd3f63 100644 --- a/src/pipeline/training.py +++ b/src/pipeline/training.py @@ -1,11 +1,12 @@ import os +from collections import Counter from concurrent.futures import ThreadPoolExecutor import numpy as np import pandas as pd from sklearn.impute import KNNImputer from sklearn.metrics import f1_score -from sklearn.model_selection import train_test_split +from sklearn.model_selection import train_test_split, RandomizedSearchCV, StratifiedKFold from sklearn.pipeline import Pipeline from sklearn.preprocessing import StandardScaler from tqdm import tqdm @@ -21,6 +22,16 @@ ENSEMBLE_MODALITIES = ('resp', 'eeg', 'ecg') DEFAULT_KNN_NEIGHBORS = 5 +# Hyperparameter search space +PARAM_DIST = { + 'max_depth': [3, 4, 5], + 'min_child_weight': [1, 2, 3], + 'subsample': [0.7, 0.8, 0.9], + 'colsample_bytree': [0.6, 0.7, 0.8], + 'reg_lambda': [0.5, 1.0, 2.0], + 'reg_alpha': [0.0, 0.05, 0.1], +} + def build_training_metadata_cache(patient_data_file): metadata = pd.read_csv(patient_data_file) @@ -76,7 +87,6 @@ def process_training_record(record, data_folder, demographics_cache, diagnosis_c 'session_id': session_id, }, None, None, f"Error processing {patient_id}: {exc}" - def prepare_feature_matrix(feature_matrix, preprocessor=None): raw_feature_matrix = np.asarray(feature_matrix, dtype=np.float32) if raw_feature_matrix.ndim == 1: @@ -91,25 +101,21 @@ def prepare_feature_matrix(feature_matrix, preprocessor=None): return raw_feature_matrix, processed_feature_matrix - def export_feature_matrix_csv(output_path, metadata_rows, feature_matrix, feature_names, labels=None): dataframe = pd.DataFrame(metadata_rows) if labels is not None: dataframe['label'] = labels - feature_frame = pd.DataFrame(feature_matrix, columns=feature_names) dataframe = pd.concat([dataframe.reset_index(drop=True), feature_frame.reset_index(drop=True)], axis=1) os.makedirs(os.path.dirname(output_path), exist_ok=True) dataframe.to_csv(output_path, index=False) - def get_feature_export_paths(export_root, prefix): return { 'raw': os.path.join(export_root, f'{prefix}_features_raw.csv'), 'preprocessed': os.path.join(export_root, f'{prefix}_features_preprocessed.csv'), } - - + def export_feature_views(export_root, prefix, metadata_rows, feature_matrix, feature_names, preprocessor=None, labels=None): raw_feature_matrix, processed_feature_matrix = prepare_feature_matrix( feature_matrix, @@ -130,7 +136,7 @@ def export_feature_views(export_root, prefix, metadata_rows, feature_matrix, fea feature_names, labels=labels, ) - return export_paths + return export_paths def best_threshold(probabilities, labels): @@ -144,49 +150,149 @@ def best_threshold(probabilities, labels): if score > best_score: best_score = score best_value = float(threshold) + if best_value < 0.5: + best_value = 0.5 return best_value -def _build_xgb_model(labels): +def _build_xgb_model(labels, extra_params=None): + neg = int(np.sum(labels == 0)) pos = int(np.sum(labels == 1)) scale_pos_weight = (neg / pos) if pos > 0 else 1.0 - - return XGBClassifier( + + base_params = dict( scale_pos_weight=scale_pos_weight, - n_estimators=300, - max_depth=4, - learning_rate=0.05, + n_estimators=500, + learning_rate=0.05, + max_depth=4, subsample=0.8, + colsample_bytree=0.7, + min_child_weight=2, + reg_alpha=0.0, + reg_lambda=1.0, random_state=42, eval_metric='auc', tree_method='hist', ) + if extra_params: + base_params.update(extra_params) + + return XGBClassifier(**base_params) - -def _fit_model(feature_matrix, labels): - model = _build_xgb_model(labels) +def _fit_model(feature_matrix, labels, consensus_params=None): + model = _build_xgb_model(labels, extra_params=consensus_params) model.fit(feature_matrix, labels) return model -def _build_preprocessor(num_samples): - neighbors = min(DEFAULT_KNN_NEIGHBORS, max(1, num_samples - 1)) if num_samples > 1 else 1 - return Pipeline([ - ('imputer', KNNImputer(n_neighbors=neighbors, keep_empty_features=True)), - ('scaler', StandardScaler()), - ]) +def _build_preprocessor(num_samples, categorical_indices=None): -def _fit_ensemble(feature_matrix, labels, feature_indices): + neighbors = min(DEFAULT_KNN_NEIGHBORS, max(1, num_samples - 1)) if num_samples > 1 else 1 + + if categorical_indices is None or len(categorical_indices) == 0: + # Original pipeline — no categorical columns + return Pipeline([ + ('imputer', KNNImputer(n_neighbors=neighbors, keep_empty_features=True)), + ('scaler', StandardScaler()), + ]) + + return _CategoricalAwarePreprocessor( + n_neighbors=neighbors, + categorical_indices=np.asarray(categorical_indices, dtype=np.int32), + ) + +class _CategoricalAwarePreprocessor: + + def __init__(self, n_neighbors, categorical_indices): + self.n_neighbors = n_neighbors + self.categorical_indices = categorical_indices + self.imputer = KNNImputer(n_neighbors=n_neighbors, keep_empty_features=True) + self.scaler = StandardScaler() + self._numerical_indices = None # set during fit + + def _get_numerical_indices(self, n_features): + all_idx = np.arange(n_features) + return np.setdiff1d(all_idx, self.categorical_indices) + + def fit_transform(self, X): + X = np.asarray(X, dtype=np.float32).copy() + X[~np.isfinite(X)] = np.nan + + # Step 1: impute everything + X_imputed = np.asarray(self.imputer.fit_transform(X), dtype=np.float32) + + # Step 2: fit scaler on numerical columns only + self._numerical_indices = self._get_numerical_indices(X.shape[1]) + X_num = X_imputed[:, self._numerical_indices] + X_num_scaled = np.asarray(self.scaler.fit_transform(X_num), dtype=np.float32) + + # Step 3: assemble output — categorical columns keep imputed values + X_out = X_imputed.copy() + X_out[:, self._numerical_indices] = X_num_scaled + return X_out + + def transform(self, X): + X = np.asarray(X, dtype=np.float32).copy() + X[~np.isfinite(X)] = np.nan + + X_imputed = np.asarray(self.imputer.transform(X), dtype=np.float32) + + X_num = X_imputed[:, self._numerical_indices] + X_num_scaled = np.asarray(self.scaler.transform(X_num), dtype=np.float32) + + X_out = X_imputed.copy() + X_out[:, self._numerical_indices] = X_num_scaled + return X_out + +def _search_hyperparams(X_train, y_train): + + base_model = XGBClassifier( + scale_pos_weight=(int(np.sum(y_train == 0)) / max(int(np.sum(y_train == 1)), 1)), + n_estimators=500, + learning_rate=0.05, + random_state=42, + eval_metric='auc', + tree_method='hist', + ) + + search = RandomizedSearchCV( + estimator=base_model, + param_distributions=PARAM_DIST, + n_iter=20, + scoring='f1', + cv=StratifiedKFold(n_splits=3, shuffle=True, random_state=42), + random_state=42, + n_jobs=-1, + refit=False, # we only need best_params_, not a refitted model + ) + search.fit(X_train, y_train) + return search.best_params_ + +def _consensus_params(params_per_fold): + + consensus = {} + for key in PARAM_DIST.keys(): + values = [p[key] for p in params_per_fold if key in p] + if not values: + continue + # majority vote + most_common = Counter(values).most_common(1)[0][0] + consensus[key] = most_common + return consensus + +def _fit_ensemble(feature_matrix, labels, feature_indices, consensus_params=None): models = { - 'all': _fit_model(feature_matrix[:, feature_indices['all']], labels), + 'all': _fit_model( + feature_matrix[:, feature_indices['all']], labels, consensus_params + ), } - for modality in ENSEMBLE_MODALITIES: - models[modality] = _fit_model(feature_matrix[:, feature_indices[modality]], labels) - + models[modality] = _fit_model( + feature_matrix[:, feature_indices[modality]], labels, consensus_params + ) return models @@ -210,7 +316,7 @@ def predict_ensemble_probabilities(model_bundle, feature_matrix): name: np.asarray(indices, dtype=np.int32) for name, indices in model_bundle['modality_presence_indices'].items() } - + probabilities = np.zeros(raw_feature_matrix.shape[0], dtype=np.float32) for row_index, raw_feature_vector in enumerate(raw_feature_matrix): processed_feature_vector = processed_feature_matrix[row_index] @@ -236,35 +342,63 @@ def predict_ensemble_labels(model_bundle, feature_matrix): labels = (probabilities >= threshold).astype(np.int32) return labels, probabilities +def _calibrate_threshold_cv( + features, + labels, + feature_indices, + modality_presence_indices, + categorical_indices=None, + n_splits=5, +): -def _calibrate_threshold(feature_matrix, labels, feature_indices, modality_presence_indices): classes, class_counts = np.unique(labels, return_counts=True) - if len(classes) != 2 or np.min(class_counts) < 2 or len(labels) < 10: - return DEFAULT_ENSEMBLE_THRESHOLD - - try: - train_features, validation_features, train_labels, validation_labels = train_test_split( - feature_matrix, - labels, - test_size=0.2, - random_state=42, - stratify=labels, - ) - except ValueError: - return DEFAULT_ENSEMBLE_THRESHOLD + if len(classes) != 2 or np.min(class_counts) < n_splits: + return DEFAULT_ENSEMBLE_THRESHOLD, None + + skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42) + oof_probabilities = np.zeros(len(labels), dtype=np.float32) + best_params_per_fold = [] + + for fold_idx, (train_idx, val_idx) in enumerate(skf.split(features, labels)): + print(f" Outer fold {fold_idx + 1}/{n_splits} — hyperparameter search + OOF prediction") + + X_train, X_val = features[train_idx], features[val_idx] + y_train, y_val = labels[train_idx], labels[val_idx] + + # Fit preprocessor ONLY on fold's training data (no leakage into val) + fold_preprocessor = _build_preprocessor(len(y_train), categorical_indices) + X_train_proc = np.asarray(fold_preprocessor.fit_transform(X_train), dtype=np.float32) + X_val_proc = np.asarray(fold_preprocessor.transform(X_val), dtype=np.float32) + + # Inner CV hyperparameter search on this fold's training data + fold_best_params = _search_hyperparams(X_train_proc[:, feature_indices['all']], y_train) + best_params_per_fold.append(fold_best_params) + print(f" Best params fold {fold_idx + 1}: {fold_best_params}") + + # Fit fold ensemble with the fold's best params + fold_models = _fit_ensemble(X_train_proc, y_train, feature_indices, consensus_params=fold_best_params) + fold_bundle = { + 'models': fold_models, + 'feature_indices': feature_indices, + 'modality_presence_indices': modality_presence_indices, + 'preprocessor': fold_preprocessor, + 'threshold': DEFAULT_ENSEMBLE_THRESHOLD, + } + + # Collect out-of-fold probabilities (never seen by this fold's model during training) + oof_probabilities[val_idx] = predict_ensemble_probabilities(fold_bundle, X_val) + + # Aggregate best params across all outer folds by majority vote + consensus = _consensus_params(best_params_per_fold) + print(f" Consensus hyperparameters across {n_splits} folds: {consensus}") + + # Threshold calibrated on clean OOF probabilities + threshold = best_threshold(oof_probabilities, labels) + + print(f" OOF threshold: {threshold:.2f}") + + return threshold, consensus - preprocessor = _build_preprocessor(len(train_labels)) - processed_train_features = np.asarray(preprocessor.fit_transform(train_features), dtype=np.float32) - - calibration_bundle = { - 'models': _fit_ensemble(processed_train_features, train_labels, feature_indices), - 'feature_indices': feature_indices, - 'modality_presence_indices': modality_presence_indices, - 'threshold': DEFAULT_ENSEMBLE_THRESHOLD, - 'preprocessor': preprocessor, - } - validation_probabilities = predict_ensemble_probabilities(calibration_bundle, validation_features) - return best_threshold(validation_probabilities, validation_labels) def train_multimodal_ensemble(data_folder, verbose, csv_path, export_folder=None): @@ -313,26 +447,44 @@ def train_multimodal_ensemble(data_folder, verbose, csv_path, export_folder=None if features.size == 0 or features.ndim != 2 or features.shape[0] == 0: raise ValueError('No valid training samples were extracted. Review feature extraction logs for the skipped records.') + feature_names = list(get_feature_names()) feature_indices = get_feature_group_indices(include_demographics=True) modality_presence_indices = get_feature_group_indices(include_demographics=False) - preprocessor = _build_preprocessor(len(labels)) + + categorical_indices = [ + i for i, name in enumerate(feature_names) + if name.lower() in ('sex', 'gender') + ] + print(f" Categorical feature indices: {categorical_indices} " + f"({[feature_names[i] for i in categorical_indices]})") + + # --- Step 1: Nested CV for threshold calibration and consensus hyperparameters --- + print("Running nested CV for threshold calibration and hyperparameter consensus...") + threshold, consensus = _calibrate_threshold_cv( + features, + labels, + feature_indices, + modality_presence_indices, + categorical_indices=categorical_indices if categorical_indices else None, + ) + + # --- Step 2: Fit final models on ALL data using consensus hyperparameters --- + print("Fitting final ensemble on all training data with consensus hyperparameters...") + preprocessor = _build_preprocessor(len(labels), categorical_indices if categorical_indices else None) processed_features = np.asarray(preprocessor.fit_transform(features), dtype=np.float32) - # threshold = _calibrate_threshold(features, labels, feature_indices, modality_presence_indices) - threshold = 0.5 - models = _fit_ensemble(processed_features, labels, feature_indices) + models = _fit_ensemble(processed_features, labels, feature_indices, consensus_params=consensus) - feature_names = list(get_feature_names()) export_root = export_folder or os.path.join(os.getcwd(), 'feature_exports') feature_exports = export_feature_views( - export_root, - 'training', - metadata_rows, - features, - feature_names, - preprocessor=preprocessor, - labels=labels, + export_root, + 'training', + metadata_rows, + features, + feature_names, + preprocessor=preprocessor, + labels=labels, ) - + return { 'type': 'multimodal_xgb_ensemble', 'threshold': threshold, From 594f40a31fb3645952b4bee85ce3e917e0ff744e Mon Sep 17 00:00:00 2001 From: rolopu1 Date: Wed, 8 Apr 2026 10:02:13 +0200 Subject: [PATCH 79/93] Add Train_test_split --- src/lib/train_test_split.py | 132 ++++++++++++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100644 src/lib/train_test_split.py diff --git a/src/lib/train_test_split.py b/src/lib/train_test_split.py new file mode 100644 index 0000000..1d4956a --- /dev/null +++ b/src/lib/train_test_split.py @@ -0,0 +1,132 @@ +import csv +import os +import shutil + + +def create_test_set_from_ids(test_ids_file, demographics_file, source_dir, dest_dir): + """ + Create a test set by moving EDF files based on patient IDs. + + Args: + test_ids_file (str): Path to the file containing test patient IDs (one per line). + demographics_file (str): Path to the demographics CSV file. + source_dir (str): Path to the source directory containing physiological data folders. + dest_dir (str): Path to the destination directory for the test set. + """ + # Read test IDs + with open(test_ids_file, 'r') as f: + test_ids = set(line.strip() for line in f if line.strip()) + + print(f"Test IDs: {len(test_ids)}") + + # Parse demographics to get mapping + id_to_bids = {} + with open(demographics_file, 'r', newline='', encoding='utf-8') as f: + reader = csv.DictReader(f) + for row in reader: + patient_id = row['BDSPPatientID'] + bids_folder = row['BidsFolder'] + if patient_id in test_ids: + if patient_id not in id_to_bids: + id_to_bids[patient_id] = [] + id_to_bids[patient_id].append(bids_folder) + + print(f"Found mappings for {len(id_to_bids)} patients") + + # Create destination directory if it doesn't exist + os.makedirs(dest_dir, exist_ok=True) + + moved_files = [] + # Find and move files + for patient_id, bids_list in id_to_bids.items(): + for bids in bids_list: + # bids is like sub-I0002150000686 + # file is sub-I0002150000686_ses-X.edf + # folder is I0002, I0006, S0001 + site = bids.split('-')[1][:5] # I0002 or I0006 or S0001 + folder = os.path.join(source_dir, site) + if os.path.exists(folder): + for file in os.listdir(folder): + if file.startswith(bids + '_') and file.lower().endswith('.edf'): + src = os.path.join(folder, file) + dst = os.path.join(dest_dir, file) + shutil.move(src, dst) + moved_files.append(dst) + print(f"Moved {src} to {dst}") + + print(f"Done. Moved {len(moved_files)} EDF files") + return moved_files + + +def get_selected_records_from_test_set(test_set_dir): + selected_records = set() + for root, _, files in os.walk(test_set_dir): + for file in files: + if file.lower().endswith('.edf'): + stem = os.path.splitext(file)[0] + if '_ses-' not in stem: + continue + bids_folder, session_id = stem.rsplit('_ses-', 1) + site_id = bids_folder.split('-')[1][:5] if '-' in bids_folder else '' + selected_records.add((site_id, bids_folder, session_id)) + return selected_records + + +def sync_demographics_between_train_and_test(source_demographics_file, test_set_dir, test_demographics_file=None): + """ + Create a filtered demographics CSV in the test set and remove those rows from the training demographics file. + + Args: + source_demographics_file (str): Path to the source training_set demographics CSV. + test_set_dir (str): Directory containing the moved EDF files for the test set. + test_demographics_file (str, optional): Output path for the test set demographics CSV. + Defaults to '/demographics.csv'. + """ + if test_demographics_file is None: + test_demographics_file = os.path.join(test_set_dir, 'demographics.csv') + + selected_records = get_selected_records_from_test_set(test_set_dir) + print(f"Found {len(selected_records)} EDF records in {test_set_dir}") + + with open(source_demographics_file, 'r', newline='', encoding='utf-8') as f: + reader = csv.DictReader(f) + fieldnames = reader.fieldnames + source_rows = list(reader) + + rows_for_test = [] + remaining_rows = [] + for row in source_rows: + key = (row.get('SiteID'), row.get('BidsFolder'), str(row.get('SessionID'))) + if key in selected_records: + rows_for_test.append(row) + else: + remaining_rows.append(row) + + os.makedirs(os.path.dirname(test_demographics_file), exist_ok=True) + with open(test_demographics_file, 'w', newline='', encoding='utf-8') as f: + writer = csv.DictWriter(f, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(rows_for_test) + + with open(source_demographics_file, 'w', newline='', encoding='utf-8') as f: + writer = csv.DictWriter(f, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(remaining_rows) + + print(f"Wrote {len(rows_for_test)} rows to {test_demographics_file}") + print(f"Removed {len(source_rows) - len(remaining_rows)} rows from {source_demographics_file}") + + +if __name__ == "__main__": + # Example usage + moved_files = create_test_set_from_ids( + test_ids_file='testids_lista.txt', + demographics_file='data/training_set/demographics.csv', + source_dir='data/training_set/physiological_data', + dest_dir='data/test_set' + ) + if moved_files: + sync_demographics_between_train_and_test( + source_demographics_file='data/training_set/demographics.csv', + test_set_dir='data/test_set' + ) \ No newline at end of file From 2deede911ce71802f595b32465154998c2bc7a5a Mon Sep 17 00:00:00 2001 From: sromagnoli-10 Date: Wed, 8 Apr 2026 11:51:06 +0200 Subject: [PATCH 80/93] Update config - step 15 minutes.py --- src/pipeline/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pipeline/config.py b/src/pipeline/config.py index 4513a56..a1a28ca 100644 --- a/src/pipeline/config.py +++ b/src/pipeline/config.py @@ -10,9 +10,9 @@ FEATURE_CACHE_FOLDER_NAME = '.feature_cache' MAX_TRAIN_WORKERS = max(1, min(4, os.cpu_count() or 1)) SEGMENT_DURATION_SECONDS = 5 * 60 -SEGMENT_STRIDE_SECONDS = 60 * 60 +SEGMENT_STRIDE_SECONDS = 15 * 60 TOTAL_PHYSIOLOGICAL_FEATURE_LENGTH = ( RESP_FEATURE_LENGTH + EEG_FEATURE_LENGTH + ECG_FEATURE_LENGTH -) \ No newline at end of file +) From 2744229a24656d8f7f2c0fdb9651b8fea522e8e4 Mon Sep 17 00:00:00 2001 From: dcajal Date: Wed, 8 Apr 2026 12:52:01 +0200 Subject: [PATCH 81/93] Add cross-validation fold evaluation --- src/pipeline/training.py | 133 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 123 insertions(+), 10 deletions(-) diff --git a/src/pipeline/training.py b/src/pipeline/training.py index 4fd3f63..a135999 100644 --- a/src/pipeline/training.py +++ b/src/pipeline/training.py @@ -5,7 +5,7 @@ import numpy as np import pandas as pd from sklearn.impute import KNNImputer -from sklearn.metrics import f1_score +from sklearn.metrics import accuracy_score, average_precision_score, f1_score, precision_score, recall_score, roc_auc_score from sklearn.model_selection import train_test_split, RandomizedSearchCV, StratifiedKFold from sklearn.pipeline import Pipeline from sklearn.preprocessing import StandardScaler @@ -150,12 +150,84 @@ def best_threshold(probabilities, labels): if score > best_score: best_score = score best_value = float(threshold) - if best_value < 0.5: - best_value = 0.5 return best_value +def _safe_auroc(labels, probabilities): + if len(np.unique(labels)) < 2: + return np.nan + return float(roc_auc_score(labels, probabilities)) + + +def _safe_auprc(labels, probabilities): + if len(np.unique(labels)) < 2: + return np.nan + return float(average_precision_score(labels, probabilities, pos_label=1)) + + +def _compute_evaluation_metrics(labels, probabilities, threshold): + labels = np.asarray(labels, dtype=np.int32) + probabilities = np.asarray(probabilities, dtype=np.float32) + predictions = (probabilities >= threshold).astype(np.int32) + + return { + 'threshold': float(threshold), + 'auroc': _safe_auroc(labels, probabilities), + 'auprc': _safe_auprc(labels, probabilities), + 'accuracy': float(accuracy_score(labels, predictions)), + 'f1': float(f1_score(labels, predictions, zero_division=0)), + 'precision': float(precision_score(labels, predictions, zero_division=0)), + 'recall': float(recall_score(labels, predictions, zero_division=0)), + } + + +def _summarize_metrics(metric_rows): + summary = {} + metric_names = ('auroc', 'auprc', 'accuracy', 'f1', 'precision', 'recall') + + for metric_name in metric_names: + metric_values = np.asarray([row[metric_name] for row in metric_rows], dtype=np.float32) + finite_values = metric_values[np.isfinite(metric_values)] + if finite_values.size == 0: + summary[metric_name] = {'mean': None, 'std': None} + continue + + summary[metric_name] = { + 'mean': float(np.mean(finite_values)), + 'std': float(np.std(finite_values)), + } + + return summary + + +def _format_metric_value(value): + return 'nan' if value is None or not np.isfinite(value) else f'{value:.3f}' + + +def _print_metrics(prefix, metrics): + print( + f"{prefix} AUROC={_format_metric_value(metrics['auroc'])}, " + f"AUPRC={_format_metric_value(metrics['auprc'])}, " + f"Accuracy={_format_metric_value(metrics['accuracy'])}, " + f"F1={_format_metric_value(metrics['f1'])}, " + f"Precision={_format_metric_value(metrics['precision'])}, " + f"Recall={_format_metric_value(metrics['recall'])}, " + f"Threshold={metrics['threshold']:.2f}" + ) + + +def _print_metric_summary(prefix, summary): + metric_names = ('auroc', 'auprc', 'accuracy', 'f1', 'precision', 'recall') + parts = [] + for metric_name in metric_names: + metric_summary = summary.get(metric_name, {}) + mean_value = _format_metric_value(metric_summary.get('mean')) + std_value = _format_metric_value(metric_summary.get('std')) + parts.append(f"{metric_name.upper()}={mean_value} +/- {std_value}") + print(f"{prefix} {', '.join(parts)}") + + def _build_xgb_model(labels, extra_params=None): neg = int(np.sum(labels == 0)) @@ -353,11 +425,16 @@ def _calibrate_threshold_cv( classes, class_counts = np.unique(labels, return_counts=True) if len(classes) != 2 or np.min(class_counts) < n_splits: - return DEFAULT_ENSEMBLE_THRESHOLD, None + return DEFAULT_ENSEMBLE_THRESHOLD, None, { + 'skipped': True, + 'reason': 'Not enough samples per class to run stratified cross-validation.', + 'n_splits': int(n_splits), + } skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42) oof_probabilities = np.zeros(len(labels), dtype=np.float32) best_params_per_fold = [] + fold_metrics = [] for fold_idx, (train_idx, val_idx) in enumerate(skf.split(features, labels)): print(f" Outer fold {fold_idx + 1}/{n_splits} — hyperparameter search + OOF prediction") @@ -386,19 +463,54 @@ def _calibrate_threshold_cv( } # Collect out-of-fold probabilities (never seen by this fold's model during training) - oof_probabilities[val_idx] = predict_ensemble_probabilities(fold_bundle, X_val) + fold_probabilities = predict_ensemble_probabilities(fold_bundle, X_val) + oof_probabilities[val_idx] = fold_probabilities + + fold_metric_row = _compute_evaluation_metrics( + y_val, + fold_probabilities, + threshold=DEFAULT_ENSEMBLE_THRESHOLD, + ) + fold_metric_row.update({ + 'fold': int(fold_idx + 1), + 'train_size': int(len(train_idx)), + 'validation_size': int(len(val_idx)), + }) + fold_metrics.append(fold_metric_row) + _print_metrics(f" Fold {fold_idx + 1} metrics:", fold_metric_row) # Aggregate best params across all outer folds by majority vote consensus = _consensus_params(best_params_per_fold) print(f" Consensus hyperparameters across {n_splits} folds: {consensus}") + + fold_metric_summary = _summarize_metrics(fold_metrics) + _print_metric_summary(" Mean fold metrics:", fold_metric_summary) # Threshold calibrated on clean OOF probabilities + oof_default_metrics = _compute_evaluation_metrics( + labels, + oof_probabilities, + threshold=DEFAULT_ENSEMBLE_THRESHOLD, + ) + _print_metrics(" OOF metrics before calibration:", oof_default_metrics) + threshold = best_threshold(oof_probabilities, labels) - - print(f" OOF threshold: {threshold:.2f}") - - return threshold, consensus + oof_calibrated_metrics = _compute_evaluation_metrics( + labels, + oof_probabilities, + threshold=threshold, + ) + _print_metrics(" OOF metrics after calibration:", oof_calibrated_metrics) + + return threshold, consensus, { + 'skipped': False, + 'n_splits': int(n_splits), + 'fold_metrics': fold_metrics, + 'fold_metric_summary': fold_metric_summary, + 'oof_default_threshold_metrics': oof_default_metrics, + 'oof_calibrated_metrics': oof_calibrated_metrics, + } def train_multimodal_ensemble(data_folder, verbose, csv_path, export_folder=None): @@ -460,7 +572,7 @@ def train_multimodal_ensemble(data_folder, verbose, csv_path, export_folder=None # --- Step 1: Nested CV for threshold calibration and consensus hyperparameters --- print("Running nested CV for threshold calibration and hyperparameter consensus...") - threshold, consensus = _calibrate_threshold_cv( + threshold, consensus, cv_metrics = _calibrate_threshold_cv( features, labels, feature_indices, @@ -501,4 +613,5 @@ def train_multimodal_ensemble(data_folder, verbose, csv_path, export_folder=None 'models': models, 'preprocessor': preprocessor, 'feature_exports': feature_exports, + 'cv_metrics': cv_metrics, } From 3ca135fb395b8cf3436323102519d607e08c8615 Mon Sep 17 00:00:00 2001 From: dcajal Date: Wed, 8 Apr 2026 13:37:42 +0200 Subject: [PATCH 82/93] Add cross-validation per site --- src/pipeline/config.py | 13 + src/pipeline/cross_validation.py | 508 +++++++++++++++++++++++++++++++ src/pipeline/training.py | 278 +++-------------- 3 files changed, 571 insertions(+), 228 deletions(-) create mode 100644 src/pipeline/cross_validation.py diff --git a/src/pipeline/config.py b/src/pipeline/config.py index a1a28ca..4dc386c 100644 --- a/src/pipeline/config.py +++ b/src/pipeline/config.py @@ -9,6 +9,19 @@ DEFAULT_CSV_PATH = os.path.join(SCRIPT_DIR, 'channel_table.csv') FEATURE_CACHE_FOLDER_NAME = '.feature_cache' MAX_TRAIN_WORKERS = max(1, min(4, os.cpu_count() or 1)) +USE_SITE_GROUPED_CV = True +OPTIMIZE_HYPERPARAMETER_SEARCH = False +RANDOM_CV_N_SPLITS = 5 +CV_RANDOM_STATE = 42 +CV_SEARCH_ITERATIONS = 20 +DEFAULT_CV_HYPERPARAMETERS = { + 'max_depth': 3, + 'min_child_weight': 3, + 'subsample': 0.9, + 'colsample_bytree': 0.6, + 'reg_lambda': 0.5, + 'reg_alpha': 0.1, +} SEGMENT_DURATION_SECONDS = 5 * 60 SEGMENT_STRIDE_SECONDS = 15 * 60 TOTAL_PHYSIOLOGICAL_FEATURE_LENGTH = ( diff --git a/src/pipeline/cross_validation.py b/src/pipeline/cross_validation.py new file mode 100644 index 0000000..e4c01e4 --- /dev/null +++ b/src/pipeline/cross_validation.py @@ -0,0 +1,508 @@ +from dataclasses import dataclass, field +from typing import Any, Callable, Optional + +import numpy as np +from sklearn.metrics import accuracy_score, average_precision_score, f1_score, precision_score, recall_score, roc_auc_score +from sklearn.model_selection import LeaveOneGroupOut, RandomizedSearchCV, StratifiedKFold + + +def normalize_site_group(site_id): + site_text = str(site_id).strip().upper() + return site_text[:5] if site_text else 'UNKNOWN' + + +@dataclass(frozen=True) +class CrossValidationConfig: + use_site_grouped_cv: bool = True + optimize_hyperparameter_search: bool = False + outer_random_splits: int = 5 + random_state: int = 42 + search_iterations: int = 20 + fixed_hyperparameters: dict[str, Any] = field(default_factory=dict) + + +@dataclass(frozen=True) +class CrossValidationResult: + threshold: float + consensus_params: Optional[dict[str, Any]] + metrics: dict[str, Any] + + +@dataclass(frozen=True) +class FoldSplit: + fold_index: int + train_idx: np.ndarray + validation_idx: np.ndarray + label: str + + +class EnsembleCrossValidator: + def __init__( + self, + config: CrossValidationConfig, + param_dist, + default_threshold, + build_preprocessor: Callable[..., Any], + build_search_model: Callable[..., Any], + fit_ensemble: Callable[..., Any], + predict_probabilities: Callable[..., Any], + ): + self.config = config + self.param_dist = param_dist + self.default_threshold = default_threshold + self.build_preprocessor = build_preprocessor + self.build_search_model = build_search_model + self.fit_ensemble = fit_ensemble + self.predict_probabilities = predict_probabilities + + def run( + self, + features, + labels, + feature_indices, + modality_presence_indices, + categorical_indices=None, + site_groups=None, + ): + labels = np.asarray(labels, dtype=np.int32) + if self.config.use_site_grouped_cv: + return self._run_grouped_cv( + features, + labels, + site_groups, + feature_indices, + modality_presence_indices, + categorical_indices=categorical_indices, + ) + + return self._run_random_cv( + features, + labels, + feature_indices, + modality_presence_indices, + categorical_indices=categorical_indices, + ) + + def _run_grouped_cv( + self, + features, + labels, + site_groups, + feature_indices, + modality_presence_indices, + categorical_indices=None, + ): + site_groups = np.asarray(site_groups) + unique_sites = np.unique(site_groups) + if unique_sites.size < 2: + return CrossValidationResult( + threshold=self.default_threshold, + consensus_params=None, + metrics={ + 'skipped': True, + 'cv_strategy': 'grouped_by_site', + 'reason': 'Not enough hospitals to run grouped cross-validation.', + 'site_groups': unique_sites.tolist(), + }, + ) + + classes = np.unique(labels) + if len(classes) != 2: + return CrossValidationResult( + threshold=self.default_threshold, + consensus_params=None, + metrics={ + 'skipped': True, + 'cv_strategy': 'grouped_by_site', + 'reason': 'Need both classes to run grouped cross-validation.', + 'site_groups': unique_sites.tolist(), + }, + ) + + group_cv = LeaveOneGroupOut() + split_plan = [] + for fold_idx, (train_idx, val_idx) in enumerate(group_cv.split(features, labels, groups=site_groups), start=1): + held_out_site = normalize_site_group(site_groups[val_idx][0]) + split_plan.append(FoldSplit( + fold_index=fold_idx, + train_idx=train_idx, + validation_idx=val_idx, + label=held_out_site, + )) + + consensus, selected_params_per_fold = self._select_consensus_params( + split_plan, + features, + labels, + feature_indices, + categorical_indices=categorical_indices, + site_groups=site_groups, + label_prefix='held-out hospital', + ) + fold_metrics, oof_probabilities = self._evaluate_with_fixed_params( + split_plan, + features, + labels, + feature_indices, + modality_presence_indices, + categorical_indices=categorical_indices, + consensus_params=consensus, + label_prefix='held-out hospital', + extra_metric_fields=lambda split: {'held_out_site': split.label}, + ) + + return self._finalize_result( + consensus_params=consensus, + labels=labels, + oof_probabilities=oof_probabilities, + best_params_per_fold=selected_params_per_fold, + fold_metrics=fold_metrics, + metadata={ + 'cv_strategy': 'grouped_by_site', + 'n_splits': int(len(split_plan)), + 'site_groups': unique_sites.tolist(), + }, + ) + + def _run_random_cv( + self, + features, + labels, + feature_indices, + modality_presence_indices, + categorical_indices=None, + ): + outer_cv, n_splits = self._build_stratified_splitter(labels, self.config.outer_random_splits) + if outer_cv is None: + return CrossValidationResult( + threshold=self.default_threshold, + consensus_params=None, + metrics={ + 'skipped': True, + 'cv_strategy': 'random_stratified', + 'reason': 'Not enough samples per class to run random stratified cross-validation.', + 'requested_n_splits': int(self.config.outer_random_splits), + }, + ) + + split_plan = [ + FoldSplit( + fold_index=fold_idx, + train_idx=train_idx, + validation_idx=val_idx, + label='random stratified split', + ) + for fold_idx, (train_idx, val_idx) in enumerate(outer_cv.split(features, labels), start=1) + ] + consensus, selected_params_per_fold = self._select_consensus_params( + split_plan, + features, + labels, + feature_indices, + categorical_indices=categorical_indices, + site_groups=None, + label_prefix='random split', + ) + fold_metrics, oof_probabilities = self._evaluate_with_fixed_params( + split_plan, + features, + labels, + feature_indices, + modality_presence_indices, + categorical_indices=categorical_indices, + consensus_params=consensus, + label_prefix='random split', + ) + + return self._finalize_result( + consensus_params=consensus, + labels=labels, + oof_probabilities=oof_probabilities, + best_params_per_fold=selected_params_per_fold, + fold_metrics=fold_metrics, + metadata={ + 'cv_strategy': 'random_stratified', + 'n_splits': int(n_splits), + }, + ) + + def _select_consensus_params( + self, + split_plan, + features, + labels, + feature_indices, + categorical_indices=None, + site_groups=None, + label_prefix='fold', + ): + if not self.config.optimize_hyperparameter_search: + fixed_params = dict(self.config.fixed_hyperparameters) + print(f" Hyperparameter search disabled. Using fixed parameters: {fixed_params}") + return fixed_params, [ + { + 'fold': int(split.fold_index), + 'label': split.label, + 'params': fixed_params, + 'source': 'fixed_defaults', + } + for split in split_plan + ] + + selected_params_per_fold = [] + + for split in split_plan: + print(f" Search fold {split.fold_index}/{len(split_plan)} - {label_prefix} {split.label}") + X_train = features[split.train_idx] + y_train = labels[split.train_idx] + search_site_groups = None if site_groups is None else site_groups[split.train_idx] + + fold_preprocessor = self.build_preprocessor(len(y_train), categorical_indices) + X_train_proc = np.asarray(fold_preprocessor.fit_transform(X_train), dtype=np.float32) + fold_best_params = self._search_hyperparams( + X_train_proc[:, feature_indices['all']], + y_train, + site_groups=search_site_groups, + ) + print(f" Best params: {fold_best_params}") + selected_params_per_fold.append({ + 'fold': int(split.fold_index), + 'label': split.label, + 'params': fold_best_params, + 'source': 'search', + }) + + consensus = self._consensus_params([item['params'] for item in selected_params_per_fold]) + print(f" Consensus hyperparameters: {consensus}") + return consensus, selected_params_per_fold + + def _evaluate_with_fixed_params( + self, + split_plan, + features, + labels, + feature_indices, + modality_presence_indices, + categorical_indices=None, + consensus_params=None, + label_prefix='fold', + extra_metric_fields=None, + ): + oof_probabilities = np.zeros(len(labels), dtype=np.float32) + fold_metrics = [] + + for split in split_plan: + print(f" Eval fold {split.fold_index}/{len(split_plan)} - {label_prefix} {split.label}") + X_train, X_val = features[split.train_idx], features[split.validation_idx] + y_train, y_val = labels[split.train_idx], labels[split.validation_idx] + + fold_preprocessor = self.build_preprocessor(len(y_train), categorical_indices) + X_train_proc = np.asarray(fold_preprocessor.fit_transform(X_train), dtype=np.float32) + fold_models = self.fit_ensemble( + X_train_proc, + y_train, + feature_indices, + consensus_params=consensus_params, + ) + fold_bundle = { + 'models': fold_models, + 'feature_indices': feature_indices, + 'modality_presence_indices': modality_presence_indices, + 'preprocessor': fold_preprocessor, + 'threshold': self.default_threshold, + } + + fold_probabilities = self.predict_probabilities(fold_bundle, X_val) + oof_probabilities[split.validation_idx] = fold_probabilities + + fold_metric_row = self._compute_evaluation_metrics( + y_val, + fold_probabilities, + threshold=self.default_threshold, + ) + extra_fields = {} if extra_metric_fields is None else extra_metric_fields(split) + fold_metric_row = { + **fold_metric_row, + 'fold': int(split.fold_index), + 'train_size': int(len(split.train_idx)), + 'validation_size': int(len(split.validation_idx)), + **extra_fields, + } + fold_metrics.append(fold_metric_row) + self._print_metrics(f" Fold {split.fold_index} metrics:", fold_metric_row) + + return fold_metrics, oof_probabilities + + def _finalize_result(self, consensus_params, labels, oof_probabilities, best_params_per_fold, fold_metrics, metadata): + consensus = dict(consensus_params or {}) + + fold_metric_summary = self._summarize_metrics(fold_metrics) + self._print_metric_summary(" Mean fold metrics:", fold_metric_summary) + + oof_default_metrics = self._compute_evaluation_metrics( + labels, + oof_probabilities, + threshold=self.default_threshold, + ) + self._print_metrics(" OOF metrics before calibration:", oof_default_metrics) + + threshold = self._best_threshold(oof_probabilities, labels) + oof_calibrated_metrics = self._compute_evaluation_metrics( + labels, + oof_probabilities, + threshold=threshold, + ) + self._print_metrics(" OOF metrics after calibration:", oof_calibrated_metrics) + + metrics = { + 'skipped': False, + **metadata, + 'hyperparameter_optimization_enabled': bool(self.config.optimize_hyperparameter_search), + 'fixed_hyperparameters': dict(self.config.fixed_hyperparameters), + 'selected_params_per_fold': best_params_per_fold, + 'fold_metrics': fold_metrics, + 'fold_metric_summary': fold_metric_summary, + 'oof_default_threshold_metrics': oof_default_metrics, + 'oof_calibrated_metrics': oof_calibrated_metrics, + } + return CrossValidationResult( + threshold=threshold, + consensus_params=consensus, + metrics=metrics, + ) + + def _search_hyperparams(self, X_train, y_train, site_groups=None): + inner_cv, fit_kwargs = self._build_inner_cv(y_train, site_groups) + if inner_cv is None: + return {} + + search = RandomizedSearchCV( + estimator=self.build_search_model(y_train), + param_distributions=self.param_dist, + n_iter=self.config.search_iterations, + scoring='f1', + cv=inner_cv, + random_state=self.config.random_state, + n_jobs=-1, + refit=False, + ) + search.fit(X_train, y_train, **fit_kwargs) + return search.best_params_ + + def _build_inner_cv(self, y_train, site_groups=None): + fit_kwargs = {} + + if self.config.use_site_grouped_cv and site_groups is not None: + site_groups = np.asarray(site_groups) + unique_groups = np.unique(site_groups) + if unique_groups.size >= 2: + fit_kwargs['groups'] = site_groups + return LeaveOneGroupOut(), fit_kwargs + + inner_cv, _ = self._build_stratified_splitter(y_train, self.config.outer_random_splits) + return inner_cv, fit_kwargs + + def _build_stratified_splitter(self, labels, requested_splits): + classes, class_counts = np.unique(labels, return_counts=True) + if len(classes) != 2: + return None, 0 + + n_splits = min(int(requested_splits), int(np.min(class_counts))) + if n_splits < 2: + return None, 0 + + return StratifiedKFold( + n_splits=n_splits, + shuffle=True, + random_state=self.config.random_state, + ), n_splits + + def _best_threshold(self, probabilities, labels): + thresholds = np.linspace(0, 1, 101) + best_score = -1.0 + best_value = self.default_threshold + + for threshold in thresholds: + predictions = (probabilities >= threshold).astype(np.int32) + score = f1_score(labels, predictions, zero_division=0) + if score > best_score: + best_score = score + best_value = float(threshold) + + return best_value + + def _safe_auroc(self, labels, probabilities): + if len(np.unique(labels)) < 2: + return np.nan + return float(roc_auc_score(labels, probabilities)) + + def _safe_auprc(self, labels, probabilities): + if len(np.unique(labels)) < 2: + return np.nan + return float(average_precision_score(labels, probabilities, pos_label=1)) + + def _compute_evaluation_metrics(self, labels, probabilities, threshold): + labels = np.asarray(labels, dtype=np.int32) + probabilities = np.asarray(probabilities, dtype=np.float32) + predictions = (probabilities >= threshold).astype(np.int32) + + return { + 'threshold': float(threshold), + 'auroc': self._safe_auroc(labels, probabilities), + 'auprc': self._safe_auprc(labels, probabilities), + 'accuracy': float(accuracy_score(labels, predictions)), + 'f1': float(f1_score(labels, predictions, zero_division=0)), + 'precision': float(precision_score(labels, predictions, zero_division=0)), + 'recall': float(recall_score(labels, predictions, zero_division=0)), + } + + def _summarize_metrics(self, metric_rows): + summary = {} + metric_names = ('auroc', 'auprc', 'accuracy', 'f1', 'precision', 'recall') + + for metric_name in metric_names: + metric_values = np.asarray([row[metric_name] for row in metric_rows], dtype=np.float32) + finite_values = metric_values[np.isfinite(metric_values)] + if finite_values.size == 0: + summary[metric_name] = {'mean': None, 'std': None} + continue + + summary[metric_name] = { + 'mean': float(np.mean(finite_values)), + 'std': float(np.std(finite_values)), + } + + return summary + + def _consensus_params(self, params_per_fold): + consensus = {} + for key in self.param_dist.keys(): + values = [params[key] for params in params_per_fold if key in params] + if not values: + continue + consensus[key] = max(set(values), key=values.count) + return consensus + + def _format_metric_value(self, value): + return 'nan' if value is None or not np.isfinite(value) else f'{value:.3f}' + + def _print_metrics(self, prefix, metrics): + print( + f"{prefix} AUROC={self._format_metric_value(metrics['auroc'])}, " + f"AUPRC={self._format_metric_value(metrics['auprc'])}, " + f"Accuracy={self._format_metric_value(metrics['accuracy'])}, " + f"F1={self._format_metric_value(metrics['f1'])}, " + f"Precision={self._format_metric_value(metrics['precision'])}, " + f"Recall={self._format_metric_value(metrics['recall'])}, " + f"Threshold={metrics['threshold']:.2f}" + ) + + def _print_metric_summary(self, prefix, summary): + metric_names = ('auroc', 'auprc', 'accuracy', 'f1', 'precision', 'recall') + parts = [] + for metric_name in metric_names: + metric_summary = summary.get(metric_name, {}) + mean_value = self._format_metric_value(metric_summary.get('mean')) + std_value = self._format_metric_value(metric_summary.get('std')) + parts.append(f"{metric_name.upper()}={mean_value} +/- {std_value}") + print(f"{prefix} {', '.join(parts)}") \ No newline at end of file diff --git a/src/pipeline/training.py b/src/pipeline/training.py index a135999..1baf832 100644 --- a/src/pipeline/training.py +++ b/src/pipeline/training.py @@ -1,12 +1,9 @@ import os -from collections import Counter from concurrent.futures import ThreadPoolExecutor import numpy as np import pandas as pd from sklearn.impute import KNNImputer -from sklearn.metrics import accuracy_score, average_precision_score, f1_score, precision_score, recall_score, roc_auc_score -from sklearn.model_selection import train_test_split, RandomizedSearchCV, StratifiedKFold from sklearn.pipeline import Pipeline from sklearn.preprocessing import StandardScaler from tqdm import tqdm @@ -14,7 +11,16 @@ from helper_code import DEMOGRAPHICS_FILE, HEADERS, find_patients, load_label -from .config import MAX_TRAIN_WORKERS +from .config import ( + CV_RANDOM_STATE, + CV_SEARCH_ITERATIONS, + DEFAULT_CV_HYPERPARAMETERS, + MAX_TRAIN_WORKERS, + OPTIMIZE_HYPERPARAMETER_SEARCH, + RANDOM_CV_N_SPLITS, + USE_SITE_GROUPED_CV, +) +from .cross_validation import CrossValidationConfig, EnsembleCrossValidator, normalize_site_group from .features import get_feature_group_indices, get_feature_names, get_or_create_record_feature_vector @@ -139,95 +145,6 @@ def export_feature_views(export_root, prefix, metadata_rows, feature_matrix, fea return export_paths -def best_threshold(probabilities, labels): - thresholds = np.linspace(0, 1, 101) - best_score = -1.0 - best_value = DEFAULT_ENSEMBLE_THRESHOLD - - for threshold in thresholds: - predictions = (probabilities >= threshold).astype(np.int32) - score = f1_score(labels, predictions, zero_division=0) - if score > best_score: - best_score = score - best_value = float(threshold) - - return best_value - - -def _safe_auroc(labels, probabilities): - if len(np.unique(labels)) < 2: - return np.nan - return float(roc_auc_score(labels, probabilities)) - - -def _safe_auprc(labels, probabilities): - if len(np.unique(labels)) < 2: - return np.nan - return float(average_precision_score(labels, probabilities, pos_label=1)) - - -def _compute_evaluation_metrics(labels, probabilities, threshold): - labels = np.asarray(labels, dtype=np.int32) - probabilities = np.asarray(probabilities, dtype=np.float32) - predictions = (probabilities >= threshold).astype(np.int32) - - return { - 'threshold': float(threshold), - 'auroc': _safe_auroc(labels, probabilities), - 'auprc': _safe_auprc(labels, probabilities), - 'accuracy': float(accuracy_score(labels, predictions)), - 'f1': float(f1_score(labels, predictions, zero_division=0)), - 'precision': float(precision_score(labels, predictions, zero_division=0)), - 'recall': float(recall_score(labels, predictions, zero_division=0)), - } - - -def _summarize_metrics(metric_rows): - summary = {} - metric_names = ('auroc', 'auprc', 'accuracy', 'f1', 'precision', 'recall') - - for metric_name in metric_names: - metric_values = np.asarray([row[metric_name] for row in metric_rows], dtype=np.float32) - finite_values = metric_values[np.isfinite(metric_values)] - if finite_values.size == 0: - summary[metric_name] = {'mean': None, 'std': None} - continue - - summary[metric_name] = { - 'mean': float(np.mean(finite_values)), - 'std': float(np.std(finite_values)), - } - - return summary - - -def _format_metric_value(value): - return 'nan' if value is None or not np.isfinite(value) else f'{value:.3f}' - - -def _print_metrics(prefix, metrics): - print( - f"{prefix} AUROC={_format_metric_value(metrics['auroc'])}, " - f"AUPRC={_format_metric_value(metrics['auprc'])}, " - f"Accuracy={_format_metric_value(metrics['accuracy'])}, " - f"F1={_format_metric_value(metrics['f1'])}, " - f"Precision={_format_metric_value(metrics['precision'])}, " - f"Recall={_format_metric_value(metrics['recall'])}, " - f"Threshold={metrics['threshold']:.2f}" - ) - - -def _print_metric_summary(prefix, summary): - metric_names = ('auroc', 'auprc', 'accuracy', 'f1', 'precision', 'recall') - parts = [] - for metric_name in metric_names: - metric_summary = summary.get(metric_name, {}) - mean_value = _format_metric_value(metric_summary.get('mean')) - std_value = _format_metric_value(metric_summary.get('std')) - parts.append(f"{metric_name.upper()}={mean_value} +/- {std_value}") - print(f"{prefix} {', '.join(parts)}") - - def _build_xgb_model(labels, extra_params=None): neg = int(np.sum(labels == 0)) @@ -259,6 +176,17 @@ def _fit_model(feature_matrix, labels, consensus_params=None): return model +def _build_search_model(labels): + return XGBClassifier( + scale_pos_weight=(int(np.sum(labels == 0)) / max(int(np.sum(labels == 1)), 1)), + n_estimators=500, + learning_rate=0.05, + random_state=CV_RANDOM_STATE, + eval_metric='auc', + tree_method='hist', + ) + + def _build_preprocessor(num_samples, categorical_indices=None): @@ -319,42 +247,6 @@ def transform(self, X): X_out[:, self._numerical_indices] = X_num_scaled return X_out -def _search_hyperparams(X_train, y_train): - - base_model = XGBClassifier( - scale_pos_weight=(int(np.sum(y_train == 0)) / max(int(np.sum(y_train == 1)), 1)), - n_estimators=500, - learning_rate=0.05, - random_state=42, - eval_metric='auc', - tree_method='hist', - ) - - search = RandomizedSearchCV( - estimator=base_model, - param_distributions=PARAM_DIST, - n_iter=20, - scoring='f1', - cv=StratifiedKFold(n_splits=3, shuffle=True, random_state=42), - random_state=42, - n_jobs=-1, - refit=False, # we only need best_params_, not a refitted model - ) - search.fit(X_train, y_train) - return search.best_params_ - -def _consensus_params(params_per_fold): - - consensus = {} - for key in PARAM_DIST.keys(): - values = [p[key] for p in params_per_fold if key in p] - if not values: - continue - # majority vote - most_common = Counter(values).most_common(1)[0][0] - consensus[key] = most_common - return consensus - def _fit_ensemble(feature_matrix, labels, feature_indices, consensus_params=None): models = { 'all': _fit_model( @@ -414,104 +306,6 @@ def predict_ensemble_labels(model_bundle, feature_matrix): labels = (probabilities >= threshold).astype(np.int32) return labels, probabilities -def _calibrate_threshold_cv( - features, - labels, - feature_indices, - modality_presence_indices, - categorical_indices=None, - n_splits=5, -): - - classes, class_counts = np.unique(labels, return_counts=True) - if len(classes) != 2 or np.min(class_counts) < n_splits: - return DEFAULT_ENSEMBLE_THRESHOLD, None, { - 'skipped': True, - 'reason': 'Not enough samples per class to run stratified cross-validation.', - 'n_splits': int(n_splits), - } - - skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42) - oof_probabilities = np.zeros(len(labels), dtype=np.float32) - best_params_per_fold = [] - fold_metrics = [] - - for fold_idx, (train_idx, val_idx) in enumerate(skf.split(features, labels)): - print(f" Outer fold {fold_idx + 1}/{n_splits} — hyperparameter search + OOF prediction") - - X_train, X_val = features[train_idx], features[val_idx] - y_train, y_val = labels[train_idx], labels[val_idx] - - # Fit preprocessor ONLY on fold's training data (no leakage into val) - fold_preprocessor = _build_preprocessor(len(y_train), categorical_indices) - X_train_proc = np.asarray(fold_preprocessor.fit_transform(X_train), dtype=np.float32) - X_val_proc = np.asarray(fold_preprocessor.transform(X_val), dtype=np.float32) - - # Inner CV hyperparameter search on this fold's training data - fold_best_params = _search_hyperparams(X_train_proc[:, feature_indices['all']], y_train) - best_params_per_fold.append(fold_best_params) - print(f" Best params fold {fold_idx + 1}: {fold_best_params}") - - # Fit fold ensemble with the fold's best params - fold_models = _fit_ensemble(X_train_proc, y_train, feature_indices, consensus_params=fold_best_params) - fold_bundle = { - 'models': fold_models, - 'feature_indices': feature_indices, - 'modality_presence_indices': modality_presence_indices, - 'preprocessor': fold_preprocessor, - 'threshold': DEFAULT_ENSEMBLE_THRESHOLD, - } - - # Collect out-of-fold probabilities (never seen by this fold's model during training) - fold_probabilities = predict_ensemble_probabilities(fold_bundle, X_val) - oof_probabilities[val_idx] = fold_probabilities - - fold_metric_row = _compute_evaluation_metrics( - y_val, - fold_probabilities, - threshold=DEFAULT_ENSEMBLE_THRESHOLD, - ) - fold_metric_row.update({ - 'fold': int(fold_idx + 1), - 'train_size': int(len(train_idx)), - 'validation_size': int(len(val_idx)), - }) - fold_metrics.append(fold_metric_row) - _print_metrics(f" Fold {fold_idx + 1} metrics:", fold_metric_row) - - # Aggregate best params across all outer folds by majority vote - consensus = _consensus_params(best_params_per_fold) - print(f" Consensus hyperparameters across {n_splits} folds: {consensus}") - - fold_metric_summary = _summarize_metrics(fold_metrics) - _print_metric_summary(" Mean fold metrics:", fold_metric_summary) - - # Threshold calibrated on clean OOF probabilities - oof_default_metrics = _compute_evaluation_metrics( - labels, - oof_probabilities, - threshold=DEFAULT_ENSEMBLE_THRESHOLD, - ) - _print_metrics(" OOF metrics before calibration:", oof_default_metrics) - - threshold = best_threshold(oof_probabilities, labels) - - oof_calibrated_metrics = _compute_evaluation_metrics( - labels, - oof_probabilities, - threshold=threshold, - ) - _print_metrics(" OOF metrics after calibration:", oof_calibrated_metrics) - - return threshold, consensus, { - 'skipped': False, - 'n_splits': int(n_splits), - 'fold_metrics': fold_metrics, - 'fold_metric_summary': fold_metric_summary, - 'oof_default_threshold_metrics': oof_default_metrics, - 'oof_calibrated_metrics': oof_calibrated_metrics, - } - def train_multimodal_ensemble(data_folder, verbose, csv_path, export_folder=None): patient_data_file = os.path.join(data_folder, DEMOGRAPHICS_FILE) @@ -567,18 +361,46 @@ def train_multimodal_ensemble(data_folder, verbose, csv_path, export_folder=None i for i, name in enumerate(feature_names) if name.lower() in ('sex', 'gender') ] + site_groups = np.asarray([ + normalize_site_group(metadata_row['site_id']) + for metadata_row in metadata_rows + ]) + cv_config = CrossValidationConfig( + use_site_grouped_cv=USE_SITE_GROUPED_CV, + optimize_hyperparameter_search=OPTIMIZE_HYPERPARAMETER_SEARCH, + outer_random_splits=RANDOM_CV_N_SPLITS, + random_state=CV_RANDOM_STATE, + search_iterations=CV_SEARCH_ITERATIONS, + fixed_hyperparameters=DEFAULT_CV_HYPERPARAMETERS, + ) + cv_runner = EnsembleCrossValidator( + config=cv_config, + param_dist=PARAM_DIST, + default_threshold=DEFAULT_ENSEMBLE_THRESHOLD, + build_preprocessor=_build_preprocessor, + build_search_model=_build_search_model, + fit_ensemble=_fit_ensemble, + predict_probabilities=predict_ensemble_probabilities, + ) print(f" Categorical feature indices: {categorical_indices} " f"({[feature_names[i] for i in categorical_indices]})") + print(f" Hospital CV groups: {sorted(np.unique(site_groups).tolist())}") + print(f" CV strategy: {'grouped by hospital' if cv_config.use_site_grouped_cv else 'random stratified folds'}") + print(f" Hyperparameter search: {'enabled' if cv_config.optimize_hyperparameter_search else 'disabled'}") # --- Step 1: Nested CV for threshold calibration and consensus hyperparameters --- print("Running nested CV for threshold calibration and hyperparameter consensus...") - threshold, consensus, cv_metrics = _calibrate_threshold_cv( + cv_result = cv_runner.run( features, labels, feature_indices, modality_presence_indices, categorical_indices=categorical_indices if categorical_indices else None, + site_groups=site_groups, ) + threshold = cv_result.threshold + consensus = cv_result.consensus_params + cv_metrics = cv_result.metrics # --- Step 2: Fit final models on ALL data using consensus hyperparameters --- print("Fitting final ensemble on all training data with consensus hyperparameters...") From 19938f72b839d6d70a85ac754814d0dc8d6fd64e Mon Sep 17 00:00:00 2001 From: dcajal Date: Wed, 8 Apr 2026 23:10:56 +0200 Subject: [PATCH 83/93] Add feature correlation threshold --- src/pipeline/config.py | 1 + src/pipeline/cross_validation.py | 14 ++- src/pipeline/preprocessing.py | 160 +++++++++++++++++++++++++++++++ src/pipeline/training.py | 130 +++++++++++-------------- team_code.py | 2 + 5 files changed, 230 insertions(+), 77 deletions(-) create mode 100644 src/pipeline/preprocessing.py diff --git a/src/pipeline/config.py b/src/pipeline/config.py index 4dc386c..5f4f9a9 100644 --- a/src/pipeline/config.py +++ b/src/pipeline/config.py @@ -8,6 +8,7 @@ SCRIPT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')) DEFAULT_CSV_PATH = os.path.join(SCRIPT_DIR, 'channel_table.csv') FEATURE_CACHE_FOLDER_NAME = '.feature_cache' +FEATURE_CORRELATION_THRESHOLD = 0.95 MAX_TRAIN_WORKERS = max(1, min(4, os.cpu_count() or 1)) USE_SITE_GROUPED_CV = True OPTIMIZE_HYPERPARAMETER_SEARCH = False diff --git a/src/pipeline/cross_validation.py b/src/pipeline/cross_validation.py index e4c01e4..5dd6331 100644 --- a/src/pipeline/cross_validation.py +++ b/src/pipeline/cross_validation.py @@ -259,8 +259,12 @@ def _select_consensus_params( fold_preprocessor = self.build_preprocessor(len(y_train), categorical_indices) X_train_proc = np.asarray(fold_preprocessor.fit_transform(X_train), dtype=np.float32) + remapped_feature_indices = fold_preprocessor.transform_feature_indices(feature_indices) + print( + f" Correlation selector kept {X_train_proc.shape[1]}/{X_train.shape[1]} features" + ) fold_best_params = self._search_hyperparams( - X_train_proc[:, feature_indices['all']], + X_train_proc[:, remapped_feature_indices['all']], y_train, site_groups=search_site_groups, ) @@ -298,15 +302,19 @@ def _evaluate_with_fixed_params( fold_preprocessor = self.build_preprocessor(len(y_train), categorical_indices) X_train_proc = np.asarray(fold_preprocessor.fit_transform(X_train), dtype=np.float32) + remapped_feature_indices = fold_preprocessor.transform_feature_indices(feature_indices) + print( + f" Correlation selector kept {X_train_proc.shape[1]}/{X_train.shape[1]} features" + ) fold_models = self.fit_ensemble( X_train_proc, y_train, - feature_indices, + remapped_feature_indices, consensus_params=consensus_params, ) fold_bundle = { 'models': fold_models, - 'feature_indices': feature_indices, + 'feature_indices': remapped_feature_indices, 'modality_presence_indices': modality_presence_indices, 'preprocessor': fold_preprocessor, 'threshold': self.default_threshold, diff --git a/src/pipeline/preprocessing.py b/src/pipeline/preprocessing.py new file mode 100644 index 0000000..cf94978 --- /dev/null +++ b/src/pipeline/preprocessing.py @@ -0,0 +1,160 @@ +import numpy as np +from sklearn.impute import KNNImputer +from sklearn.preprocessing import StandardScaler + +from .config import FEATURE_CORRELATION_THRESHOLD + + +DEFAULT_KNN_NEIGHBORS = 5 + + +def build_preprocessor(num_samples, categorical_indices=None): + neighbors = min(DEFAULT_KNN_NEIGHBORS, max(1, num_samples - 1)) if num_samples > 1 else 1 + return CorrelationAwarePreprocessor( + n_neighbors=neighbors, + categorical_indices=categorical_indices, + correlation_threshold=FEATURE_CORRELATION_THRESHOLD, + ) + + +def get_processed_feature_names(feature_names, preprocessor=None): + if preprocessor is None: + return list(feature_names) + + if hasattr(preprocessor, 'get_feature_names_out'): + return list(preprocessor.get_feature_names_out(feature_names)) + + return list(feature_names) + + +def remap_feature_indices(preprocessor, feature_indices): + if preprocessor is None or not hasattr(preprocessor, 'transform_feature_indices'): + return { + name: np.asarray(indices, dtype=np.int32) + for name, indices in feature_indices.items() + } + + return preprocessor.transform_feature_indices(feature_indices) + + +class CorrelationThresholdSelector: + def __init__(self, threshold): + self.threshold = float(threshold) + self.selected_indices_ = None + + def fit(self, X): + X = np.asarray(X, dtype=np.float32) + n_samples, n_features = X.shape + + if n_features == 0: + self.selected_indices_ = np.array([], dtype=np.int32) + return self + + if n_samples < 2 or n_features == 1: + self.selected_indices_ = np.arange(n_features, dtype=np.int32) + return self + + with np.errstate(divide='ignore', invalid='ignore'): + corr = np.corrcoef(X, rowvar=False) + corr = np.asarray(corr, dtype=np.float32) + corr = np.nan_to_num(np.abs(corr), nan=0.0, posinf=0.0, neginf=0.0) + np.fill_diagonal(corr, 0.0) + + keep_mask = np.ones(n_features, dtype=bool) + for index in range(n_features): + if not keep_mask[index]: + continue + + correlated_indices = np.where(corr[index, index + 1:] > self.threshold)[0] + if correlated_indices.size: + keep_mask[correlated_indices + index + 1] = False + + if not np.any(keep_mask): + keep_mask[0] = True + + self.selected_indices_ = np.flatnonzero(keep_mask).astype(np.int32) + return self + + def transform(self, X): + X = np.asarray(X, dtype=np.float32) + if self.selected_indices_ is None: + raise ValueError('Correlation selector has not been fitted.') + return X[:, self.selected_indices_] + + def get_feature_names_out(self, input_features=None): + if self.selected_indices_ is None: + raise ValueError('Correlation selector has not been fitted.') + + if input_features is None: + input_features = [f'feature_{index}' for index in self.selected_indices_] + + return np.asarray([input_features[index] for index in self.selected_indices_], dtype=object) + + +class CorrelationAwarePreprocessor: + def __init__(self, n_neighbors, categorical_indices, correlation_threshold): + self.n_neighbors = n_neighbors + if categorical_indices is None: + self.categorical_indices = np.array([], dtype=np.int32) + else: + self.categorical_indices = np.asarray(categorical_indices, dtype=np.int32) + self.imputer = KNNImputer(n_neighbors=n_neighbors, keep_empty_features=True) + self.scaler = StandardScaler() + self.selector = CorrelationThresholdSelector(correlation_threshold) + self._numerical_indices = np.array([], dtype=np.int32) + + def _get_numerical_indices(self, n_features): + all_idx = np.arange(n_features, dtype=np.int32) + return np.setdiff1d(all_idx, self.categorical_indices) + + def _scale_numerical_columns(self, X_imputed, fit=False): + X_out = X_imputed.copy() + if self._numerical_indices.size == 0: + return X_out + + X_num = X_imputed[:, self._numerical_indices] + if fit: + X_num_scaled = np.asarray(self.scaler.fit_transform(X_num), dtype=np.float32) + else: + X_num_scaled = np.asarray(self.scaler.transform(X_num), dtype=np.float32) + + X_out[:, self._numerical_indices] = X_num_scaled + return X_out + + def fit_transform(self, X): + X = np.asarray(X, dtype=np.float32).copy() + X[~np.isfinite(X)] = np.nan + + X_imputed = np.asarray(self.imputer.fit_transform(X), dtype=np.float32) + self._numerical_indices = self._get_numerical_indices(X.shape[1]) + X_out = self._scale_numerical_columns(X_imputed, fit=True) + self.selector.fit(X_out) + return np.asarray(self.selector.transform(X_out), dtype=np.float32) + + def transform(self, X): + X = np.asarray(X, dtype=np.float32).copy() + X[~np.isfinite(X)] = np.nan + X_imputed = np.asarray(self.imputer.transform(X), dtype=np.float32) + X_out = self._scale_numerical_columns(X_imputed, fit=False) + return np.asarray(self.selector.transform(X_out), dtype=np.float32) + + def transform_feature_indices(self, feature_indices): + if self.selector.selected_indices_ is None: + raise ValueError('Preprocessor has not been fitted.') + + index_lookup = { + int(raw_index): int(processed_index) + for processed_index, raw_index in enumerate(self.selector.selected_indices_) + } + remapped = {} + for name, indices in feature_indices.items(): + kept_indices = [ + index_lookup[int(raw_index)] + for raw_index in np.asarray(indices, dtype=np.int32) + if int(raw_index) in index_lookup + ] + remapped[name] = np.asarray(kept_indices, dtype=np.int32) + return remapped + + def get_feature_names_out(self, input_features=None): + return self.selector.get_feature_names_out(input_features) \ No newline at end of file diff --git a/src/pipeline/training.py b/src/pipeline/training.py index 1baf832..b0087ef 100644 --- a/src/pipeline/training.py +++ b/src/pipeline/training.py @@ -3,9 +3,6 @@ import numpy as np import pandas as pd -from sklearn.impute import KNNImputer -from sklearn.pipeline import Pipeline -from sklearn.preprocessing import StandardScaler from tqdm import tqdm from xgboost import XGBClassifier @@ -22,11 +19,11 @@ ) from .cross_validation import CrossValidationConfig, EnsembleCrossValidator, normalize_site_group from .features import get_feature_group_indices, get_feature_names, get_or_create_record_feature_vector +from .preprocessing import build_preprocessor, get_processed_feature_names, remap_feature_indices DEFAULT_ENSEMBLE_THRESHOLD = 0.5 ENSEMBLE_MODALITIES = ('resp', 'eeg', 'ecg') -DEFAULT_KNN_NEIGHBORS = 5 # Hyperparameter search space PARAM_DIST = { @@ -121,6 +118,29 @@ def get_feature_export_paths(export_root, prefix): 'raw': os.path.join(export_root, f'{prefix}_features_raw.csv'), 'preprocessed': os.path.join(export_root, f'{prefix}_features_preprocessed.csv'), } + + +def _get_feature_group_name(feature_index, modality_presence_indices): + for group_name in ('resp', 'eeg', 'ecg'): + group_index_set = set(np.asarray(modality_presence_indices[group_name], dtype=np.int32).tolist()) + if feature_index in group_index_set: + return group_name + + return 'demographics' + + +def export_selected_features_csv(output_path, feature_names, selected_raw_feature_indices, modality_presence_indices): + selected_rows = [] + for processed_index, raw_index in enumerate(np.asarray(selected_raw_feature_indices, dtype=np.int32)): + selected_rows.append({ + 'processed_index': int(processed_index), + 'raw_index': int(raw_index), + 'feature_name': feature_names[int(raw_index)], + 'group': _get_feature_group_name(int(raw_index), modality_presence_indices), + }) + + os.makedirs(os.path.dirname(output_path), exist_ok=True) + pd.DataFrame(selected_rows).to_csv(output_path, index=False) def export_feature_views(export_root, prefix, metadata_rows, feature_matrix, feature_names, preprocessor=None, labels=None): raw_feature_matrix, processed_feature_matrix = prepare_feature_matrix( @@ -139,7 +159,7 @@ def export_feature_views(export_root, prefix, metadata_rows, feature_matrix, fea export_paths['preprocessed'], metadata_rows, processed_feature_matrix, - feature_names, + get_processed_feature_names(feature_names, preprocessor=preprocessor), labels=labels, ) return export_paths @@ -186,74 +206,17 @@ def _build_search_model(labels): tree_method='hist', ) +def _fit_ensemble(feature_matrix, labels, feature_indices, consensus_params=None): + models = {} + if feature_indices['all'].size == 0: + raise ValueError('Correlation selector removed all features for the global model.') - -def _build_preprocessor(num_samples, categorical_indices=None): - - neighbors = min(DEFAULT_KNN_NEIGHBORS, max(1, num_samples - 1)) if num_samples > 1 else 1 - - if categorical_indices is None or len(categorical_indices) == 0: - # Original pipeline — no categorical columns - return Pipeline([ - ('imputer', KNNImputer(n_neighbors=neighbors, keep_empty_features=True)), - ('scaler', StandardScaler()), - ]) - - return _CategoricalAwarePreprocessor( - n_neighbors=neighbors, - categorical_indices=np.asarray(categorical_indices, dtype=np.int32), + models['all'] = _fit_model( + feature_matrix[:, feature_indices['all']], labels, consensus_params ) - -class _CategoricalAwarePreprocessor: - - def __init__(self, n_neighbors, categorical_indices): - self.n_neighbors = n_neighbors - self.categorical_indices = categorical_indices - self.imputer = KNNImputer(n_neighbors=n_neighbors, keep_empty_features=True) - self.scaler = StandardScaler() - self._numerical_indices = None # set during fit - - def _get_numerical_indices(self, n_features): - all_idx = np.arange(n_features) - return np.setdiff1d(all_idx, self.categorical_indices) - - def fit_transform(self, X): - X = np.asarray(X, dtype=np.float32).copy() - X[~np.isfinite(X)] = np.nan - - # Step 1: impute everything - X_imputed = np.asarray(self.imputer.fit_transform(X), dtype=np.float32) - - # Step 2: fit scaler on numerical columns only - self._numerical_indices = self._get_numerical_indices(X.shape[1]) - X_num = X_imputed[:, self._numerical_indices] - X_num_scaled = np.asarray(self.scaler.fit_transform(X_num), dtype=np.float32) - - # Step 3: assemble output — categorical columns keep imputed values - X_out = X_imputed.copy() - X_out[:, self._numerical_indices] = X_num_scaled - return X_out - - def transform(self, X): - X = np.asarray(X, dtype=np.float32).copy() - X[~np.isfinite(X)] = np.nan - - X_imputed = np.asarray(self.imputer.transform(X), dtype=np.float32) - - X_num = X_imputed[:, self._numerical_indices] - X_num_scaled = np.asarray(self.scaler.transform(X_num), dtype=np.float32) - - X_out = X_imputed.copy() - X_out[:, self._numerical_indices] = X_num_scaled - return X_out - -def _fit_ensemble(feature_matrix, labels, feature_indices, consensus_params=None): - models = { - 'all': _fit_model( - feature_matrix[:, feature_indices['all']], labels, consensus_params - ), - } for modality in ENSEMBLE_MODALITIES: + if feature_indices[modality].size == 0: + continue models[modality] = _fit_model( feature_matrix[:, feature_indices[modality]], labels, consensus_params ) @@ -286,6 +249,8 @@ def predict_ensemble_probabilities(model_bundle, feature_matrix): processed_feature_vector = processed_feature_matrix[row_index] modality_probabilities = [] for modality in ENSEMBLE_MODALITIES: + if modality not in models or feature_indices[modality].size == 0: + continue if _has_modality_signal(raw_feature_vector, modality_presence_indices[modality]): modality_vector = processed_feature_vector[feature_indices[modality]].reshape(1, -1) modality_probability = models[modality].predict_proba(modality_vector)[0][1] @@ -377,7 +342,7 @@ def train_multimodal_ensemble(data_folder, verbose, csv_path, export_folder=None config=cv_config, param_dist=PARAM_DIST, default_threshold=DEFAULT_ENSEMBLE_THRESHOLD, - build_preprocessor=_build_preprocessor, + build_preprocessor=build_preprocessor, build_search_model=_build_search_model, fit_ensemble=_fit_ensemble, predict_probabilities=predict_ensemble_probabilities, @@ -387,6 +352,7 @@ def train_multimodal_ensemble(data_folder, verbose, csv_path, export_folder=None print(f" Hospital CV groups: {sorted(np.unique(site_groups).tolist())}") print(f" CV strategy: {'grouped by hospital' if cv_config.use_site_grouped_cv else 'random stratified folds'}") print(f" Hyperparameter search: {'enabled' if cv_config.optimize_hyperparameter_search else 'disabled'}") + print(' Feature selection is fitted inside each CV fold and then re-fitted on all training data for the final model.') # --- Step 1: Nested CV for threshold calibration and consensus hyperparameters --- print("Running nested CV for threshold calibration and hyperparameter consensus...") @@ -404,9 +370,15 @@ def train_multimodal_ensemble(data_folder, verbose, csv_path, export_folder=None # --- Step 2: Fit final models on ALL data using consensus hyperparameters --- print("Fitting final ensemble on all training data with consensus hyperparameters...") - preprocessor = _build_preprocessor(len(labels), categorical_indices if categorical_indices else None) + preprocessor = build_preprocessor(len(labels), categorical_indices if categorical_indices else None) processed_features = np.asarray(preprocessor.fit_transform(features), dtype=np.float32) - models = _fit_ensemble(processed_features, labels, feature_indices, consensus_params=consensus) + selected_feature_indices = remap_feature_indices(preprocessor, feature_indices) + processed_feature_names = get_processed_feature_names(feature_names, preprocessor=preprocessor) + selected_raw_feature_indices = np.asarray(preprocessor.selector.selected_indices_, dtype=np.int32) + print( + f"Correlation selector: kept {len(processed_feature_names)}/{len(feature_names)} features" + ) + models = _fit_ensemble(processed_features, labels, selected_feature_indices, consensus_params=consensus) export_root = export_folder or os.path.join(os.getcwd(), 'feature_exports') feature_exports = export_feature_views( @@ -418,14 +390,24 @@ def train_multimodal_ensemble(data_folder, verbose, csv_path, export_folder=None preprocessor=preprocessor, labels=labels, ) + selected_features_csv = os.path.join(export_root, 'training_features_selected.csv') + export_selected_features_csv( + selected_features_csv, + feature_names, + selected_raw_feature_indices, + modality_presence_indices, + ) + feature_exports['selected'] = selected_features_csv return { 'type': 'multimodal_xgb_ensemble', 'threshold': threshold, 'feature_names': feature_names, + 'processed_feature_names': processed_feature_names, + 'selected_raw_feature_indices': selected_raw_feature_indices.tolist(), 'feature_indices': { name: indices.tolist() - for name, indices in feature_indices.items() + for name, indices in selected_feature_indices.items() if name in {'all', 'resp', 'eeg', 'ecg'} }, 'modality_presence_indices': { diff --git a/team_code.py b/team_code.py index 9cd788a..86386ee 100644 --- a/team_code.py +++ b/team_code.py @@ -129,6 +129,8 @@ def train_model(data_folder, model_folder, verbose, csv_path=DEFAULT_CSV_PATH): if verbose and feature_exports: print(f"Raw features CSV: {feature_exports.get('raw')}") print(f"Preprocessed features CSV: {feature_exports.get('preprocessed')}") + if feature_exports.get('selected'): + print(f"Selected features CSV: {feature_exports.get('selected')}") if verbose: print('Done.') From 538c6061b2b3f99a69f4362de9417dbb193729c8 Mon Sep 17 00:00:00 2001 From: dcajal Date: Wed, 8 Apr 2026 23:48:40 +0200 Subject: [PATCH 84/93] Add mean calculation condition --- src/pipeline/config.py | 16 +++++++++++----- src/pipeline/features.py | 8 ++------ 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/pipeline/config.py b/src/pipeline/config.py index 5f4f9a9..becbf0d 100644 --- a/src/pipeline/config.py +++ b/src/pipeline/config.py @@ -8,9 +8,15 @@ SCRIPT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')) DEFAULT_CSV_PATH = os.path.join(SCRIPT_DIR, 'channel_table.csv') FEATURE_CACHE_FOLDER_NAME = '.feature_cache' -FEATURE_CORRELATION_THRESHOLD = 0.95 +INCLUDE_SUBJECT_FEATURE_MEAN = False +SEGMENT_AGGREGATION_NAMES = ( + ('Max', 'Min', 'Mean', 'Median', 'Std') + if INCLUDE_SUBJECT_FEATURE_MEAN + else ('Max', 'Min', 'Median', 'Std') +) +FEATURE_CORRELATION_THRESHOLD = 0.8 MAX_TRAIN_WORKERS = max(1, min(4, os.cpu_count() or 1)) -USE_SITE_GROUPED_CV = True +USE_SITE_GROUPED_CV = False OPTIMIZE_HYPERPARAMETER_SEARCH = False RANDOM_CV_N_SPLITS = 5 CV_RANDOM_STATE = 42 @@ -26,7 +32,7 @@ SEGMENT_DURATION_SECONDS = 5 * 60 SEGMENT_STRIDE_SECONDS = 15 * 60 TOTAL_PHYSIOLOGICAL_FEATURE_LENGTH = ( - RESP_FEATURE_LENGTH - + EEG_FEATURE_LENGTH - + ECG_FEATURE_LENGTH + RESP_FEATURE_LENGTH * len(SEGMENT_AGGREGATION_NAMES) + + EEG_FEATURE_LENGTH * len(SEGMENT_AGGREGATION_NAMES) + + ECG_FEATURE_LENGTH * len(SEGMENT_AGGREGATION_NAMES) ) diff --git a/src/pipeline/features.py b/src/pipeline/features.py index 08b736f..6afe27c 100644 --- a/src/pipeline/features.py +++ b/src/pipeline/features.py @@ -14,14 +14,15 @@ from .config import ( DEFAULT_CSV_PATH, FEATURE_CACHE_FOLDER_NAME, + SEGMENT_AGGREGATION_NAMES, SCRIPT_DIR, SEGMENT_DURATION_SECONDS, SEGMENT_STRIDE_SECONDS, + TOTAL_PHYSIOLOGICAL_FEATURE_LENGTH, ) REQUIRED_SIGNAL_ALIASES_CACHE = {} -SEGMENT_AGGREGATION_NAMES = ('Max', 'Min', 'Mean', 'Median', 'Std') DEMOGRAPHIC_FEATURE_NAMES = ( 'Age', 'Sex', @@ -48,11 +49,6 @@ def _build_aggregated_feature_names(segment_feature_names): *FEATURE_NAME_GROUPS['eeg'], *FEATURE_NAME_GROUPS['ecg'], ) -TOTAL_PHYSIOLOGICAL_FEATURE_LENGTH = ( - len(FEATURE_NAME_GROUPS['resp']) - + len(FEATURE_NAME_GROUPS['eeg']) - + len(FEATURE_NAME_GROUPS['ecg']) -) def _get_feature_cache_root(data_folder): From 1b6d3659150e6266c5171e9c627cfe427880c7a8 Mon Sep 17 00:00:00 2001 From: dcajal Date: Wed, 8 Apr 2026 23:54:28 +0200 Subject: [PATCH 85/93] Fix column number missmatch --- src/pipeline/features.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/pipeline/features.py b/src/pipeline/features.py index 6afe27c..0d46084 100644 --- a/src/pipeline/features.py +++ b/src/pipeline/features.py @@ -50,6 +50,14 @@ def _build_aggregated_feature_names(segment_feature_names): *FEATURE_NAME_GROUPS['ecg'], ) +SEGMENT_AGGREGATION_FUNCTIONS = { + 'Max': lambda values: float(np.max(values)), + 'Min': lambda values: float(np.min(values)), + 'Mean': lambda values: float(np.mean(values)), + 'Median': lambda values: float(np.median(values)), + 'Std': lambda values: float(np.std(values)), +} + def _get_feature_cache_root(data_folder): return os.path.join(SCRIPT_DIR, FEATURE_CACHE_FOLDER_NAME) @@ -297,13 +305,10 @@ def _aggregate_segment_feature_vectors(feature_vectors, segment_feature_names): aggregated_values.extend([np.nan] * len(SEGMENT_AGGREGATION_NAMES)) continue - aggregated_values.extend([ - float(np.max(finite_values)), - float(np.min(finite_values)), - float(np.mean(finite_values)), - float(np.median(finite_values)), - float(np.std(finite_values)), - ]) + aggregated_values.extend( + SEGMENT_AGGREGATION_FUNCTIONS[aggregation_name](finite_values) + for aggregation_name in SEGMENT_AGGREGATION_NAMES + ) return np.asarray(aggregated_values, dtype=np.float32) From b923e26ab9659a86603bf1959f309feabd22ef98 Mon Sep 17 00:00:00 2001 From: dcajal Date: Thu, 9 Apr 2026 13:07:13 +0200 Subject: [PATCH 86/93] Enable hyperparameter optimization and change scoring metric to ROC AUC in cross-validation --- src/pipeline/config.py | 2 +- src/pipeline/cross_validation.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pipeline/config.py b/src/pipeline/config.py index becbf0d..3c9d95d 100644 --- a/src/pipeline/config.py +++ b/src/pipeline/config.py @@ -17,7 +17,7 @@ FEATURE_CORRELATION_THRESHOLD = 0.8 MAX_TRAIN_WORKERS = max(1, min(4, os.cpu_count() or 1)) USE_SITE_GROUPED_CV = False -OPTIMIZE_HYPERPARAMETER_SEARCH = False +OPTIMIZE_HYPERPARAMETER_SEARCH = True RANDOM_CV_N_SPLITS = 5 CV_RANDOM_STATE = 42 CV_SEARCH_ITERATIONS = 20 diff --git a/src/pipeline/cross_validation.py b/src/pipeline/cross_validation.py index 5dd6331..0f6d5b4 100644 --- a/src/pipeline/cross_validation.py +++ b/src/pipeline/cross_validation.py @@ -388,7 +388,7 @@ def _search_hyperparams(self, X_train, y_train, site_groups=None): estimator=self.build_search_model(y_train), param_distributions=self.param_dist, n_iter=self.config.search_iterations, - scoring='f1', + scoring='roc_auc', cv=inner_cv, random_state=self.config.random_state, n_jobs=-1, From 142a2d3ff15b8eb4aeb61c170a2d3a7b322a51f0 Mon Sep 17 00:00:00 2001 From: sromagnoli-10 Date: Thu, 9 Apr 2026 14:27:17 +0200 Subject: [PATCH 87/93] Update cross_validation.py --- src/pipeline/cross_validation.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/pipeline/cross_validation.py b/src/pipeline/cross_validation.py index 0f6d5b4..0c249ed 100644 --- a/src/pipeline/cross_validation.py +++ b/src/pipeline/cross_validation.py @@ -436,6 +436,8 @@ def _best_threshold(self, probabilities, labels): if score > best_score: best_score = score best_value = float(threshold) + if best_value < 0.5: + best_value = 0.5 return best_value @@ -513,4 +515,4 @@ def _print_metric_summary(self, prefix, summary): mean_value = self._format_metric_value(metric_summary.get('mean')) std_value = self._format_metric_value(metric_summary.get('std')) parts.append(f"{metric_name.upper()}={mean_value} +/- {std_value}") - print(f"{prefix} {', '.join(parts)}") \ No newline at end of file + print(f"{prefix} {', '.join(parts)}") From f1aff83e33dc6217da009f946051fee635d46d3b Mon Sep 17 00:00:00 2001 From: jnmadrid Date: Thu, 28 May 2026 11:57:48 +0200 Subject: [PATCH 88/93] changes --- scripts/create_smoke.sh | 2 +- src/pipeline/training_JM.py | 445 ++++++++++++++++++++++++++++++++++++ 2 files changed, 446 insertions(+), 1 deletion(-) create mode 100644 src/pipeline/training_JM.py diff --git a/scripts/create_smoke.sh b/scripts/create_smoke.sh index 88765ec..bc01bf8 100644 --- a/scripts/create_smoke.sh +++ b/scripts/create_smoke.sh @@ -13,7 +13,7 @@ set -euo pipefail FULL_DATA_PATH="${FULL_DATA_PATH:-data/training_set}" # Override with env var if needed SMOKE_PATH="data/training_smoke" -N_RECORDS="${N_RECORDS:-5}" +N_RECORDS="${N_RECORDS:-20}" echo "Creating smoke dataset..." echo "Source: ${FULL_DATA_PATH}" diff --git a/src/pipeline/training_JM.py b/src/pipeline/training_JM.py new file mode 100644 index 0000000..b98f0fe --- /dev/null +++ b/src/pipeline/training_JM.py @@ -0,0 +1,445 @@ +import os +from concurrent.futures import ThreadPoolExecutor + +import numpy as np +import pandas as pd +from sklearn.metrics import f1_score +from sklearn.model_selection import train_test_split +from sklearn.pipeline import Pipeline +from tqdm import tqdm +from xgboost import XGBClassifier, data + +from sklearn.compose import ColumnTransformer +from sklearn.impute import KNNImputer, SimpleImputer +from sklearn.preprocessing import StandardScaler, OneHotEncoder + +from helper_code import DEMOGRAPHICS_FILE, HEADERS, find_patients, load_label + +from .config import MAX_TRAIN_WORKERS +from .features import get_feature_group_indices, get_feature_names, get_or_create_record_feature_vector + + +DEFAULT_ENSEMBLE_THRESHOLD = 0.5 +ENSEMBLE_MODALITIES = ('resp', 'eeg', 'ecg') +DEFAULT_KNN_NEIGHBORS = 5 + + +def build_training_metadata_cache(patient_data_file): + metadata = pd.read_csv(patient_data_file) + demographics_cache = {} + diagnosis_cache = {} + + for row in metadata.to_dict('records'): + patient_id = row[HEADERS['bids_folder']] + session_id = row[HEADERS['session_id']] + demographics_cache[(patient_id, session_id)] = row + diagnosis_cache[patient_id] = load_label(row) + + return demographics_cache, diagnosis_cache + + +def process_training_record(record, data_folder, demographics_cache, diagnosis_cache, csv_path): + patient_id = record[HEADERS['bids_folder']] + session_id = record[HEADERS['session_id']] + site_id = record[HEADERS['site_id']] + + try: + patient_data = demographics_cache.get((patient_id, session_id), {}) + feature_vector = get_or_create_record_feature_vector( + record, + data_folder, + patient_data, + csv_path=csv_path, + require_physiological_data=True, + ) + + label = diagnosis_cache.get(patient_id) + metadata = { + 'patient_id': patient_id, + 'site_id': site_id, + 'session_id': session_id, + } + + if label == 0 or label == 1: + return metadata, feature_vector, label, None + + return metadata, None, None, f"Invalid label for {patient_id}. Skipping..." + + except FileNotFoundError as exc: + return { + 'patient_id': patient_id, + 'site_id': site_id, + 'session_id': session_id, + }, None, None, f"{exc} Skipping..." + except Exception as exc: + return { + 'patient_id': patient_id, + 'site_id': site_id, + 'session_id': session_id, + }, None, None, f"Error processing {patient_id}: {exc}" + + +def _export_feature_matrix_csv(output_path, metadata_rows, labels, feature_matrix, feature_names): + dataframe = pd.DataFrame(metadata_rows) + dataframe['label'] = labels + feature_frame = pd.DataFrame(feature_matrix, columns=feature_names) + dataframe = pd.concat([dataframe.reset_index(drop=True), feature_frame.reset_index(drop=True)], axis=1) + os.makedirs(os.path.dirname(output_path), exist_ok=True) + dataframe.to_csv(output_path, index=False) + + + + + + +def best_threshold(probabilities, labels): + thresholds = np.linspace(0, 1, 101) + best_score = -1.0 + best_value = DEFAULT_ENSEMBLE_THRESHOLD + + for threshold in thresholds: + predictions = (probabilities >= threshold).astype(np.int32) + score = f1_score(labels, predictions, zero_division=0) + if score > best_score: + best_score = score + best_value = float(threshold) + + return best_value + + +def _build_xgb_model(labels): + neg = int(np.sum(labels == 0)) + pos = int(np.sum(labels == 1)) + scale_pos_weight = (neg / pos) if pos > 0 else 1.0 + + return XGBClassifier( + scale_pos_weight=scale_pos_weight, + n_estimators=300, + max_depth=4, + learning_rate=0.05, + subsample=0.8, + random_state=42, + eval_metric='auc', + tree_method='hist', + enable_categorical=False, + early_stopping_rounds=20) + +def preprocess_multimodal_data( df_model): + + #Llamamos al dataset de entrada que contiene las variables calculadas y confirma que no hay IDs duplicados + df_model2 = df_model.drop_duplicates(subset='ID') + # Extrae las columnas que se usarán + df_model2=df_model2[['ID','label', 'Age','Sex','Nasal_Peakedness_Max', 'Nasal_Peakedness_Min', 'Nasal_Peakedness_Median','Nasal_Peakedness_Std', + 'Chest_Peakedness_Max', 'Chest_Peakedness_Min','Abdomen_Peakedness_Max','Abdomen_Peakedness_Min','Abdomen_Peakedness_Std', + 'Flow_Peakedness_Max','Flow_Peakedness_Min','Flow_Peakedness_Median','Flow_Peakedness_Std','SpO2_Max','SpO2_Min','SpO2_Std', + 'CET90','ODI_Mean','ODI_deepness','C3-M2_Hjorth_Complexity','C4-M1_Hjorth_Complexity','F3-M2_Hjorth_Complexity', + 'F4-M1_Hjorth_Complexity','C3-M2_Hjorth_Mobility','F3-M2_Hjorth_Mobility','F4-M1_Hjorth_Mobility','C3-M2_Ratio_Slow_Fast', + 'C4-M1_Ratio_Slow_Fast','F3-M2_Ratio_Slow_Fast','F4-M1_Ratio_Slow_Fast','C3-M2_Rel_Beta','F3-M2_Rel_Beta','F4-M1_Rel_Beta', + 'C4-M1_Rel_Sigma','F3-M2_Rel_Sigma','C3-M2_Relative_Delta_Power','C4-M1_Relative_Delta_Power','F3-M2_Relative_Delta_Power', + 'F4-M1_Relative_Delta_Power','C3-M2_Theta_Alpha_Ratio','C4-M1_Theta_Alpha_Ratio','F3-M2_Theta_Alpha_Ratio','F4-M1_Theta_Alpha_Ratio', + 'C3-M2_Theta_Beta_Ratio','C4-M1_Theta_Beta_Ratio','F3-M2_Theta_Beta_Ratio','F4-M1_Theta_Beta_Ratio','C3-M2_kurtosis_Alpha', + 'C3-M2_kurtosis_Beta','C4-M1_kurtosis_Beta','F3-M2_kurtosis_Beta','F4-M1_kurtosis_Beta','C3-M2_kurtosis_Delta','C4-M1_kurtosis_Delta', + 'F3-M2_kurtosis_Delta','F4-M1_kurtosis_Delta','C3-M2_kurtosis_Sigma','C4-M1_kurtosis_Sigma','F3-M2_kurtosis_Sigma','F4-M1_kurtosis_Sigma', + 'C3-M2_kurtosis_Theta','C4-M1_kurtosis_Theta','F3-M2_kurtosis_Theta','F4-M1_kurtosis_Theta','C3-M2_variability_Delta', + 'C4-M1_variability_Delta','F3-M2_variability_Delta','F4-M1_variability_Delta','PIP_med','PIP_std','PNNSS_med', + 'PNNSS_std','AVNN_med','AVNN_std','SDNN_med','SDNN_std','RMSSD_std','HF_med','HF_std','ECTOPIC_med','ECTOPIC_std']] + + + # Ajusta el tipo de dato, todas deben ser numericas excepto Sex, Label y ID. Sex y label tendría que ser logical? o categorical? + df_model2['Sex'] = df_model2['Sex'].astype('category') + df_model2['label'] = df_model2['label'].astype(int) + + #Unificar los valores faltantes, que sean consistentes + df_model2 = df_model2.replace([-999, "NA", "null", "None", "nan", ""], np.nan) + # Eliminar filas con demasiados NaNs (mínimo 50% de datos válidos) + umbral = int(len(df_model2.columns) * 0.5) + df_model2 = df_model2.dropna(thresh=umbral) + + #Procesa los datos, imputar nans y estandarizar por la media y desviacion estandar (zscore) + y = df_model2['label'] + X = df_model2.drop(columns=[ 'label','ID']) # Excluimos ID, label del entrenamiento + + return {"X": X, + "y": y} + +def prepare_multimodal_data(dddf, testsize=0.1): + + proc_data=preprocess_multimodal_data(dddf) + X = proc_data["X"] + y = proc_data["y"] + X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=testsize, random_state=1999, stratify=y) + + cols_categoricas = ['Sex'] + cols_numericas = [c for c in X_train.columns if c not in cols_categoricas] + + preprocessor = ColumnTransformer([ + ("num", Pipeline([ + ("imputer", KNNImputer(n_neighbors=5)), + ("scaler", StandardScaler()) + ]), cols_numericas), + + ("cat", Pipeline([ + ("imputer", SimpleImputer(strategy="most_frequent")), + ("encoder", OneHotEncoder(handle_unknown="ignore", sparse_output=False)) + ]), cols_categoricas) + ]) + preprocessor.set_output(transform="pandas") + + X_train = preprocessor.fit_transform(X_train) + X_test = preprocessor.transform(X_test) + + feature_names = preprocessor.get_feature_names_out() + resp_mask = np.array([ any(k in col for k in ["Age","Sex","Nasal_Peakedness", "Chest_Peakedness", "Abdomen_Peakedness", "Flow_Peakedness", "SpO2", "CET90", "ODI" ]) + for col in feature_names]) + eeg_mask = np.array([ any(k in col for k in ["Age","Sex","C3-M2", "C4-M1", "F3-M2", "F4-M1"]) + for col in feature_names]) + ecg_mask = np.array([ any(k in col for k in ["Age","Sex","PIP_", "PNNSS_", "AVNN_", "SDNN_", "RMSSD_", "HF_", "ECTOPIC_"]) + for col in feature_names]) + + # Subsets finales + return { + "X_train": X_train, + "X_test": X_test, + "y_train": y_train, + "y_test": y_test, + "resp": (X_train.loc[:, resp_mask], X_test.loc[:, resp_mask]), + "eeg": (X_train.loc[:, eeg_mask], X_test.loc[:, eeg_mask]), + "ecg": (X_train.loc[:, ecg_mask], X_test.loc[:, ecg_mask]), + "preprocessor": preprocessor + } + +def _fit_ensembleJM(data): + + models = {} + y_train = data["y_train"] + y_test = data["y_test"] + + probas_test = [] + + # =============================== + # 1. Entrenamiento por modalidad + # =============================== + for modality in ['resp', 'eeg', 'ecg']: + + if modality not in data or data[modality] is None: + continue + + X_train, X_test = data[modality] + + if X_train.shape[1] == 0: + continue + + model = _build_xgb_model(y_train) + + model.fit( + X_train, + y_train, + eval_set=[(X_test, y_test)], + verbose=False + ) + + models[modality] = model + + proba = model.predict_proba(X_test)[:, 1] + probas_test.append(proba) + + if not probas_test: + raise ValueError("No hay modalidades válidas para entrenar.") + + # =============================== + # 2. Ensemble + # =============================== + proba_final = np.mean(probas_test, axis=0) + + # =============================== + # 3. Threshold (mejorado) + # =============================== + best_thr = best_threshold(proba_final, y_test) + + return { + "models": models, + "threshold": best_thr + } + +from sklearn.model_selection import StratifiedKFold + +def _fit_ensemble_with_cv(data, n_splits=5): + + X_train_full = data["X_train"] + y_train_full = data["y_train"] + + skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42) + + oof_probas = [] + oof_labels = [] + + for fold, (train_idx, val_idx) in enumerate(skf.split(X_train_full, y_train_full)): + + print(f"Fold {fold+1}/{n_splits}") + + probas_fold = [] + + for modality in ['resp', 'eeg', 'ecg']: + + X_train_mod, _ = data[modality] + + X_tr = X_train_mod.iloc[train_idx] + X_val = X_train_mod.iloc[val_idx] + + y_tr = y_train_full.iloc[train_idx] + y_val = y_train_full.iloc[val_idx] + + if X_tr.shape[1] == 0: + continue + + model = _build_xgb_model(y_tr) + + model.fit( + X_tr, + y_tr, + eval_set=[(X_val, y_val)], + verbose=False + ) + + proba = model.predict_proba(X_val)[:, 1] + probas_fold.append(proba) + + if not probas_fold: + continue + + # Ensemble por fold + proba_ensemble = np.mean(probas_fold, axis=0) + + oof_probas.extend(proba_ensemble) + oof_labels.extend(y_val) + + oof_probas = np.array(oof_probas) + oof_labels = np.array(oof_labels) + + # Threshold robusto (sin leakage) + best_thr = best_threshold(oof_probas, oof_labels) + + # =============================== + # Entrenar modelos finales (full data) + # =============================== + final_models = {} + + for modality in ['resp', 'eeg', 'ecg']: + + X_train_mod, X_test_mod = data[modality] + + if X_train_mod.shape[1] == 0: + continue + + model = _build_xgb_model(y_train_full) + + model.fit( + X_train_mod, + y_train_full, + eval_set=[(X_test_mod, data["y_test"])], + verbose=False + ) + + final_models[modality] = model + + return { + "models": final_models, + "threshold": best_thr + } + +def train_multimodal_ensemble(data_folder, verbose, csv_path, export_folder=None): + + # =============================== + # 1. CARGA Y EXTRACCIÓN + # =============================== + patient_data_file = os.path.join(data_folder, DEMOGRAPHICS_FILE) + patient_metadata_list = find_patients(patient_data_file) + + demographics_cache, diagnosis_cache = build_training_metadata_cache(patient_data_file) + num_records = len(patient_metadata_list) + + if num_records == 0: + raise FileNotFoundError('No data were provided.') + + features = [] + labels = [] + metadata_rows = [] + + with ThreadPoolExecutor(max_workers=MAX_TRAIN_WORKERS) as executor: + results = executor.map( + lambda record: process_training_record( + record, + data_folder, + demographics_cache, + diagnosis_cache, + csv_path, + ), + patient_metadata_list, + ) + + pbar = tqdm(results, total=num_records, desc='Extracting Features', + unit='record', disable=not verbose) + + for metadata, feature_vector, label, message in pbar: + + if verbose: + pbar.set_postfix({'patient': metadata['patient_id']}) + + if message is not None: + tqdm.write(f" ! {message}") + continue + + features.append(feature_vector) + labels.append(label) + metadata_rows.append(metadata) + + pbar.close() + + # =============================== + # 2. VALIDACIÓN + # =============================== + features = np.asarray(features, dtype=np.float32) + labels = np.asarray(labels, dtype=np.int32) + + if features.size == 0 or features.ndim != 2 or features.shape[0] == 0: + raise ValueError('No valid training samples were extracted.') + + feature_names = list(get_feature_names()) + + # =============================== + # 3. EXPORT RAW (opcional) + # =============================== + export_root = export_folder or os.path.join(os.getcwd(), 'feature_exports') + raw_feature_export_path = os.path.join(export_root, 'training_features_raw.csv') + + _export_feature_matrix_csv( + raw_feature_export_path, + metadata_rows, + labels, + features, + feature_names + ) + + # =============================== + # 2. CONVERTIR A DATAFRAME + df = pd.DataFrame(features, columns=feature_names) + df["label"] = labels + + data = prepare_multimodal_data(df, testsize=0.1) + preprocessor=data["preprocessor"] + + models, threshold = _fit_ensembleJM(data) + #result = _fit_ensemble_with_cv(data, n_splits=5) + #models = result["models"] + #threshold = result["threshold"] + + return { + 'type': 'multimodal_xgb_ensemble', + 'threshold': threshold, + 'feature_names': feature_names, + 'models': models, + 'preprocessor': preprocessor, + 'feature_exports': { + 'raw': raw_feature_export_path, + }, + } \ No newline at end of file From dabab2c87f6d03c3d93383cbc6e22b857354ed8a Mon Sep 17 00:00:00 2001 From: rolopu1 Date: Tue, 2 Jun 2026 11:27:22 +0200 Subject: [PATCH 89/93] Add new PowerShell script for copying new files and enhance preprocessing with PCA - Introduced `copy_new_files.ps1` script to copy new files from the source directory to the test set, ensuring only files not present in the training set are copied. - Updated `preprocessing.py` to include PCA for dimensionality reduction, allowing users to specify a variance threshold. - Modified `training.py` to support PCA in the preprocessing pipeline and added debug information for PCA components. - Adjusted `team_code.py` to include command-line argument examples for data and model paths. - Created `train_output.txt` to store training output logs. --- .gitignore | 6 +- RemoveTestCache.py | 20 ++ outputs_test/demographics.csv | 416 ++++++++++++++++++++++++++++++++++ scripts/copy_new_files.ps1 | 69 ++++++ src/pipeline/preprocessing.py | 95 +++++++- src/pipeline/training.py | 47 +++- team_code.py | 3 +- train_output.txt | Bin 0 -> 138354 bytes 8 files changed, 640 insertions(+), 16 deletions(-) create mode 100644 RemoveTestCache.py create mode 100644 outputs_test/demographics.csv create mode 100644 scripts/copy_new_files.ps1 create mode 100644 train_output.txt diff --git a/.gitignore b/.gitignore index 7a3bfcc..a656935 100644 --- a/.gitignore +++ b/.gitignore @@ -236,7 +236,5 @@ cython_debug/ marimo/_static/ marimo/_lsp/ __marimo__/ -results_summaryEEG_I0002.csv -results_summaryEEG_I0004.csv -results_summaryEEG_I0006.csv -results_summaryEEG_I0007.csv + +movedata.py \ No newline at end of file diff --git a/RemoveTestCache.py b/RemoveTestCache.py new file mode 100644 index 0000000..2046a39 --- /dev/null +++ b/RemoveTestCache.py @@ -0,0 +1,20 @@ +import os +import pandas as pd + +demotest = pd.read_csv("data/test_set/demographics.csv") + +cache_path = ".feature_cache" + +for i in demotest.index: + h = demotest.loc[i, "SiteID"] + f = demotest.loc[i, "BidsFolder"] + session = demotest.loc[i, "SessionID"] + if os.path.exists(os.path.join(cache_path, h, f+"_ses-" + str(session)+".csv")): + print(f"Removing {os.path.join(cache_path, h, f+"_ses-" + str(session)+".csv")}") + os.remove(os.path.join(cache_path, h, f+"_ses-" + str(session)+".csv")) + if os.path.exists(os.path.join(cache_path, h, f+"_ses-" + str(session)+".sav")): + print(f"Removing {os.path.join(cache_path, h, f+"_ses-" + str(session)+".sav")}") + os.remove(os.path.join(cache_path, h, f+"_ses-" + str(session)+".sav")) + +os.remove(os.path.join(cache_path, "exports", "test_features_raw.csv")) +os.remove(os.path.join(cache_path, "exports", "test_features_preprocessed.csv")) \ No newline at end of file diff --git a/outputs_test/demographics.csv b/outputs_test/demographics.csv new file mode 100644 index 0000000..3dba359 --- /dev/null +++ b/outputs_test/demographics.csv @@ -0,0 +1,416 @@ +SiteID,BDSPPatientID,CreationTime,BidsFolder,SessionID,Age,Sex,Race,Ethnicity,BMI,Time_to_Event,Cognitive_Impairment,Last_Known_Visit_Date,Time_to_Last_Visit,Cognitive_Impairment_Probability +S0001,111239981,11/02/2016 20:55,sub-S0001111239981,1.0,77.0,Female,White,Hispanic,37.09,1961.0,True,30/05/2022,2299.0,0.14538460969924927 +S0001,111249761,18/06/2011 2:33,sub-S0001111249761,1.0,75.0,Female,White,Not Hispanic,30.37,,True,02/04/2024,4671.0,0.6455416083335876 +S0001,111261966,30/07/2010 2:55,sub-S0001111261966,1.0,71.0,Female,White,Not Hispanic,30.13,,True,21/09/2023,4800.0,0.5782973766326904 +S0001,111390308,20/12/2008 21:51,sub-S0001111390308,1.0,70.0,Female,White,Not Hispanic,26.9,,True,18/11/2023,5445.0,0.40108874440193176 +S0001,111439208,15/10/2011 0:00,sub-S0001111439208,1.0,63.0,Male,White,Not Hispanic,26.12,,True,05/09/2022,3978.0,0.4706968069076538 +S0001,111457455,04/01/2017 21:47,sub-S0001111457455,1.0,73.0,Female,White,Not Hispanic,28.32,,True,21/04/2024,2663.0,0.9163349866867065 +S0001,111520696,25/02/2011 3:36,sub-S0001111520696,1.0,78.0,Male,White,Not Hispanic,27.68,,True,11/09/2023,4580.0,0.5654754638671875 +S0001,111561964,04/12/2015 22:38,sub-S0001111561964,1.0,68.0,Male,White,Not Hispanic,31.51,,True,30/03/2023,2672.0,0.39139872789382935 +S0001,111608665,07/06/2010 23:28,sub-S0001111608665,1.0,69.0,Male,White,Not Hispanic,44.2,,True,18/05/2022,4362.0,0.6186583042144775 +S0001,111613441,18/11/2016 22:32,sub-S0001111613441,1.0,64.0,Male,White,Not Hispanic,33.52,1632.0,True,24/12/2022,2226.0,0.8189074397087097 +S0001,111625940,01/07/2013 2:02,sub-S0001111625940,1.0,68.0,Male,White,Not Hispanic,27.91,,True,19/06/2023,3639.0,0.39139872789382935 +S0001,111640757,28/04/2010 2:11,sub-S0001111640757,1.0,73.0,Male,White,Not Hispanic,25.72,,True,03/03/2022,4326.0,0.8519553542137146 +S0001,111683364,01/06/2014 22:07,sub-S0001111683364,1.0,66.0,Male,White,Not Hispanic,27.41,,True,11/11/2022,3084.0,0.45043623447418213 +S0001,111695024,07/03/2013 3:03,sub-S0001111695024,1.0,78.0,Female,White,Not Hispanic,34.31,,True,01/09/2024,4195.0,0.6120396256446838 +S0001,111722285,08/03/2009 22:22,sub-S0001111722285,1.0,72.0,Female,Asian,Not Hispanic,22.96,,True,08/02/2018,3258.0,0.5070144534111023 +S0001,111781598,03/08/2019 18:20,sub-S0001111781598,1.0,79.0,Male,White,Not Hispanic,26.15,1190.0,True,15/03/2024,1685.0,0.4199504256248474 +S0001,111844094,22/02/2010 23:31,sub-S0001111844094,1.0,61.0,Male,White,Hispanic,35.28,1560.0,True,20/05/2023,4834.0,0.6737512946128845 +S0001,111860137,05/03/2011 23:11,sub-S0001111860137,1.0,67.0,Female,White,Not Hispanic,34.86,,True,01/01/2023,4319.0,0.6473630666732788 +S0001,111916963,19/07/2010 22:17,sub-S0001111916963,1.0,81.0,Female,White,Not Hispanic,23.53,,True,16/02/2024,4959.0,0.40572720766067505 +S0001,111932587,12/06/2011 22:01,sub-S0001111932587,1.0,81.0,Male,White,Not Hispanic,37.26,,True,22/09/2020,3389.0,0.8153370022773743 +S0001,112040083,14/08/2011 2:36,sub-S0001112040083,1.0,70.0,Male,White,Not Hispanic,25.55,1394.0,True,03/01/2023,4159.0,0.483925461769104 +S0001,112059742,19/12/2011 2:10,sub-S0001112059742,1.0,68.0,Female,White,Not Hispanic,19.13,,True,27/12/2022,4025.0,0.346417635679245 +S0001,112078298,05/06/2019 19:28,sub-S0001112078298,1.0,78.0,Male,White,Not Hispanic,29.24,1126.0,True,12/01/2023,1316.0,0.5654754638671875 +S0001,112087330,09/01/2017 23:07,sub-S0001112087330,1.0,75.0,Female,White,Not Hispanic,36.73,,True,12/06/2024,2710.0,0.6455416083335876 +S0001,112141915,11/11/2012 22:05,sub-S0001112141915,1.0,74.0,Male,White,Not Hispanic,26.39,,True,23/12/2022,3693.0,0.39002496004104614 +S0001,112156568,09/06/2011 3:31,sub-S0001112156568,1.0,71.0,Male,White,Not Hispanic,28.62,,True,15/04/2024,4693.0,0.40006473660469055 +S0001,112167991,20/10/2010 3:38,sub-S0001112167991,1.0,61.0,Female,White,Unavailable,41.93,,True,17/02/2024,4867.0,0.824192225933075 +S0001,112194534,06/05/2017 21:17,sub-S0001112194534,1.0,63.0,Male,White,Not Hispanic,32.76,1169.0,True,03/09/2022,1945.0,0.4706968069076538 +S0001,112229480,03/01/2011 3:14,sub-S0001112229480,1.0,68.0,Male,Black,Not Hispanic,42.22,,True,18/12/2021,4001.0,0.39139872789382935 +S0001,112287792,22/01/2013 21:32,sub-S0001112287792,1.0,56.0,Male,White,Not Hispanic,36.01,,True,05/01/2023,3634.0,0.501916766166687 +S0001,112293206,05/04/2009 22:59,sub-S0001112293206,1.0,66.0,Male,White,Not Hispanic,26.99,,True,09/01/2022,4661.0,0.45043623447418213 +S0001,112313733,26/04/2011 21:45,sub-S0001112313733,1.0,76.0,Male,White,Not Hispanic,28.42,,True,04/10/2022,4178.0,0.2753254175186157 +S0001,112331277,14/12/2012 22:49,sub-S0001112331277,1.0,75.0,Male,White,Not Hispanic,24.57,,True,16/09/2024,4293.0,0.27257463335990906 +S0001,112361957,13/12/2019 20:01,sub-S0001112361957,2.0,53.0,Male,White,Not Hispanic,28.75,1258.0,True,25/05/2023,1258.0,0.6771898865699768 +S0001,112379247,09/09/2011 1:15,sub-S0001112379247,1.0,85.0,Male,White,Not Hispanic,30.75,2240.0,True,17/03/2021,3476.0,0.6813521385192871 +S0001,112382548,15/04/2017 22:31,sub-S0001112382548,1.0,72.0,Male,Black,Not Hispanic,33.76,,True,26/04/2024,2567.0,0.5197765827178955 +S0001,112453372,27/04/2017 22:56,sub-S0001112453372,1.0,75.0,Female,White,Not Hispanic,30.72,1475.0,True,26/08/2023,2311.0,0.6455416083335876 +S0001,112507998,28/08/2012 1:58,sub-S0001112507998,1.0,60.0,Female,White,Not Hispanic,27.2,,True,12/09/2023,4031.0,0.5012294054031372 +S0001,112516993,12/04/2010 22:51,sub-S0001112516993,1.0,64.0,Male,White,Not Hispanic,39.63,,True,16/09/2024,5270.0,0.8189074397087097 +S0001,112601406,05/02/2008 22:55,sub-S0001112601406,1.0,61.0,Female,White,Not Hispanic,30.94,,True,28/03/2023,5529.0,0.824192225933075 +S0001,112649481,09/02/2012 2:23,sub-S0001112649481,1.0,71.0,Male,White,Not Hispanic,29.1,,True,05/11/2023,4286.0,0.40006473660469055 +S0001,112722303,14/05/2016 22:30,sub-S0001112722303,1.0,73.0,Female,White,Not Hispanic,24.55,,True,03/01/2024,2789.0,0.9163349866867065 +S0001,112724496,26/07/2013 0:00,sub-S0001112724496,1.0,68.0,Male,White,Not Hispanic,25.28,,True,05/07/2024,3997.0,0.39139872789382935 +S0001,112747541,23/05/2015 21:47,sub-S0001112747541,1.0,58.0,Female,White,Not Hispanic,29.78,,True,23/12/2023,3135.0,0.46824002265930176 +S0001,112827546,16/09/2017 22:23,sub-S0001112827546,1.0,89.0,Male,White,Not Hispanic,24.14,1236.0,True,08/06/2023,2090.0,0.634381115436554 +S0001,112860352,01/10/2014 21:54,sub-S0001112860352,1.0,63.0,Female,White,Not Hispanic,45.44,,True,03/08/2023,3227.0,0.7037391066551208 +S0001,112893779,08/12/2016 22:35,sub-S0001112893779,1.0,74.0,Male,White,Not Hispanic,29.08,,True,15/12/2023,2562.0,0.39002496004104614 +S0001,112894687,11/11/2009 2:31,sub-S0001112894687,1.0,61.0,Male,Black,Not Hispanic,32.23,,True,18/04/2021,4175.0,0.6737512946128845 +S0001,112895854,12/08/2015 22:19,sub-S0001112895854,1.0,77.0,Female,White,Not Hispanic,29.2,,True,09/12/2022,2675.0,0.14538460969924927 +S0001,112930038,11/03/2017 21:59,sub-S0001112930038,1.0,60.0,Female,White,Not Hispanic,28.67,,True,03/09/2024,2732.0,0.5012294054031372 +S0001,112936324,15/11/2009 21:51,sub-S0001112936324,1.0,73.0,Male,White,Not Hispanic,29.84,,True,27/01/2018,2994.0,0.8519553542137146 +S0001,112960676,23/06/2008 22:16,sub-S0001112960676,1.0,78.0,Male,White,Not Hispanic,31.97,,True,29/08/2017,3353.0,0.5654754638671875 +S0001,112979679,24/04/2013 3:29,sub-S0001112979679,1.0,64.0,Male,White,Not Hispanic,29.51,,True,30/03/2023,3626.0,0.8189074397087097 +S0001,112993336,23/09/2015 23:09,sub-S0001112993336,1.0,87.0,Male,White,Not Hispanic,32.19,,True,11/06/2023,2817.0,0.506742537021637 +S0001,113038747,02/02/2013 0:01,sub-S0001113038747,1.0,76.0,Male,White,Not Hispanic,32.75,,True,26/07/2024,4191.0,0.2753254175186157 +S0001,113078805,15/10/2009 21:18,sub-S0001113078805,1.0,69.0,Female,Unavailable,Not Hispanic,18.54,,True,05/06/2023,4980.0,0.6564435362815857 +S0001,113112903,17/06/2011 2:19,sub-S0001113112903,1.0,80.0,Male,White,Not Hispanic,29.32,,True,11/06/2023,4376.0,0.34214648604393005 +S0001,113127215,18/05/2009 21:48,sub-S0001113127215,1.0,73.0,Female,White,Not Hispanic,24.34,,True,15/04/2019,3618.0,0.9163349866867065 +S0001,113127704,14/01/2013 1:53,sub-S0001113127704,1.0,65.0,Male,White,Not Hispanic,29.42,,True,04/11/2023,3945.0,0.4400545656681061 +S0001,113135566,22/05/2014 23:03,sub-S0001113135566,2.0,67.0,Female,White,Not Hispanic,40.53,1780.0,True,10/02/2023,3185.0,0.6473630666732788 +S0001,113137391,03/09/2009 23:16,sub-S0001113137391,1.0,70.0,Male,White,Not Hispanic,28.64,,True,15/10/2018,3328.0,0.483925461769104 +S0001,113169703,18/01/2012 3:08,sub-S0001113169703,1.0,70.0,Male,White,Not Hispanic,30.71,,True,02/05/2022,3756.0,0.483925461769104 +S0001,113333305,03/01/2013 22:09,sub-S0001113333305,1.0,75.0,Female,White,Not Hispanic,20.92,1170.0,True,04/09/2017,1704.0,0.6455416083335876 +S0001,113356108,22/07/2008 23:05,sub-S0001113356108,1.0,62.0,Female,White,Not Hispanic,37.15,,True,14/07/2023,5469.0,0.7453621029853821 +S0001,113407451,12/04/2011 1:58,sub-S0001113407451,1.0,51.0,Male,White,Not Hispanic,32.42,,True,05/09/2020,3433.0,0.4153312146663666 +S0001,113446885,01/09/2013 3:07,sub-S0001113446885,1.0,75.0,Male,White,Not Hispanic,23.07,,True,17/07/2023,3605.0,0.27257463335990906 +S0001,113452157,14/01/2010 1:40,sub-S0001113452157,1.0,71.0,Female,White,Unavailable,26.79,,True,21/06/2017,2714.0,0.5782973766326904 +S0001,113469297,29/05/2011 23:32,sub-S0001113469297,1.0,70.0,Female,White,Not Hispanic,32.91,,True,05/08/2023,4450.0,0.40108874440193176 +S0001,113578395,31/07/2010 23:00,sub-S0001113578395,1.0,75.0,Male,Others,Not Hispanic,35.67,,True,27/08/2018,2948.0,0.27257463335990906 +S0001,113620011,01/05/2013 3:08,sub-S0001113620011,1.0,67.0,Male,White,Unavailable,26.53,,True,11/03/2022,3235.0,0.9195594787597656 +S0001,113641132,12/12/2010 2:09,sub-S0001113641132,1.0,59.0,Female,White,Unavailable,42.29,,True,09/07/2023,4591.0,0.5383100509643555 +S0001,113841543,11/12/2010 1:35,sub-S0001113841543,1.0,65.0,Male,White,Not Hispanic,25.09,,True,05/05/2023,4527.0,0.4400545656681061 +S0001,113855605,06/05/2010 23:03,sub-S0001113855605,1.0,76.0,Male,White,Not Hispanic,26.95,,True,07/03/2023,4687.0,0.2753254175186157 +S0001,114053187,11/05/2009 22:13,sub-S0001114053187,1.0,61.0,Male,White,Not Hispanic,30.14,,True,27/04/2023,5098.0,0.6737512946128845 +S0001,114102759,13/10/2010 21:57,sub-S0001114102759,1.0,76.0,Male,White,Not Hispanic,30.63,,True,28/04/2024,4945.0,0.2753254175186157 +S0001,114110665,08/07/2010 2:00,sub-S0001114110665,1.0,65.0,Male,White,Not Hispanic,32.62,,True,08/12/2023,4900.0,0.4400545656681061 +S0001,114119353,27/07/2016 22:16,sub-S0001114119353,1.0,67.0,Female,White,Not Hispanic,24.47,,True,14/02/2024,2757.0,0.6473630666732788 +S0001,114124779,10/12/2016 21:48,sub-S0001114124779,1.0,55.0,Male,White,Not Hispanic,34.02,,True,10/02/2024,2617.0,0.19085277616977692 +S0001,114137420,15/03/2017 21:55,sub-S0001114137420,1.0,67.0,Male,White,Not Hispanic,27.8,,True,14/08/2024,2708.0,0.9195594787597656 +S0001,114214742,06/11/2011 2:31,sub-S0001114214742,1.0,61.0,Male,White,Not Hispanic,34.93,,True,03/07/2023,4256.0,0.6737512946128845 +S0001,114249812,27/02/2010 2:34,sub-S0001114249812,1.0,71.0,Female,White,Not Hispanic,32.02,,True,17/06/2023,4857.0,0.5782973766326904 +S0001,114287643,12/11/2011 2:54,sub-S0001114287643,1.0,77.0,Male,White,Not Hispanic,25.71,,True,02/02/2023,4099.0,0.8424844145774841 +S0001,114378497,17/03/2014 21:39,sub-S0001114378497,1.0,71.0,Male,White,Not Hispanic,28.59,,True,08/06/2023,3369.0,0.40006473660469055 +S0001,114400993,14/07/2013 1:41,sub-S0001114400993,1.0,76.0,Male,White,Not Hispanic,22.84,,True,25/06/2024,3998.0,0.2753254175186157 +S0001,114429880,14/04/2010 23:36,sub-S0001114429880,1.0,82.0,Male,White,Not Hispanic,29.93,,True,29/11/2023,4976.0,0.44877195358276367 +S0001,114431014,08/05/2011 3:10,sub-S0001114431014,1.0,60.0,Male,White,Not Hispanic,37.33,,True,19/11/2023,4577.0,0.24748243391513824 +S0001,114564587,27/10/2016 21:17,sub-S0001114564587,1.0,67.0,Female,Black,Not Hispanic,34.14,1947.0,True,01/12/2022,2225.0,0.6473630666732788 +S0001,114617339,13/03/2010 2:08,sub-S0001114617339,1.0,71.0,Female,White,Not Hispanic,26.9,,True,22/03/2022,4391.0,0.5782973766326904 +S0001,114730678,23/11/2010 21:06,sub-S0001114730678,1.0,66.0,Male,White,Not Hispanic,38.26,,True,13/04/2023,4523.0,0.45043623447418213 +S0001,114750697,08/05/2009 21:24,sub-S0001114750697,1.0,85.0,Male,White,Not Hispanic,29.47,,True,28/11/2022,4951.0,0.6813521385192871 +S0001,114833800,28/07/2008 23:31,sub-S0001114833800,1.0,57.0,Female,White,Not Hispanic,23.07,,True,27/04/2023,5385.0,0.6525198221206665 +S0001,114841089,28/08/2010 21:43,sub-S0001114841089,1.0,67.0,Male,White,Not Hispanic,24.17,,True,25/09/2024,5141.0,0.9195594787597656 +S0001,114856764,09/04/2016 22:56,sub-S0001114856764,1.0,62.0,Female,White,Not Hispanic,32.14,,True,20/07/2023,2657.0,0.7453621029853821 +S0001,114912372,29/06/2012 2:14,sub-S0001114912372,1.0,74.0,Female,White,Not Hispanic,33.44,,True,03/09/2022,3717.0,0.33417800068855286 +S0001,114920071,10/09/2014 23:20,sub-S0001114920071,1.0,71.0,Male,White,Not Hispanic,30.85,,True,13/03/2023,3105.0,0.40006473660469055 +S0001,114948418,03/05/2012 3:05,sub-S0001114948418,1.0,72.0,Male,White,Not Hispanic,35.75,,True,29/04/2024,4378.0,0.5197765827178955 +S0001,114970247,22/04/2013 1:50,sub-S0001114970247,1.0,59.0,Male,Unavailable,Hispanic,49.92,,True,06/09/2024,4154.0,0.2705105245113373 +S0001,115035582,26/11/2011 2:06,sub-S0001115035582,1.0,76.0,Male,White,Not Hispanic,24.34,2291.0,True,01/02/2022,3719.0,0.2753254175186157 +S0001,115036729,22/06/2010 21:49,sub-S0001115036729,1.0,69.0,Male,White,Not Hispanic,26.59,,True,18/06/2024,5109.0,0.6186583042144775 +S0001,115045037,18/10/2011 3:50,sub-S0001115045037,1.0,57.0,Male,Others,Hispanic,37.54,,True,11/12/2020,3341.0,0.5427361130714417 +S0001,115047250,25/10/2010 2:08,sub-S0001115047250,1.0,59.0,Male,White,Not Hispanic,25.95,,True,28/11/2023,4781.0,0.2705105245113373 +S0001,115075565,02/03/2017 21:20,sub-S0001115075565,1.0,80.0,Male,White,Not Hispanic,32.37,2128.0,True,23/04/2024,2608.0,0.34214648604393005 +S0001,115092772,06/11/2016 22:21,sub-S0001115092772,1.0,70.0,Male,White,Not Hispanic,34.84,2539.0,True,16/06/2024,2778.0,0.483925461769104 +S0001,115124810,16/11/2014 22:53,sub-S0001115124810,1.0,73.0,Female,White,Not Hispanic,32.09,,True,07/02/2023,3004.0,0.9163349866867065 +S0001,115306466,18/12/2009 2:31,sub-S0001115306466,1.0,71.0,Male,White,Not Hispanic,30.5,,True,05/02/2022,4431.0,0.40006473660469055 +S0001,115509440,10/03/2013 0:00,sub-S0001115509440,1.0,70.0,Male,White,Not Hispanic,23.27,,True,17/07/2021,3051.0,0.483925461769104 +S0001,115549333,12/02/2011 3:15,sub-S0001115549333,1.0,55.0,Female,Black,Not Hispanic,30.99,,True,09/10/2022,4256.0,0.64033043384552 +S0001,115614313,30/08/2008 23:00,sub-S0001115614313,1.0,75.0,Male,White,Not Hispanic,23.15,,True,16/09/2016,2938.0,0.27257463335990906 +S0001,115627888,24/07/2012 1:31,sub-S0001115627888,1.0,85.0,Female,White,Not Hispanic,35.67,1246.0,True,05/06/2016,1411.0,0.6497296094894409 +S0001,115685485,30/01/2009 22:03,sub-S0001115685485,1.0,50.0,Male,White,Not Hispanic,39.77,,True,20/06/2023,5253.0,0.2891353964805603 +S0001,115724572,10/12/2011 1:50,sub-S0001115724572,1.0,57.0,Female,White,Not Hispanic,34.94,,True,27/01/2024,4430.0,0.6525198221206665 +S0001,115804130,02/04/2016 22:00,sub-S0001115804130,2.0,78.0,Male,White,Not Hispanic,26.91,,True,09/06/2023,2623.0,0.5654754638671875 +S0001,115810419,28/08/2011 1:56,sub-S0001115810419,1.0,60.0,Female,White,Not Hispanic,20.89,,True,16/02/2023,4189.0,0.5012294054031372 +S0001,115823207,19/08/2009 23:28,sub-S0001115823207,1.0,80.0,Female,White,Not Hispanic,20.97,,True,02/12/2023,5217.0,0.4938763380050659 +S0001,115866465,22/03/2017 23:41,sub-S0001115866465,1.0,75.0,Male,Unavailable,Not Hispanic,17.78,1445.0,True,13/09/2023,2365.0,0.27257463335990906 +S0001,115881418,14/03/2016 22:35,sub-S0001115881418,1.0,54.0,Male,White,Not Hispanic,36.74,,True,06/06/2023,2639.0,0.47246167063713074 +S0001,115883585,03/03/2018 19:18,sub-S0001115883585,1.0,71.0,Male,White,Not Hispanic,24.12,1138.0,True,05/01/2023,1768.0,0.40006473660469055 +S0001,115946863,19/09/2008 22:55,sub-S0001115946863,1.0,65.0,Male,White,Not Hispanic,24.34,1450.0,True,06/08/2022,5068.0,0.4400545656681061 +S0001,116017075,08/10/2012 22:29,sub-S0001116017075,1.0,61.0,Male,White,Not Hispanic,24.93,,True,26/12/2023,4095.0,0.6737512946128845 +S0001,116045066,20/02/2010 2:15,sub-S0001116045066,1.0,64.0,Female,Black,Not Hispanic,24.11,,True,16/03/2023,4771.0,0.8236727714538574 +S0001,116052557,05/01/2013 21:59,sub-S0001116052557,1.0,70.0,Female,White,Not Hispanic,20.03,,True,04/12/2023,3984.0,0.40108874440193176 +S0001,116140777,30/06/2009 2:02,sub-S0001116140777,1.0,70.0,Male,Asian,Not Hispanic,24.41,,True,10/01/2023,4941.0,0.483925461769104 +S0001,116264624,05/07/2012 1:38,sub-S0001116264624,1.0,71.0,Male,White,Not Hispanic,40.19,1789.0,True,18/11/2021,3422.0,0.40006473660469055 +S0001,116283126,08/04/2009 0:43,sub-S0001116283126,1.0,66.0,Male,White,Not Hispanic,35.58,1892.0,True,29/01/2023,5043.0,0.45043623447418213 +S0001,116289727,10/09/2010 1:57,sub-S0001116289727,1.0,72.0,Female,White,Not Hispanic,25.21,,True,10/04/2021,3864.0,0.5070144534111023 +S0001,116308865,22/08/2011 2:44,sub-S0001116308865,1.0,75.0,Female,White,Not Hispanic,35.02,,True,17/11/2023,4469.0,0.6455416083335876 +S0001,116330821,25/12/2009 1:49,sub-S0001116330821,1.0,74.0,Male,White,Not Hispanic,25.25,,True,26/11/2022,4718.0,0.39002496004104614 +S0001,116342936,07/07/2015 22:35,sub-S0001116342936,1.0,76.0,Male,White,Not Hispanic,24.57,,True,26/08/2023,2971.0,0.2753254175186157 +S0001,116399255,15/05/2016 22:12,sub-S0001116399255,1.0,59.0,Female,White,Not Hispanic,23.36,,True,24/10/2023,2717.0,0.5383100509643555 +S0001,116473509,13/11/2013 22:10,sub-S0001116473509,1.0,77.0,Male,Asian,Not Hispanic,23.16,,True,10/02/2024,3740.0,0.8424844145774841 +S0001,116587467,09/10/2010 2:08,sub-S0001116587467,1.0,71.0,Male,White,Not Hispanic,26.62,,True,24/03/2023,4548.0,0.40006473660469055 +S0001,116604699,02/03/2014 22:54,sub-S0001116604699,1.0,63.0,Female,Others,Hispanic,38.6,,True,19/09/2021,2757.0,0.7037391066551208 +S0001,116662355,26/02/2019 19:45,sub-S0001116662355,1.0,77.0,Male,White,Not Hispanic,23.92,1372.0,True,20/09/2023,1666.0,0.8424844145774841 +S0001,116697035,17/10/2011 1:41,sub-S0001116697035,1.0,61.0,Female,White,Not Hispanic,43.93,,True,23/06/2020,3171.0,0.824192225933075 +S0001,116772588,14/02/2011 1:37,sub-S0001116772588,1.0,74.0,Male,Unavailable,Hispanic,26.24,,True,06/10/2022,4251.0,0.39002496004104614 +S0001,116774437,17/03/2009 21:55,sub-S0001116774437,1.0,68.0,Female,White,Not Hispanic,23.47,,True,04/02/2023,5071.0,0.346417635679245 +S0001,116889560,02/09/2009 21:53,sub-S0001116889560,1.0,53.0,Female,White,Not Hispanic,43.15,,True,19/07/2023,5067.0,0.3207913041114807 +S0001,116901749,15/05/2014 22:43,sub-S0001116901749,1.0,66.0,Male,White,Not Hispanic,33.33,,True,16/04/2024,3623.0,0.45043623447418213 +S0001,116904925,06/11/2012 3:18,sub-S0001116904925,1.0,54.0,Male,White,Not Hispanic,27.32,,True,14/12/2023,4054.0,0.47246167063713074 +S0001,116988457,01/05/2010 22:22,sub-S0001116988457,1.0,63.0,Male,White,Not Hispanic,30.81,,True,14/08/2023,4852.0,0.4706968069076538 +S0001,116998567,21/08/2011 22:05,sub-S0001116998567,1.0,75.0,Male,White,Not Hispanic,27.69,,True,16/07/2024,4712.0,0.27257463335990906 +S0001,117019749,28/08/2009 23:34,sub-S0001117019749,1.0,70.0,Male,White,Not Hispanic,24.03,,True,05/10/2023,5150.0,0.483925461769104 +S0001,117057718,11/02/2015 22:57,sub-S0001117057718,2.0,66.0,Male,White,Not Hispanic,38.24,,True,15/10/2022,2802.0,0.45043623447418213 +S0001,117102202,21/10/2009 1:36,sub-S0001117102202,1.0,59.0,Female,White,Hispanic,20.65,,True,29/11/2021,4421.0,0.5383100509643555 +S0001,117168117,06/10/2008 22:04,sub-S0001117168117,1.0,66.0,Male,White,Not Hispanic,25.79,,True,18/06/2020,4272.0,0.45043623447418213 +S0001,117181095,21/01/2009 22:21,sub-S0001117181095,1.0,55.0,Male,White,Not Hispanic,36.51,,True,26/01/2023,5117.0,0.19085277616977692 +S0001,117236297,16/01/2009 22:09,sub-S0001117236297,1.0,63.0,Female,White,Not Hispanic,31.02,,True,26/02/2024,5518.0,0.7037391066551208 +S0001,117262757,30/05/2013 3:07,sub-S0001117262757,1.0,75.0,Female,White,Not Hispanic,23.16,,True,18/12/2023,3853.0,0.6455416083335876 +S0001,117356069,30/06/2012 2:10,sub-S0001117356069,1.0,62.0,Female,White,Not Hispanic,32.01,1643.0,True,25/05/2018,2154.0,0.7453621029853821 +S0001,117392261,18/08/2016 22:06,sub-S0001117392261,2.0,69.0,Male,White,Not Hispanic,32.33,,True,22/12/2023,2681.0,0.6186583042144775 +S0001,117404228,20/09/2008 22:05,sub-S0001117404228,1.0,79.0,Male,White,Unavailable,22.6,1732.0,True,27/08/2017,3262.0,0.4199504256248474 +S0001,117408887,06/01/2009 22:38,sub-S0001117408887,1.0,80.0,Female,White,Not Hispanic,30.87,,True,22/05/2023,5248.0,0.4938763380050659 +S0001,117438884,17/08/2013 22:52,sub-S0001117438884,1.0,64.0,Female,White,Not Hispanic,33.45,,True,15/05/2023,3557.0,0.8236727714538574 +S0001,117440861,11/11/2011 1:44,sub-S0001117440861,1.0,70.0,Male,White,Not Hispanic,28.95,,True,03/02/2020,3005.0,0.483925461769104 +S0001,117553277,28/09/2015 22:54,sub-S0001117553277,1.0,77.0,Male,White,Not Hispanic,35.95,2467.0,True,28/12/2022,2647.0,0.8424844145774841 +S0001,117671783,20/11/2008 22:59,sub-S0001117671783,1.0,69.0,Male,White,Not Hispanic,29.45,2045.0,True,22/08/2020,4292.0,0.6186583042144775 +S0001,117676967,10/10/2015 23:18,sub-S0001117676967,1.0,78.0,Male,White,Not Hispanic,27.23,,True,23/02/2023,2692.0,0.5654754638671875 +S0001,117784533,12/04/2015 21:35,sub-S0001117784533,1.0,79.0,Female,White,Not Hispanic,30.49,,True,24/11/2023,3147.0,0.7526390552520752 +S0001,117788322,08/09/2010 22:03,sub-S0001117788322,1.0,73.0,Male,White,Not Hispanic,27.8,,True,01/11/2022,4436.0,0.8519553542137146 +S0001,117805802,07/01/2008 23:23,sub-S0001117805802,1.0,58.0,Male,White,Not Hispanic,,,True,30/05/2017,3430.0,0.23777179419994354 +S0001,117809853,24/06/2017 21:33,sub-S0001117809853,1.0,80.0,Male,White,Not Hispanic,26.23,1186.0,True,01/11/2023,2320.0,0.34214648604393005 +S0001,117863261,07/06/2016 21:35,sub-S0001117863261,1.0,69.0,Male,White,Not Hispanic,27.69,1358.0,True,11/08/2023,2620.0,0.6186583042144775 +S0001,117930679,04/12/2013 21:34,sub-S0001117930679,1.0,79.0,Male,White,Not Hispanic,29.23,,True,28/08/2023,3553.0,0.4199504256248474 +S0001,117949855,23/08/2008 22:57,sub-S0001117949855,1.0,68.0,Male,White,Not Hispanic,27.93,,True,27/03/2017,3137.0,0.39139872789382935 +S0001,117953563,10/02/2017 22:57,sub-S0001117953563,1.0,62.0,Female,White,Not Hispanic,30.64,,True,30/05/2024,2665.0,0.7453621029853821 +S0001,117960082,11/08/2012 0:00,sub-S0001117960082,1.0,67.0,Female,White,Not Hispanic,24.79,,True,03/01/2023,3797.0,0.6473630666732788 +S0001,117998576,22/01/2010 23:33,sub-S0001117998576,1.0,70.0,Male,White,Not Hispanic,32.53,,True,09/03/2022,4428.0,0.483925461769104 +S0001,118007341,05/12/2009 2:10,sub-S0001118007341,1.0,68.0,Male,White,Not Hispanic,34.21,,True,06/07/2017,2769.0,0.39139872789382935 +S0001,118007817,08/02/2009 22:36,sub-S0001118007817,1.0,80.0,Female,White,Not Hispanic,28.21,,True,16/01/2024,5454.0,0.4938763380050659 +S0001,118035891,06/11/2008 22:18,sub-S0001118035891,1.0,80.0,Female,White,Not Hispanic,22.57,,True,22/02/2022,4855.0,0.4938763380050659 +S0001,118065598,18/07/2010 2:26,sub-S0001118065598,1.0,70.0,Male,White,Not Hispanic,31.32,,True,19/09/2017,2619.0,0.483925461769104 +S0001,118092066,15/10/2014 22:56,sub-S0001118092066,1.0,88.0,Male,White,Not Hispanic,24.11,1268.0,True,15/06/2019,1703.0,0.6507008671760559 +S0001,118147076,18/09/2015 23:25,sub-S0001118147076,2.0,72.0,Male,White,Not Hispanic,24.44,,True,02/06/2023,2813.0,0.5197765827178955 +S0001,118163191,01/12/2009 22:32,sub-S0001118163191,1.0,68.0,Female,White,Not Hispanic,21.85,,True,23/05/2024,5286.0,0.346417635679245 +S0001,118184191,23/04/2010 3:20,sub-S0001118184191,1.0,77.0,Male,White,Not Hispanic,24.97,,True,26/09/2023,4903.0,0.8424844145774841 +S0001,118216285,07/02/2016 23:01,sub-S0001118216285,1.0,67.0,Male,Black,Not Hispanic,27.08,,True,02/09/2023,2763.0,0.9195594787597656 +S0001,118280628,19/04/2013 22:56,sub-S0001118280628,1.0,65.0,Male,White,Not Hispanic,25.29,,True,27/05/2023,3689.0,0.4400545656681061 +S0001,118283012,31/03/2011 22:17,sub-S0001118283012,1.0,77.0,Female,White,Not Hispanic,39.32,,True,24/03/2023,4375.0,0.14538460969924927 +S0001,118324303,29/06/2011 3:32,sub-S0001118324303,1.0,70.0,Male,White,Not Hispanic,31.54,,True,20/08/2023,4434.0,0.483925461769104 +S0001,118326393,05/03/2017 21:52,sub-S0001118326393,1.0,78.0,Male,White,Not Hispanic,38.26,1298.0,True,07/11/2022,2072.0,0.5654754638671875 +S0001,118395905,10/11/2011 22:16,sub-S0001118395905,1.0,71.0,Male,White,Not Hispanic,22.34,,True,22/03/2024,4515.0,0.40006473660469055 +S0001,118411164,06/02/2016 21:42,sub-S0001118411164,1.0,57.0,Male,White,Not Hispanic,28.5,,True,08/04/2023,2617.0,0.5427361130714417 +S0001,118411262,28/08/2008 22:49,sub-S0001118411262,1.0,78.0,Male,White,Not Hispanic,28.21,,True,12/02/2023,5280.0,0.5654754638671875 +S0001,118413807,29/10/2016 23:38,sub-S0001118413807,1.0,56.0,Male,White,Not Hispanic,24.02,,True,19/09/2024,2881.0,0.501916766166687 +S0001,118429956,15/10/2011 22:04,sub-S0001118429956,1.0,86.0,Male,Others,Not Hispanic,30.79,,True,26/05/2020,3145.0,0.7442228198051453 +S0001,118495502,19/10/2012 22:55,sub-S0001118495502,1.0,77.0,Female,White,Not Hispanic,36.87,,True,09/01/2023,3733.0,0.14538460969924927 +S0001,118513036,03/05/2008 22:37,sub-S0001118513036,1.0,71.0,Male,White,Unavailable,39.17,2462.0,True,22/04/2015,2544.0,0.40006473660469055 +S0001,118515170,13/05/2015 22:34,sub-S0001118515170,2.0,72.0,Male,White,Not Hispanic,23.55,1940.0,True,20/07/2021,2259.0,0.5197765827178955 +S0001,118574571,15/01/2015 22:17,sub-S0001118574571,1.0,63.0,Male,White,Hispanic,29.55,,True,16/02/2023,2953.0,0.4706968069076538 +S0001,118574603,27/05/2016 21:55,sub-S0001118574603,1.0,70.0,Male,White,Not Hispanic,29.05,,True,10/06/2023,2569.0,0.483925461769104 +S0001,118617888,03/11/2012 2:38,sub-S0001118617888,1.0,67.0,Female,Others,Hispanic,38.01,,True,19/04/2024,4184.0,0.6473630666732788 +S0001,118623840,28/03/2008 23:10,sub-S0001118623840,1.0,70.0,Female,Black,Not Hispanic,30.23,,True,27/04/2021,4777.0,0.40108874440193176 +S0001,118652295,07/08/2012 2:17,sub-S0001118652295,1.0,75.0,Female,Unavailable,Not Hispanic,34.06,1705.0,True,21/02/2021,3119.0,0.6455416083335876 +S0001,118930074,03/06/2015 21:53,sub-S0001118930074,1.0,71.0,Male,White,Not Hispanic,25.36,,True,26/11/2022,2732.0,0.40006473660469055 +S0001,118950203,16/04/2010 1:40,sub-S0001118950203,1.0,65.0,Male,White,Not Hispanic,41.19,,True,20/01/2023,4661.0,0.4400545656681061 +S0001,119000158,08/12/2012 1:41,sub-S0001119000158,1.0,74.0,Male,Black,Not Hispanic,40.67,,True,19/12/2020,2932.0,0.39002496004104614 +S0001,119040852,20/01/2009 22:11,sub-S0001119040852,1.0,74.0,Female,White,Not Hispanic,25.16,,True,01/05/2023,5213.0,0.33417800068855286 +S0001,119144776,06/06/2010 2:43,sub-S0001119144776,1.0,86.0,Male,White,Not Hispanic,21.04,,True,05/07/2019,3315.0,0.7442228198051453 +S0001,119200505,23/01/2009 3:56,sub-S0001119200505,1.0,51.0,Male,White,Not Hispanic,38.57,,True,31/01/2024,5485.0,0.4153312146663666 +S0001,119214666,01/09/2011 1:57,sub-S0001119214666,1.0,73.0,Male,White,Not Hispanic,25.02,2032.0,True,28/04/2021,3526.0,0.8519553542137146 +S0001,119239504,20/03/2011 2:25,sub-S0001119239504,1.0,66.0,Male,White,Not Hispanic,39.37,,True,05/09/2019,3090.0,0.45043623447418213 +S0001,119240200,02/08/2011 3:40,sub-S0001119240200,1.0,65.0,Male,White,Not Hispanic,33.83,2404.0,True,27/07/2023,4376.0,0.4400545656681061 +S0001,119336916,30/07/2013 1:41,sub-S0001119336916,1.0,71.0,Male,White,Not Hispanic,32.69,,True,18/07/2023,3639.0,0.40006473660469055 +S0001,119363044,04/03/2013 22:20,sub-S0001119363044,1.0,70.0,Male,White,Not Hispanic,32.85,,True,05/06/2021,3014.0,0.483925461769104 +S0001,119390537,28/07/2011 22:53,sub-S0001119390537,1.0,77.0,Female,White,Not Hispanic,26.17,2185.0,True,19/12/2022,4161.0,0.14538460969924927 +S0001,119415377,25/07/2015 22:19,sub-S0001119415377,1.0,70.0,Female,White,Not Hispanic,30.95,,True,10/08/2023,2937.0,0.40108874440193176 +S0001,119444122,17/10/2010 1:58,sub-S0001119444122,1.0,83.0,Male,White,Not Hispanic,22.98,,True,16/02/2020,3408.0,0.8366506099700928 +S0001,119501950,02/05/2009 1:36,sub-S0001119501950,1.0,77.0,Male,White,Not Hispanic,28.68,,True,13/10/2022,4911.0,0.8424844145774841 +S0001,119540284,05/02/2011 1:52,sub-S0001119540284,1.0,73.0,Male,White,Not Hispanic,28.82,,True,27/09/2022,4251.0,0.8519553542137146 +S0001,119592132,06/07/2017 22:58,sub-S0001119592132,1.0,82.0,Female,White,Hispanic,22.59,1752.0,True,08/01/2024,2376.0,0.5433937311172485 +S0001,119662091,05/05/2010 22:56,sub-S0001119662091,1.0,66.0,Female,White,Not Hispanic,41.12,,True,26/01/2022,4283.0,0.40600818395614624 +S0001,119693286,03/11/2012 22:36,sub-S0001119693286,1.0,70.0,Male,White,Not Hispanic,48.82,,True,05/05/2021,3104.0,0.483925461769104 +S0001,119775090,12/06/2013 22:39,sub-S0001119775090,1.0,53.0,Female,Unavailable,Not Hispanic,27.06,,True,01/07/2023,3670.0,0.3207913041114807 +S0001,119799817,06/05/2012 21:50,sub-S0001119799817,1.0,84.0,Male,White,Not Hispanic,24.1,,True,10/09/2023,4143.0,0.7442463636398315 +S0001,119825976,11/07/2013 3:04,sub-S0001119825976,1.0,77.0,Male,White,Not Hispanic,32.42,,True,25/10/2020,2662.0,0.8424844145774841 +S0001,119843121,03/08/2015 22:34,sub-S0001119843121,1.0,83.0,Male,White,Not Hispanic,27.21,1281.0,True,29/08/2021,2217.0,0.8366506099700928 +S0001,119862121,02/11/2011 21:35,sub-S0001119862121,1.0,64.0,Female,White,Not Hispanic,33.96,,True,12/02/2024,4484.0,0.8236727714538574 +S0001,119871328,15/01/2013 23:53,sub-S0001119871328,1.0,70.0,Male,White,Not Hispanic,33.2,,True,26/06/2024,4179.0,0.483925461769104 +S0001,119889263,30/11/2018 22:50,sub-S0001119889263,1.0,79.0,Male,White,Not Hispanic,35.13,1665.0,True,06/07/2024,2044.0,0.4199504256248474 +S0001,119924456,27/03/2010 21:12,sub-S0001119924456,1.0,79.0,Male,White,Unavailable,27.3,2153.0,True,06/11/2017,2780.0,0.4199504256248474 +S0001,119932412,08/07/2013 0:48,sub-S0001119932412,1.0,63.0,Male,White,Not Hispanic,36.85,,True,22/01/2024,3849.0,0.4706968069076538 +S0001,119965834,17/06/2010 2:19,sub-S0001119965834,1.0,60.0,Male,Unavailable,Not Hispanic,29.45,,True,27/07/2022,4422.0,0.24748243391513824 +S0001,119989726,03/05/2012 1:34,sub-S0001119989726,1.0,68.0,Male,White,Not Hispanic,28.51,1597.0,True,24/08/2023,4129.0,0.39139872789382935 +S0001,119998090,18/08/2019 20:27,sub-S0001119998090,1.0,75.0,Male,White,Not Hispanic,34.29,1402.0,True,12/08/2024,1820.0,0.27257463335990906 +S0001,120005093,19/10/2010 21:46,sub-S0001120005093,1.0,50.0,Male,White,Hispanic,30.31,,True,12/05/2024,4953.0,0.2891353964805603 +S0001,120033988,30/12/2009 21:56,sub-S0001120033988,1.0,82.0,Male,White,Not Hispanic,20.01,,True,20/11/2022,4707.0,0.44877195358276367 +S0001,120089914,03/05/2008 21:46,sub-S0001120089914,1.0,69.0,Male,White,Not Hispanic,32.67,,True,01/06/2021,4776.0,0.6186583042144775 +S0001,120158803,09/01/2011 1:59,sub-S0001120158803,1.0,72.0,Male,White,Not Hispanic,24.77,,True,07/11/2019,3223.0,0.5197765827178955 +S0001,120211143,14/02/2017 21:06,sub-S0001120211143,1.0,73.0,Female,White,Not Hispanic,24.17,,True,16/03/2024,2586.0,0.9163349866867065 +S0001,120276000,07/07/2015 21:46,sub-S0001120276000,1.0,66.0,Male,White,Not Hispanic,30.72,,True,28/02/2024,3157.0,0.45043623447418213 +S0001,120276384,18/05/2012 0:00,sub-S0001120276384,1.0,70.0,Male,White,Not Hispanic,30.18,1323.0,True,22/10/2023,4174.0,0.483925461769104 +S0001,120297473,04/11/2016 22:17,sub-S0001120297473,1.0,80.0,Female,White,Not Hispanic,29.32,,True,11/03/2024,2683.0,0.4938763380050659 +S0001,120336844,25/07/2012 1:57,sub-S0001120336844,1.0,74.0,Female,Others,Hispanic,20.67,,True,24/01/2023,3834.0,0.33417800068855286 +S0001,120377604,17/05/2011 2:00,sub-S0001120377604,1.0,67.0,Male,White,Not Hispanic,24.47,,True,31/01/2023,4276.0,0.9195594787597656 +S0001,120378159,28/07/2016 21:53,sub-S0001120378159,1.0,88.0,Male,White,Not Hispanic,32.29,,True,22/01/2024,2733.0,0.6507008671760559 +S0001,120385725,12/09/2008 23:04,sub-S0001120385725,1.0,70.0,Male,White,Not Hispanic,29.36,,True,14/10/2023,5509.0,0.483925461769104 +S0001,120409024,17/12/2011 1:36,sub-S0001120409024,1.0,75.0,Male,White,Not Hispanic,27.39,,True,19/10/2022,3958.0,0.27257463335990906 +S0001,120412309,11/04/2014 21:30,sub-S0001120412309,1.0,75.0,Male,White,Not Hispanic,27.46,,True,23/05/2023,3328.0,0.27257463335990906 +S0001,120527660,11/06/2010 3:10,sub-S0001120527660,1.0,77.0,Female,White,Not Hispanic,26.56,,True,17/09/2023,4845.0,0.14538460969924927 +S0001,120544140,13/12/2013 22:51,sub-S0001120544140,1.0,64.0,Female,Others,Hispanic,34.87,,True,23/07/2024,3874.0,0.8236727714538574 +S0001,120577416,02/05/2016 21:39,sub-S0001120577416,1.0,72.0,Female,White,Not Hispanic,31.51,1151.0,True,21/07/2023,2635.0,0.5070144534111023 +S0001,120596213,25/04/2016 22:23,sub-S0001120596213,1.0,54.0,Female,White,Not Hispanic,26.8,,True,19/08/2023,2671.0,0.6143807172775269 +S0001,120633409,09/02/2012 22:12,sub-S0001120633409,1.0,72.0,Female,White,Not Hispanic,37.74,,True,22/03/2022,3693.0,0.5070144534111023 +S0001,120673661,23/06/2012 2:51,sub-S0001120673661,2.0,55.0,Male,White,Not Hispanic,28.4,,True,21/12/2022,3832.0,0.19085277616977692 +S0001,120688704,02/11/2016 21:58,sub-S0001120688704,1.0,81.0,Female,White,Not Hispanic,34.85,,True,30/12/2023,2613.0,0.40572720766067505 +S0001,120707455,08/07/2017 21:51,sub-S0001120707455,1.0,60.0,Male,White,Not Hispanic,26.21,1222.0,True,06/10/2024,2646.0,0.24748243391513824 +S0001,120717749,21/06/2018 19:21,sub-S0001120717749,1.0,59.0,Male,White,Not Hispanic,25.44,1453.0,True,27/01/2023,1680.0,0.2705105245113373 +S0001,120747382,30/05/2010 2:51,sub-S0001120747382,1.0,74.0,Male,White,Not Hispanic,31.95,,True,06/03/2023,4662.0,0.39002496004104614 +S0001,120752810,28/12/2013 21:44,sub-S0001120752810,1.0,70.0,Male,Black,Not Hispanic,31.37,,True,18/02/2024,3703.0,0.483925461769104 +S0001,120764087,17/07/2010 2:53,sub-S0001120764087,1.0,72.0,Female,White,Not Hispanic,25.84,1113.0,True,19/08/2018,2954.0,0.5070144534111023 +S0001,120783618,22/07/2012 22:06,sub-S0001120783618,1.0,80.0,Female,Black,Not Hispanic,35.09,,True,05/05/2020,2843.0,0.4938763380050659 +S0001,120821892,04/10/2015 21:47,sub-S0001120821892,1.0,62.0,Male,Black,Hispanic,35.6,,True,13/12/2022,2626.0,0.7553417086601257 +S0001,120827718,15/06/2013 21:46,sub-S0001120827718,1.0,58.0,Male,White,Not Hispanic,28.74,,True,25/05/2023,3630.0,0.23777179419994354 +S0001,120830792,07/08/2011 3:20,sub-S0001120830792,1.0,79.0,Male,White,Not Hispanic,26.98,,True,26/03/2024,4614.0,0.4199504256248474 +S0001,120869188,08/08/2009 3:19,sub-S0001120869188,1.0,71.0,Male,White,Not Hispanic,28.06,2074.0,True,08/10/2022,4808.0,0.40006473660469055 +S0001,121017443,03/05/2013 23:30,sub-S0001121017443,1.0,64.0,Female,White,Not Hispanic,28.07,,True,23/06/2023,3702.0,0.8236727714538574 +S0001,121017473,19/01/2015 21:38,sub-S0001121017473,1.0,69.0,Female,White,Not Hispanic,34.77,,True,18/09/2022,2798.0,0.6564435362815857 +S0001,121038180,02/08/2014 1:48,sub-S0001121038180,1.0,62.0,Female,Unavailable,Hispanic,36.6,,True,03/04/2023,3165.0,0.7453621029853821 +S0001,121063935,12/05/2010 3:05,sub-S0001121063935,1.0,74.0,Female,White,Not Hispanic,32.44,,True,27/10/2023,4915.0,0.33417800068855286 +S0001,121092939,03/03/2010 22:03,sub-S0001121092939,1.0,71.0,Female,White,Not Hispanic,33.16,,True,23/06/2024,5225.0,0.5782973766326904 +S0001,121121018,01/01/2016 23:04,sub-S0001121121018,1.0,66.0,Male,White,Not Hispanic,24.78,,True,09/07/2024,3111.0,0.45043623447418213 +S0001,121255336,08/09/2015 23:09,sub-S0001121255336,2.0,76.0,Female,White,Not Hispanic,34.34,,True,06/10/2022,2584.0,0.5227447152137756 +S0001,121264507,22/02/2011 1:53,sub-S0001121264507,1.0,61.0,Male,White,Not Hispanic,27.58,,True,09/05/2023,4458.0,0.6737512946128845 +S0001,121295515,17/08/2015 23:17,sub-S0001121295515,1.0,63.0,Female,White,Not Hispanic,24.72,,True,28/01/2023,2720.0,0.7037391066551208 +S0001,121337883,25/12/2012 2:39,sub-S0001121337883,1.0,63.0,Female,White,Not Hispanic,37.31,,True,09/07/2024,4213.0,0.7037391066551208 +S0001,121370986,22/08/2011 2:21,sub-S0001121370986,1.0,69.0,Female,White,Not Hispanic,29.64,,True,30/07/2021,3629.0,0.6564435362815857 +S0001,121449477,17/07/2009 22:08,sub-S0001121449477,1.0,65.0,Male,White,Not Hispanic,28.18,,True,20/02/2023,4965.0,0.4400545656681061 +S0001,121488998,04/08/2012 1:56,sub-S0001121488998,1.0,73.0,Male,White,Not Hispanic,34.9,,True,17/01/2024,4182.0,0.8519553542137146 +S0001,121529475,08/11/2010 2:17,sub-S0001121529475,1.0,67.0,Male,White,Not Hispanic,30.7,,True,02/04/2023,4527.0,0.9195594787597656 +S0001,121573952,15/03/2009 22:57,sub-S0001121573952,1.0,72.0,Male,White,Not Hispanic,45.57,,True,07/01/2018,3219.0,0.5197765827178955 +S0001,121574219,18/10/2010 3:07,sub-S0001121574219,1.0,65.0,Female,White,Hispanic,28.78,,True,03/02/2023,4490.0,0.7371628880500793 +S0001,121594285,11/09/2008 22:20,sub-S0001121594285,1.0,79.0,Female,White,Not Hispanic,28.01,,True,27/01/2020,4154.0,0.7526390552520752 +S0001,121693594,02/08/2009 2:26,sub-S0001121693594,1.0,85.0,Male,White,Not Hispanic,24.7,,True,03/06/2019,3591.0,0.6813521385192871 +S0001,121695905,02/08/2010 3:15,sub-S0001121695905,1.0,75.0,Male,White,Not Hispanic,30.41,2063.0,True,14/07/2016,2172.0,0.27257463335990906 +S0001,121716656,16/10/2015 22:19,sub-S0001121716656,1.0,60.0,Male,White,Not Hispanic,29.17,,True,04/10/2023,2909.0,0.24748243391513824 +S0001,121887520,05/10/2010 2:03,sub-S0001121887520,1.0,81.0,Male,White,Not Hispanic,26.6,,True,25/10/2017,2576.0,0.8153370022773743 +S0001,121903400,26/03/2015 22:04,sub-S0001121903400,1.0,50.0,Female,White,Not Hispanic,21.84,,True,01/09/2023,3080.0,0.5013509392738342 +S0001,121953316,21/08/2014 21:46,sub-S0001121953316,1.0,73.0,Male,White,Not Hispanic,29.21,,True,02/08/2023,3267.0,0.8519553542137146 +S0001,122023932,15/02/2011 0:00,sub-S0001122023932,1.0,79.0,Male,White,Not Hispanic,26.2,1762.0,True,18/12/2022,4324.0,0.4199504256248474 +S0001,122024052,22/07/2012 3:19,sub-S0001122024052,1.0,62.0,Male,White,Not Hispanic,28.21,,True,24/03/2022,3531.0,0.7553417086601257 +S0001,122053605,22/10/2012 2:57,sub-S0001122053605,1.0,62.0,Female,White,Not Hispanic,27.31,,True,06/09/2024,4336.0,0.7453621029853821 +S0001,122094999,23/02/2011 1:41,sub-S0001122094999,1.0,67.0,Male,White,Not Hispanic,36.13,,True,28/06/2023,4507.0,0.9195594787597656 +S0001,122115803,16/07/2013 23:07,sub-S0001122115803,1.0,60.0,Male,White,Not Hispanic,34.03,,True,08/08/2023,3674.0,0.24748243391513824 +S0001,122160941,09/10/2015 22:55,sub-S0001122160941,1.0,76.0,Female,White,Not Hispanic,32.75,,True,21/09/2023,2903.0,0.5227447152137756 +S0001,122221329,11/10/2015 22:40,sub-S0001122221329,2.0,68.0,Male,White,Not Hispanic,31.48,,True,19/01/2024,3021.0,0.39139872789382935 +S0001,122226335,22/08/2012 0:00,sub-S0001122226335,1.0,73.0,Female,White,Not Hispanic,38.24,1143.0,True,11/07/2020,2880.0,0.9163349866867065 +S0001,122360618,06/10/2008 21:30,sub-S0001122360618,1.0,70.0,Male,White,Not Hispanic,30.17,1121.0,True,04/01/2023,5202.0,0.483925461769104 +I0002,150000686,04/01/2019 22:29,sub-I0002150000686,1.0,82.0,Male,Black,Not Hispanic,34.25,1104.0,True,24/11/2023,1784.0,0.44877195358276367 +I0002,150002879,18/12/2016 20:58,sub-I0002150002879,1.0,75.0,Male,White,Not Hispanic,19.62,1313.0,True,21/12/2020,1463.0,0.27257463335990906 +I0002,150003895,10/08/2017 21:46,sub-I0002150003895,1.0,76.0,Male,White,Not Hispanic,,,True,08/04/2024,2432.0,0.2753254175186157 +I0002,150004526,14/12/2016 22:36,sub-I0002150004526,1.0,76.0,Male,White,Not Hispanic,22.19,1487.0,True,03/07/2021,1661.0,0.2753254175186157 +I0002,150004858,27/08/2016 21:53,sub-I0002150004858,2.0,78.0,Male,White,Not Hispanic,,,True,15/06/2023,2482.0,0.5654754638671875 +I0002,150005910,03/06/2018 22:06,sub-I0002150005910,1.0,55.0,Male,White,Not Hispanic,,1335.0,True,02/02/2025,2435.0,0.19085277616977692 +I0002,150006084,30/05/2017 21:42,sub-I0002150006084,1.0,70.0,Female,White,Not Hispanic,32.53,,True,03/06/2023,2194.0,0.40108874440193176 +I0002,150022286,03/11/2016 21:18,sub-I0002150022286,1.0,69.0,Male,White,Not Hispanic,,,True,20/10/2023,2541.0,0.6186583042144775 +I0002,150023281,14/04/2018 21:02,sub-I0002150023281,1.0,66.0,Female,White,Not Hispanic,34.24,1234.0,True,13/06/2023,1885.0,0.40600818395614624 +I0002,150024576,22/07/2017 22:11,sub-I0002150024576,2.0,74.0,Male,White,Not Hispanic,,1303.0,True,17/06/2024,2521.0,0.39002496004104614 +I0002,150027698,01/03/2017 22:43,sub-I0002150027698,1.0,70.0,Male,White,Not Hispanic,,,True,09/08/2023,2351.0,0.483925461769104 +I0002,150028152,12/09/2016 22:19,sub-I0002150028152,1.0,62.0,Male,Others,Not Hispanic,,,True,13/09/2023,2556.0,0.7553417086601257 +I0002,150028177,26/08/2016 22:33,sub-I0002150028177,1.0,64.0,Male,White,Not Hispanic,26.31,,True,09/03/2023,2385.0,0.8189074397087097 +I0002,150028179,27/12/2017 21:15,sub-I0002150028179,2.0,81.0,Female,Black,Not Hispanic,,,True,10/12/2024,2539.0,0.40572720766067505 +I0002,150028223,05/01/2018 21:39,sub-I0002150028223,1.0,71.0,Female,White,Not Hispanic,,,True,05/01/2025,2556.0,0.5782973766326904 +I0002,150028247,09/09/2017 23:38,sub-I0002150028247,1.0,74.0,Female,White,Not Hispanic,,,True,26/08/2024,2542.0,0.33417800068855286 +I0002,150028374,17/08/2016 22:47,sub-I0002150028374,1.0,66.0,Female,Asian,Not Hispanic,,,True,24/04/2023,2440.0,0.40600818395614624 +I0002,150028391,06/02/2017 22:43,sub-I0002150028391,1.0,55.0,Male,White,Not Hispanic,,,True,13/09/2023,2409.0,0.19085277616977692 +I0002,150028394,22/10/2016 21:53,sub-I0002150028394,5.0,66.0,Male,Unavailable,Unavailable,,,True,09/07/2023,2450.0,0.45043623447418213 +I0002,150028694,23/10/2016 21:32,sub-I0002150028694,1.0,73.0,Female,Black,Not Hispanic,28.67,1502.0,True,01/06/2023,2411.0,0.9163349866867065 +I0002,150028714,11/06/2017 22:07,sub-I0002150028714,1.0,74.0,Female,White,Not Hispanic,15.1,,True,10/12/2023,2372.0,0.33417800068855286 +I0002,150028859,25/07/2018 22:20,sub-I0002150028859,1.0,52.0,Male,Unavailable,Not Hispanic,,,True,13/01/2025,2363.0,0.6616770625114441 +I0002,150028912,27/12/2017 22:59,sub-I0002150028912,2.0,73.0,Male,White,Not Hispanic,40.7,1865.0,True,11/02/2024,2236.0,0.8519553542137146 +I0002,150028988,26/04/2017 23:09,sub-I0002150028988,1.0,76.0,Female,Unavailable,Unavailable,,,True,03/11/2023,2381.0,0.5227447152137756 +I0002,150029061,15/05/2018 22:13,sub-I0002150029061,1.0,80.0,Male,White,Not Hispanic,30.17,,True,08/10/2024,2337.0,0.34214648604393005 +I0002,150029213,08/06/2018 22:29,sub-I0002150029213,1.0,64.0,Female,Black,Not Hispanic,,,True,30/09/2024,2305.0,0.8236727714538574 +I0002,150029266,06/09/2017 21:18,sub-I0002150029266,1.0,86.0,Male,White,Not Hispanic,26.63,,True,15/01/2024,2321.0,0.7442228198051453 +I0002,150029589,17/12/2018 22:39,sub-I0002150029589,1.0,75.0,Male,Asian,Not Hispanic,,,True,11/02/2025,2247.0,0.27257463335990906 +I0002,150029618,20/07/2018 22:28,sub-I0002150029618,1.0,77.0,Male,White,Not Hispanic,,,True,22/09/2024,2255.0,0.8424844145774841 +I0002,150031893,01/05/2019 22:24,sub-I0002150031893,1.0,74.0,Female,Black,Not Hispanic,,1113.0,True,19/01/2024,1723.0,0.33417800068855286 +I0002,150033481,09/08/2020 22:47,sub-I0002150033481,1.0,76.0,Female,White,Not Hispanic,,1164.0,True,01/11/2023,1178.0,0.5227447152137756 +I0002,150058421,08/11/2016 22:27,sub-I0002150058421,1.0,71.0,Female,White,Not Hispanic,,,True,22/05/2023,2385.0,0.5782973766326904 +I0002,150656547,06/04/2017 21:58,sub-I0002150656547,1.0,73.0,Male,White,Not Hispanic,,,True,10/02/2024,2500.0,0.8519553542137146 +I0006,179000007,14/11/2014,sub-I0006179000007,1.0,70.0,Female,White,Not Hispanic,,,True,08/01/2022,2612.0,0.40108874440193176 +I0006,179000101,01/09/2014,sub-I0006179000101,1.0,78.0,Female,Black,Not Hispanic,,,True,02/04/2022,2770.0,0.6120396256446838 +I0006,179000125,25/03/2015,sub-I0006179000125,1.0,74.0,Female,White,Not Hispanic,41.13,,True,26/12/2022,2833.0,0.33417800068855286 +I0006,179000270,17/01/2015,sub-I0006179000270,1.0,77.0,Male,White,Not Hispanic,24.74,,True,14/05/2022,2674.0,0.8424844145774841 +I0006,179000424,15/03/2013,sub-I0006179000424,1.0,68.0,Male,White,Not Hispanic,33.78,,True,07/05/2022,3340.0,0.39139872789382935 +I0006,179000812,06/11/2012,sub-I0006179000812,1.0,66.0,Male,White,Not Hispanic,28.19,,True,28/11/2022,3674.0,0.45043623447418213 +I0006,179000833,05/06/2012,sub-I0006179000833,1.0,51.0,Male,Black,Not Hispanic,31.2,,True,03/12/2021,3468.0,0.4153312146663666 +I0006,179001123,03/04/2014,sub-I0006179001123,1.0,68.0,Male,Black,Not Hispanic,21.53,,True,20/11/2021,2788.0,0.39139872789382935 +I0006,179001138,06/03/2016,sub-I0006179001138,1.0,72.0,Male,White,Unavailable,33.43,1798.0,True,06/12/2021,2101.0,0.5197765827178955 +I0006,179001182,29/08/2013,sub-I0006179001182,1.0,51.0,Female,Black,Not Hispanic,40.18,,True,13/07/2023,3605.0,0.509002149105072 +I0006,179001184,08/06/2012,sub-I0006179001184,1.0,68.0,Male,White,Not Hispanic,,1546.0,True,20/04/2022,3603.0,0.39139872789382935 +I0006,179001218,19/10/2018,sub-I0006179001218,1.0,73.0,Female,White,Not Hispanic,,1470.0,True,17/06/2023,1702.0,0.9163349866867065 +I0006,179001240,25/12/2016,sub-I0006179001240,1.0,78.0,Female,White,Not Hispanic,19.21,1220.0,True,16/01/2022,1848.0,0.6120396256446838 +I0006,179001308,19/02/2014,sub-I0006179001308,1.0,67.0,Female,Asian,Not Hispanic,32.98,,True,15/02/2022,2918.0,0.6473630666732788 +I0006,179001370,22/04/2012,sub-I0006179001370,1.0,68.0,Male,White,Not Hispanic,,1126.0,True,14/06/2016,1514.0,0.39139872789382935 +I0006,179001582,10/03/2015,sub-I0006179001582,1.0,74.0,Female,White,Not Hispanic,29.77,,True,20/04/2023,2963.0,0.33417800068855286 +I0006,179001769,10/02/2013,sub-I0006179001769,1.0,65.0,Male,White,Hispanic,29.65,2031.0,True,10/10/2021,3164.0,0.4400545656681061 +I0006,179001910,05/09/2017,sub-I0006179001910,1.0,60.0,Female,White,Hispanic,29.43,1849.0,True,25/01/2023,1968.0,0.5012294054031372 +I0006,179001928,05/01/2012,sub-I0006179001928,1.0,70.0,Male,White,Not Hispanic,,,True,15/01/2022,3663.0,0.483925461769104 +I0006,179002049,18/05/2016,sub-I0006179002049,1.0,77.0,Male,White,Not Hispanic,31.73,,True,28/05/2023,2566.0,0.8424844145774841 +I0006,179002155,22/07/2013,sub-I0006179002155,1.0,71.0,Male,White,Not Hispanic,30.92,,True,27/10/2021,3019.0,0.40006473660469055 +I0006,179002261,29/10/2014,sub-I0006179002261,1.0,64.0,Male,White,Not Hispanic,40.2,,True,02/08/2022,2834.0,0.8189074397087097 +I0006,179002374,09/08/2014,sub-I0006179002374,1.0,86.0,Male,White,Not Hispanic,21.67,1104.0,True,03/09/2019,1851.0,0.7442228198051453 +I0006,179002452,22/06/2013,sub-I0006179002452,1.0,71.0,Female,White,Not Hispanic,27.14,,True,23/05/2022,3257.0,0.5782973766326904 +I0006,179002531,26/03/2018,sub-I0006179002531,1.0,73.0,Male,White,Not Hispanic,,1212.0,True,30/04/2022,1496.0,0.8519553542137146 +I0006,179002653,25/11/2013,sub-I0006179002653,1.0,66.0,Male,White,Not Hispanic,22.71,,True,12/03/2022,3029.0,0.45043623447418213 +I0006,179003102,26/09/2013,sub-I0006179003102,1.0,60.0,Female,Black,Not Hispanic,26.24,,True,11/01/2022,3029.0,0.5012294054031372 +I0006,179003203,05/03/2015,sub-I0006179003203,1.0,78.0,Male,Black,Not Hispanic,25.39,1309.0,True,23/03/2020,1845.0,0.5654754638671875 +I0006,179003237,18/10/2013,sub-I0006179003237,1.0,61.0,Female,Black,Not Hispanic,,,True,26/10/2022,3295.0,0.824192225933075 +I0006,179003508,17/07/2015,sub-I0006179003508,1.0,73.0,Female,White,Not Hispanic,24.38,2461.0,True,10/06/2023,2885.0,0.9163349866867065 +I0006,179003575,05/03/2015,sub-I0006179003575,1.0,72.0,Female,Black,Not Hispanic,41.0,,True,11/06/2023,3020.0,0.5070144534111023 +I0006,179004102,19/02/2017,sub-I0006179004102,1.0,66.0,Female,White,Not Hispanic,24.57,1468.0,True,31/10/2022,2080.0,0.40600818395614624 +I0006,179004330,25/12/2014,sub-I0006179004330,1.0,69.0,Female,Black,Not Hispanic,43.81,1513.0,True,25/03/2022,2647.0,0.6564435362815857 +I0006,179004461,04/01/2015,sub-I0006179004461,3.0,69.0,Female,White,Not Hispanic,26.07,,True,07/03/2022,2619.0,0.6564435362815857 +I0006,179004462,27/01/2016,sub-I0006179004462,1.0,68.0,Male,White,Not Hispanic,27.48,,True,15/02/2023,2576.0,0.39139872789382935 +I0006,179004563,09/07/2014,sub-I0006179004563,1.0,64.0,Male,White,Not Hispanic,,,True,01/01/2022,2733.0,0.8189074397087097 +I0006,179004611,29/04/2014,sub-I0006179004611,2.0,64.0,Male,White,Not Hispanic,54.38,,True,23/08/2022,3038.0,0.8189074397087097 +I0006,179004802,04/05/2014,sub-I0006179004802,1.0,71.0,Male,White,Not Hispanic,24.84,,True,23/11/2021,2760.0,0.40006473660469055 +I0006,179004989,30/09/2013,sub-I0006179004989,1.0,71.0,Female,Black,Not Hispanic,,,True,04/04/2021,2743.0,0.5782973766326904 +I0006,179005139,26/10/2014,sub-I0006179005139,1.0,70.0,Male,Asian,Not Hispanic,,,True,24/02/2022,2678.0,0.483925461769104 +I0006,179005300,24/11/2013,sub-I0006179005300,1.0,85.0,Male,White,Not Hispanic,23.37,,True,14/06/2022,3124.0,0.6813521385192871 +I0006,179005468,30/11/2015,sub-I0006179005468,1.0,79.0,Male,White,Not Hispanic,43.16,1302.0,True,01/11/2021,2163.0,0.4199504256248474 +I0006,179005535,14/01/2014,sub-I0006179005535,1.0,66.0,Female,Black,Not Hispanic,24.65,,True,22/08/2022,3142.0,0.40600818395614624 +I0006,179005646,15/06/2012,sub-I0006179005646,1.0,82.0,Female,White,Not Hispanic,29.72,2553.0,True,24/07/2021,3326.0,0.5433937311172485 +I0006,179005751,16/06/2012,sub-I0006179005751,1.0,68.0,Female,White,Not Hispanic,26.96,,True,30/07/2021,3331.0,0.346417635679245 +I0006,179005925,14/08/2014,sub-I0006179005925,1.0,72.0,Male,White,Not Hispanic,28.16,,True,19/06/2022,2866.0,0.5197765827178955 +I0006,179006062,22/12/2015,sub-I0006179006062,1.0,69.0,Male,White,Not Hispanic,29.5,,True,15/02/2023,2612.0,0.6186583042144775 +I0006,179006142,31/12/2014,sub-I0006179006142,1.0,73.0,Male,White,Not Hispanic,27.19,2220.0,True,04/07/2022,2742.0,0.8519553542137146 +I0006,179006234,18/11/2013,sub-I0006179006234,1.0,50.0,Female,White,Not Hispanic,35.17,,True,29/07/2021,2810.0,0.5013509392738342 +I0006,179006362,21/08/2015,sub-I0006179006362,1.0,68.0,Male,White,Unavailable,,,True,02/12/2022,2660.0,0.39139872789382935 +I0006,179006516,06/08/2015,sub-I0006179006516,1.0,67.0,Male,White,Not Hispanic,31.8,2451.0,True,05/07/2022,2525.0,0.9195594787597656 +I0006,179006608,07/02/2017,sub-I0006179006608,1.0,74.0,Male,White,Not Hispanic,30.88,1565.0,True,27/10/2021,1723.0,0.39002496004104614 +I0006,179006626,24/04/2013,sub-I0006179006626,1.0,71.0,Female,White,Not Hispanic,31.41,,True,14/02/2022,3218.0,0.5782973766326904 +I0006,179006729,15/04/2013,sub-I0006179006729,1.0,75.0,Female,White,Not Hispanic,26.66,,True,07/02/2022,3220.0,0.6455416083335876 +I0006,179006730,15/09/2012,sub-I0006179006730,1.0,65.0,Female,White,Not Hispanic,27.18,,True,01/06/2022,3546.0,0.7371628880500793 +I0006,179006973,21/05/2013,sub-I0006179006973,1.0,68.0,Male,Black,Not Hispanic,28.46,,True,24/01/2022,3170.0,0.39139872789382935 +I0006,179007120,29/06/2012,sub-I0006179007120,1.0,70.0,Male,White,Not Hispanic,27.02,,True,19/10/2022,3764.0,0.483925461769104 +I0006,179007327,24/06/2013,sub-I0006179007327,1.0,75.0,Female,Black,Not Hispanic,34.41,,True,01/11/2022,3417.0,0.6455416083335876 +I0006,179007370,04/09/2015,sub-I0006179007370,1.0,62.0,Female,Black,Not Hispanic,31.17,,True,06/09/2022,2559.0,0.7453621029853821 +I0006,179007802,10/02/2017,sub-I0006179007802,1.0,69.0,Male,White,Not Hispanic,,1344.0,True,12/11/2021,1736.0,0.6186583042144775 +I0006,179007995,12/03/2012,sub-I0006179007995,1.0,66.0,Male,White,Not Hispanic,35.86,,True,14/01/2021,3230.0,0.45043623447418213 +I0006,179008230,22/09/2013,sub-I0006179008230,1.0,66.0,Female,Black,Not Hispanic,30.32,1130.0,True,27/04/2023,3504.0,0.40600818395614624 +I0006,179008398,31/07/2015,sub-I0006179008398,1.0,73.0,Female,White,Not Hispanic,35.08,,True,01/06/2023,2862.0,0.9163349866867065 +I0006,179008802,11/01/2018,sub-I0006179008802,1.0,77.0,Male,White,Not Hispanic,,1256.0,True,23/06/2022,1624.0,0.8424844145774841 +I0006,179008931,25/12/2015,sub-I0006179008931,1.0,73.0,Female,White,Not Hispanic,34.72,,True,10/03/2023,2632.0,0.9163349866867065 +I0006,179009256,08/08/2015,sub-I0006179009256,1.0,76.0,Male,White,Not Hispanic,26.1,,True,23/05/2023,2845.0,0.2753254175186157 +I0006,179009387,14/11/2014,sub-I0006179009387,1.0,84.0,Female,Black,Not Hispanic,21.9,,True,07/01/2022,2611.0,0.48952510952949524 +I0006,179009535,12/11/2014,sub-I0006179009535,1.0,66.0,Female,Black,Not Hispanic,,,True,24/04/2022,2720.0,0.40600818395614624 +I0006,179009648,24/01/2015,sub-I0006179009648,1.0,54.0,Male,White,Unavailable,,,True,27/02/2022,2591.0,0.47246167063713074 +I0006,179009818,05/06/2013,sub-I0006179009818,1.0,79.0,Female,Black,Not Hispanic,,1149.0,True,01/04/2023,3587.0,0.7526390552520752 +I0006,179009848,22/12/2014,sub-I0006179009848,1.0,51.0,Female,White,Hispanic,22.51,,True,21/02/2022,2618.0,0.509002149105072 +I0006,179010140,30/04/2013,sub-I0006179010140,1.0,73.0,Female,Black,Not Hispanic,27.14,,True,10/02/2022,3208.0,0.9163349866867065 +I0006,179010216,14/11/2015,sub-I0006179010216,1.0,62.0,Female,Black,Not Hispanic,41.21,,True,25/08/2023,2841.0,0.7453621029853821 +I0006,179010224,21/10/2012,sub-I0006179010224,1.0,71.0,Male,White,Not Hispanic,29.27,,True,13/07/2022,3552.0,0.40006473660469055 +I0006,179010345,10/10/2015,sub-I0006179010345,1.0,59.0,Male,Black,Not Hispanic,41.62,,True,27/04/2023,2756.0,0.2705105245113373 +I0006,179010644,20/04/2014,sub-I0006179010644,1.0,69.0,Female,White,Not Hispanic,27.44,,True,11/01/2023,3188.0,0.6564435362815857 +I0006,179010757,16/04/2014,sub-I0006179010757,1.0,69.0,Female,Black,Not Hispanic,40.38,,True,17/12/2022,3167.0,0.6564435362815857 +I0006,179010780,20/06/2014,sub-I0006179010780,1.0,57.0,Female,Black,Not Hispanic,44.08,,True,10/11/2023,3430.0,0.6525198221206665 +I0006,179011124,30/07/2011,sub-I0006179011124,1.0,79.0,Male,Black,Not Hispanic,29.84,,True,23/04/2021,3555.0,0.4199504256248474 +I0006,179011383,20/07/2015,sub-I0006179011383,1.0,61.0,Female,Black,Not Hispanic,30.85,,True,25/09/2022,2624.0,0.824192225933075 +I0006,179011615,04/11/2016,sub-I0006179011615,1.0,62.0,Male,White,Not Hispanic,32.43,1282.0,True,10/12/2021,1862.0,0.7553417086601257 +I0006,179011701,01/11/2015,sub-I0006179011701,1.0,76.0,Male,Black,Not Hispanic,65.65,,True,05/06/2023,2773.0,0.2753254175186157 +I0006,179012080,11/02/2012,sub-I0006179012080,1.0,62.0,Male,White,Not Hispanic,41.97,,True,29/06/2022,3791.0,0.7553417086601257 +I0006,179012153,12/10/2015,sub-I0006179012153,1.0,69.0,Male,White,Not Hispanic,32.44,,True,19/10/2022,2564.0,0.6186583042144775 +I0006,179012255,27/01/2013,sub-I0006179012255,1.0,73.0,Male,White,Not Hispanic,41.54,,True,07/09/2020,2780.0,0.8519553542137146 +I0006,179012526,01/12/2015,sub-I0006179012526,1.0,67.0,Male,White,Not Hispanic,,,True,26/04/2023,2703.0,0.9195594787597656 +I0006,179013071,07/07/2017,sub-I0006179013071,1.0,79.0,Male,White,Not Hispanic,25.45,1114.0,True,26/08/2021,1511.0,0.4199504256248474 +I0006,179013154,18/10/2015,sub-I0006179013154,1.0,69.0,Female,White,Not Hispanic,22.75,,True,21/11/2022,2591.0,0.6564435362815857 +I0006,179014393,13/07/2015,sub-I0006179014393,1.0,50.0,Male,White,Not Hispanic,35.92,,True,17/05/2023,2865.0,0.2891353964805603 +I0006,179014446,24/06/2016,sub-I0006179014446,1.0,76.0,Female,White,Not Hispanic,24.42,1710.0,True,19/03/2023,2459.0,0.5227447152137756 +I0006,179014825,16/06/2015,sub-I0006179014825,1.0,75.0,Male,Unavailable,Unavailable,,1537.0,True,14/06/2020,1825.0,0.27257463335990906 +I0006,179015621,03/05/2014,sub-I0006179015621,1.0,77.0,Male,Black,Not Hispanic,36.76,,True,19/07/2023,3364.0,0.8424844145774841 +I0006,179015933,17/10/2016,sub-I0006179015933,1.0,70.0,Female,White,Not Hispanic,22.28,1542.0,True,09/07/2021,1726.0,0.40108874440193176 +I0006,179016086,23/03/2017,sub-I0006179016086,1.0,72.0,Male,White,Not Hispanic,33.8,1192.0,True,27/01/2023,2136.0,0.5197765827178955 +I0006,179016109,26/03/2018,sub-I0006179016109,1.0,51.0,Female,Black,Not Hispanic,26.82,1280.0,True,03/01/2022,1379.0,0.509002149105072 diff --git a/scripts/copy_new_files.ps1 b/scripts/copy_new_files.ps1 new file mode 100644 index 0000000..3e8052c --- /dev/null +++ b/scripts/copy_new_files.ps1 @@ -0,0 +1,69 @@ +# Script para copiar archivos nuevos de prev_data a test_set +# Copia archivos que existen en X:\bsicos01\__comun\Physionet\prev_data\training_set +# pero que NO existen en C:\BSICoS\CincChallenge2026\python-example-2026\data\training_set +# Los archivos nuevos se copian a data\test_set + +# Definir rutas +$sourceDir = "X:\bsicos01\__comun\Physionet\prev_data\training_set" +$trainingSetDir = "C:\BSICoS\CincChallenge2026\python-example-2026\data\training_set" +$testSetDir = "C:\BSICoS\CincChallenge2026\python-example-2026\data\test_set" + +# Validar que existan los directorios de origen y destino +if (-not (Test-Path $sourceDir)) { + Write-Host "ERROR: Directorio fuente no encontrado: $sourceDir" -ForegroundColor Red + exit 1 +} + +if (-not (Test-Path $trainingSetDir)) { + Write-Host "ERROR: Directorio training_set no encontrado: $trainingSetDir" -ForegroundColor Red + exit 1 +} + +# Crear directorio test_set si no existe +if (-not (Test-Path $testSetDir)) { + Write-Host "Creando directorio: $testSetDir" -ForegroundColor Cyan + New-Item -ItemType Directory -Path $testSetDir -Force | Out-Null +} + +# Obtener archivos recursivamente de ambos directorios +Write-Host "Leyendo archivos..." -ForegroundColor Cyan + +# Obtener nombres relativos de archivos en training_set actual +$trainingFiles = @{} +Get-ChildItem -Path $trainingSetDir -Recurse -File | ForEach-Object { + $relativePath = $_.FullName.Substring($trainingSetDir.Length + 1) + $trainingFiles[$relativePath] = $true +} + +Write-Host "Se encontraron $($trainingFiles.Count) archivos en training_set actual" -ForegroundColor Yellow + +# Procesar archivos de la fuente +$newFilesCount = 0 +$sourceFiles = Get-ChildItem -Path $sourceDir -Recurse -File + +Write-Host "Procesando $($sourceFiles.Count) archivos de la fuente..." -ForegroundColor Cyan + +foreach ($file in $sourceFiles) { + $relativePath = $file.FullName.Substring($sourceDir.Length + 1) + + # Si el archivo NO existe en training_set, copiar a test_set + if (-not $trainingFiles.ContainsKey($relativePath)) { + $destFile = Join-Path $testSetDir $relativePath + $destDir = Split-Path -Parent $destFile + + # Crear directorio de destino si no existe + if (-not (Test-Path $destDir)) { + New-Item -ItemType Directory -Path $destDir -Force | Out-Null + } + + Write-Host "Copiando: $relativePath" -ForegroundColor Green + Copy-Item -Path $file.FullName -Destination $destFile -Force + $newFilesCount++ + } +} + +Write-Host "" +Write-Host "RESUMEN:" -ForegroundColor Yellow +Write-Host "Archivos nuevos copiados: $newFilesCount" -ForegroundColor Green +Write-Host "Ubicación: $testSetDir" -ForegroundColor Cyan +Write-Host "Script completado exitosamente" -ForegroundColor Green diff --git a/src/pipeline/preprocessing.py b/src/pipeline/preprocessing.py index cf94978..1d37657 100644 --- a/src/pipeline/preprocessing.py +++ b/src/pipeline/preprocessing.py @@ -1,19 +1,22 @@ import numpy as np from sklearn.impute import KNNImputer from sklearn.preprocessing import StandardScaler +from sklearn.decomposition import PCA from .config import FEATURE_CORRELATION_THRESHOLD DEFAULT_KNN_NEIGHBORS = 5 +PCA_VARIANCE_THRESHOLD = 0.7 -def build_preprocessor(num_samples, categorical_indices=None): +def build_preprocessor(num_samples, categorical_indices=None, apply_pca=False): neighbors = min(DEFAULT_KNN_NEIGHBORS, max(1, num_samples - 1)) if num_samples > 1 else 1 return CorrelationAwarePreprocessor( n_neighbors=neighbors, categorical_indices=categorical_indices, correlation_threshold=FEATURE_CORRELATION_THRESHOLD, + apply_pca=apply_pca, ) @@ -91,8 +94,67 @@ def get_feature_names_out(self, input_features=None): return np.asarray([input_features[index] for index in self.selected_indices_], dtype=object) +class PCAReducer: + """Applies PCA for dimensionality reduction while preserving a specified variance threshold.""" + + def __init__(self, variance_threshold=PCA_VARIANCE_THRESHOLD): + self.variance_threshold = float(variance_threshold) + self.pca = None + self.n_components_used = None + + def fit(self, X): + """Fit PCA to the data, determining optimal number of components.""" + X = np.asarray(X, dtype=np.float32) + + if X.shape[0] < 2 or X.shape[1] < 2: + self.pca = None + self.n_components_used = X.shape[1] + return self + + # Fit PCA with all possible components initially + n_components = min(X.shape[0] - 1, X.shape[1]) + temp_pca = PCA(n_components=n_components, random_state=42) + temp_pca.fit(X) + + # Calculate cumulative variance explained + cumsum_var = np.cumsum(temp_pca.explained_variance_ratio_) + + # Find number of components needed for target variance + n_comp = np.argmax(cumsum_var >= self.variance_threshold) + 1 + n_comp = max(1, min(n_comp, n_components)) + + # Fit final PCA with optimal components + self.pca = PCA(n_components=n_comp, random_state=42) + self.pca.fit(X) + self.n_components_used = n_comp + + return self + + def transform(self, X): + """Transform data using fitted PCA.""" + X = np.asarray(X, dtype=np.float32) + + if self.pca is None: + return X + + return np.asarray(self.pca.transform(X), dtype=np.float32) + + def fit_transform(self, X): + """Fit PCA and transform data.""" + self.fit(X) + return self.transform(X) + + def get_feature_names_out(self, input_features=None): + """Generate names for PCA components.""" + if self.pca is None: + return input_features if input_features is not None else np.array(['feature_0']) + + n_features = self.pca.n_components_ + return np.asarray([f'PCA_{i}' for i in range(n_features)], dtype=object) + + class CorrelationAwarePreprocessor: - def __init__(self, n_neighbors, categorical_indices, correlation_threshold): + def __init__(self, n_neighbors, categorical_indices, correlation_threshold, apply_pca=True): self.n_neighbors = n_neighbors if categorical_indices is None: self.categorical_indices = np.array([], dtype=np.int32) @@ -101,6 +163,7 @@ def __init__(self, n_neighbors, categorical_indices, correlation_threshold): self.imputer = KNNImputer(n_neighbors=n_neighbors, keep_empty_features=True) self.scaler = StandardScaler() self.selector = CorrelationThresholdSelector(correlation_threshold) + self.pca = PCAReducer() if apply_pca else None self._numerical_indices = np.array([], dtype=np.int32) def _get_numerical_indices(self, n_features): @@ -129,14 +192,30 @@ def fit_transform(self, X): self._numerical_indices = self._get_numerical_indices(X.shape[1]) X_out = self._scale_numerical_columns(X_imputed, fit=True) self.selector.fit(X_out) - return np.asarray(self.selector.transform(X_out), dtype=np.float32) + X_selected = np.asarray(self.selector.transform(X_out), dtype=np.float32) + + # Apply PCA if enabled + if self.pca is not None: + X_final = self.pca.fit_transform(X_selected) + else: + X_final = X_selected + + return np.asarray(X_final, dtype=np.float32) def transform(self, X): X = np.asarray(X, dtype=np.float32).copy() X[~np.isfinite(X)] = np.nan X_imputed = np.asarray(self.imputer.transform(X), dtype=np.float32) X_out = self._scale_numerical_columns(X_imputed, fit=False) - return np.asarray(self.selector.transform(X_out), dtype=np.float32) + X_selected = np.asarray(self.selector.transform(X_out), dtype=np.float32) + + # Apply PCA if enabled + if self.pca is not None: + X_final = self.pca.transform(X_selected) + else: + X_final = X_selected + + return np.asarray(X_final, dtype=np.float32) def transform_feature_indices(self, feature_indices): if self.selector.selected_indices_ is None: @@ -157,4 +236,10 @@ def transform_feature_indices(self, feature_indices): return remapped def get_feature_names_out(self, input_features=None): - return self.selector.get_feature_names_out(input_features) \ No newline at end of file + selector_names = self.selector.get_feature_names_out(input_features) + + # If PCA is applied, return PCA component names + if self.pca is not None: + return self.pca.get_feature_names_out(selector_names) + + return selector_names \ No newline at end of file diff --git a/src/pipeline/training.py b/src/pipeline/training.py index b0087ef..9fb1630 100644 --- a/src/pipeline/training.py +++ b/src/pipeline/training.py @@ -19,7 +19,7 @@ ) from .cross_validation import CrossValidationConfig, EnsembleCrossValidator, normalize_site_group from .features import get_feature_group_indices, get_feature_names, get_or_create_record_feature_vector -from .preprocessing import build_preprocessor, get_processed_feature_names, remap_feature_indices +from .preprocessing import build_preprocessor, get_processed_feature_names, remap_feature_indices, PCA_VARIANCE_THRESHOLD DEFAULT_ENSEMBLE_THRESHOLD = 0.5 @@ -36,6 +36,11 @@ } +def build_preprocessor_for_cv(num_samples, categorical_indices=None): + """Build preprocessor without PCA for cross-validation.""" + return build_preprocessor(num_samples, categorical_indices, apply_pca=False) + + def build_training_metadata_cache(patient_data_file): metadata = pd.read_csv(patient_data_file) demographics_cache = {} @@ -233,7 +238,12 @@ def predict_ensemble_probabilities(model_bundle, feature_matrix): feature_matrix, preprocessor=model_bundle.get('preprocessor'), ) - + + # Debug: Print feature dimensions + preprocessor = model_bundle.get('preprocessor') + if preprocessor and hasattr(preprocessor, 'pca') and preprocessor.pca is not None: + print(f" [DEBUG] Using preprocessor with PCA: {raw_feature_matrix.shape[1]} → {processed_feature_matrix.shape[1]} features") + models = model_bundle['models'] feature_indices = { name: np.asarray(indices, dtype=np.int32) @@ -342,7 +352,7 @@ def train_multimodal_ensemble(data_folder, verbose, csv_path, export_folder=None config=cv_config, param_dist=PARAM_DIST, default_threshold=DEFAULT_ENSEMBLE_THRESHOLD, - build_preprocessor=build_preprocessor, + build_preprocessor=build_preprocessor_for_cv, build_search_model=_build_search_model, fit_ensemble=_fit_ensemble, predict_probabilities=predict_ensemble_probabilities, @@ -370,14 +380,29 @@ def train_multimodal_ensemble(data_folder, verbose, csv_path, export_folder=None # --- Step 2: Fit final models on ALL data using consensus hyperparameters --- print("Fitting final ensemble on all training data with consensus hyperparameters...") - preprocessor = build_preprocessor(len(labels), categorical_indices if categorical_indices else None) + preprocessor = build_preprocessor(len(labels), categorical_indices if categorical_indices else None, apply_pca=True) processed_features = np.asarray(preprocessor.fit_transform(features), dtype=np.float32) - selected_feature_indices = remap_feature_indices(preprocessor, feature_indices) processed_feature_names = get_processed_feature_names(feature_names, preprocessor=preprocessor) selected_raw_feature_indices = np.asarray(preprocessor.selector.selected_indices_, dtype=np.int32) print( - f"Correlation selector: kept {len(processed_feature_names)}/{len(feature_names)} features" + f"Correlation selector: kept {len(preprocessor.selector.selected_indices_)}/{len(feature_names)} features" ) + if preprocessor.pca is not None: + print( + f"PCA: reduced {len(preprocessor.selector.selected_indices_)} features to {preprocessor.pca.n_components_used} components " + f"(explaining {PCA_VARIANCE_THRESHOLD*100:.1f}% of variance)" + ) + # When PCA is applied, all components are used for all modalities since PCA creates new synthetic features + n_pca_components = preprocessor.pca.n_components_used + selected_feature_indices = { + 'all': np.arange(n_pca_components, dtype=np.int32), + 'resp': np.arange(n_pca_components, dtype=np.int32), + 'eeg': np.arange(n_pca_components, dtype=np.int32), + 'ecg': np.arange(n_pca_components, dtype=np.int32), + } + else: + selected_feature_indices = remap_feature_indices(preprocessor, feature_indices) + models = _fit_ensemble(processed_features, labels, selected_feature_indices, consensus_params=consensus) export_root = export_folder or os.path.join(os.getcwd(), 'feature_exports') @@ -398,6 +423,13 @@ def train_multimodal_ensemble(data_folder, verbose, csv_path, export_folder=None modality_presence_indices, ) feature_exports['selected'] = selected_features_csv + + print(f" [INFO] Final model information:") + print(f" - PCA enabled: {preprocessor.pca is not None}") + if preprocessor.pca is not None: + print(f" - PCA components: {preprocessor.pca.n_components_used}") + print(f" - Feature indices keys: {list(selected_feature_indices.keys())}") + print(f" - All modality features: {selected_feature_indices['all']}") return { 'type': 'multimodal_xgb_ensemble', @@ -416,6 +448,9 @@ def train_multimodal_ensemble(data_folder, verbose, csv_path, export_folder=None }, 'models': models, 'preprocessor': preprocessor, + 'pca_enabled': preprocessor.pca is not None, + 'pca_n_components': preprocessor.pca.n_components_used if preprocessor.pca is not None else None, + 'pca_variance_threshold': 0.95, 'feature_exports': feature_exports, 'cv_metrics': cv_metrics, } diff --git a/team_code.py b/team_code.py index 86386ee..39abf01 100644 --- a/team_code.py +++ b/team_code.py @@ -1,5 +1,6 @@ #!/usr/bin/env python - +# -d data/test_set -m model -o outputs -v +# -d data/training_set -m model -v # Edit this script to add your team's code. Some functions are *required*, but you can edit most parts of the required functions, # change or remove non-required functions, and add your own functions. diff --git a/train_output.txt b/train_output.txt new file mode 100644 index 0000000000000000000000000000000000000000..e5cb2aa574df69583965b4033bed03c9dd4fd219 GIT binary patch literal 138354 zcmeI*?QUF0at7f3ZNT?1fZ=!-Ye|%-Ps7f_Q7o*1#C9wvK%iX+vP4Tn^aYZ#SBR5) z%?0EZlGJlbM>8CfmON`?rwYMYo4opIc*pJRJ|Ft0;Hh*wWn?tJnt z&&SK@d-J@H;#~LsY`W_4`0se$i*ND%#rV4!uMft5jr(!UUcAcpUGcm4^!UAwUbpY= zShnc=bk<^=ixw{8FQvK=_d71l-EkP-a1`g9i>ntacks^l@y@03x8rZ7R{a?NX^+0k z`z%)aNz4B8ukz3@i&qb=(0khXFtl#FrL4Tv-G&{`fU6;uHB1!9L3eNyxyu?qwi0hu$kFXZ^gCl@*?iM z7n*iokDc-Upv{vH@9vBX<7(*rQoOzs|Lw$ejvtTTPjdf$y4(G!y$|Bt!|0E{ej8{+ zE1pjh)8NyP{F7+ii}7vP^5>HbJ0aD@apl$Pi{o;&aTSEBuwLV}w8e`#Ec2TQBk z{_D~H8}X>x{u^QY7Yq0Qjj~yZH(~~> z+y56Q4O}m4KYag{cvQ82?Y~xb{@RVX_iA|0s`zJjvYl&X{lB^EN}#u@{jcpe!UNUC z|JS4cMPN1k&!xZ)W`L^pqy5*z167@Wwf|DIziRxBVc7{ERMGw~M!U_t=CI3!$M2P} zpNjp@>t71*um9`%pUW|VE60zQqxE(9Lrmzk&_G@O%KWuc*#9JZSa<%q5%yyysEgl3 z0G08N*{dr50KY+Rb@2o5S&e_z}Z_ zzh2mX;?AqppRA5w=u<6L)%jQT6&9{?`{iKjwjcVb>whi>_Qasn?f+|26j{0bmHj{8 zS&hG4_5WAG_N(!?$J>v8t2=*P3F}dTSatkn^-SIP{c7}nW&f}3*9zlj>>C|awSV>V z)#_JQ{U`3{a^e0r_g*S&|5|SaKvnx!e=Y}HxBW&#UH>EhTdjU$>3?9)-N^&hv>)xS zj6Zpv-59-9`OmriRsE0b9sE|Seq(j~YQGAgs^gdMtX99b+W)&Tc4?q)|6dIcTG#$B zNB>t_|FUX7us;s8s{L3&Qmy`Kwg0tWB~ab|w|cEA|8MT)J*&1Ka|?gg#c#$g4!WlO zX9Zu~`B(HJgI(AD#V#V4y7t3=*5&`Lf2p>9Z8`t)oMzy<_QU^ItKUA}|JJ|P#h<)q z<@#ayYZ;81{*&cnPpa1cVBdJps`w3hQ-51Ee&_#5P#1q@_N#9D`F<>VP5;ApT0veH zKf^zwfU5m3cCQ>iT7QQ_uiF3k^J@EVtm04k)4Kd2{sDVf6~85ZsJ8#i(tgZb>L2RP zUsi8kE{mVp4-ZtAKM}i_1FHJ}Tzy>?|A-&qx4QOY{ZwuK%?PNAfB62&?Z-e==0D8< zmDgX3A60PF^#56Z1rOD1KhKYUu8N=azx+{M{D6I^>p%6s9b~KGf0b|8|GM?h`2X7L zcle;n{J9Kp)%*+Q)1zws#rReKP}l#n{<`&tnft9$|8_g(DUY1B9I>DLe~t0G-u~SA z!RW2wKTozls|T#1s(_zQYd_ypg}+*F|89&0v2WG>$9tN+?BZC_eym+k`%ne{oV1@g zdar>W*4v+zpBR)H{O!s1XZFz!d=>3KYfsE#RqaRQE_$rl|2!ujTBUwt-Trg!g6e^q z_($a!JZc0~9KYI+{i%|_Ubmmjom&4`6aTCI=Y6W||F+)$yeIpwiJy&Kk#AN1qy2WC zQU6ebKRIqc`rZ6x_sNR*LHo0MxXS)NC;MOR1|L)-|FzzJ)}wZyO8wHw_UnK7yNdV+ z>Nj^8#TD_hR+B*QSHy4H&-RUin&VgAo%g{3)x*!YD`Oy6FxPCq**E`cw%_Uz5>(`W zWUi9|tx~_bK7Zjpvwol=f1AJknBKES{lj|y=h_igRI~jm|JhGX{w+O^`k$KlJMD)a zh<&$cHAQMdi_zc6$?{3docbHUFQ zUtPyP$vvoHs@nh1kL*Ro_$z-oE92ja{jdEH1pHqSKVmM-Z`13mZmmZ(La_1Qlj2N=w_=TQjG1SpiY`@mqJ*Ue0uXXzo zx##YUHSGs;sP?d`|IzwfJykJ&lr_re7lN)gexV*SSn8l+`{gdoVk$suwqFd)i&n%B zSO?s*QBc$Vvw|Zts1@x8@0R$#YX4g~m2+^#{)au)(AUgALOvNk)?EL>a^?(bMgBq7 zJu9HaVHNG429klR$Y1GyGnd)7B7c(Y$G}ycfAg0|#4i==53yEqVCJxz_Rss{{;T}{ z$np5y=*9i8{+i<#dQ*K>bNy!WK5B5yVio5v(YyB7;u^wv}>e)}u zUdAvPDz;zbf_80nY;9UMf|D#*6vtAwaNTt)_#BdINp2pxHJ~8A3w&cL?##F z?)HaVj63lyJL9A2`0>i?592c&4jguV!t)1XZ`>cx#=&?nE{wzBjCaR7dj` z`ZS(DiBDdPZ^yZDJg)33m(_ZDRn_0mA1~XhFLH^Y&4j(=qAm8g2|vHh`cijc%x)5Q zY_nW+g;CgqkJ@&*M2nm7LOsNUgOJN zJGt*~FS#&bV#{7}3Yq2|UOVXv>zcJ+y?6kOvTVv`>!G&SA3}#&r`?O+AQvRHS^dd2 z>k9`Ux7SNwnQu}v(Q97L6+m!QCwz#nv9`v%+{w?Ai;p+{JK+s16eQ0IH_7L2t3OP+ zvd(RjI^pe>%ly@gKO~Keymf`0{5))!wKkj8+i$NOW|k;-BJ2~Vn7c5CD(rgA%eihP z*S_}B7fv*HmFy%JB(KtWvpUc1^&2?sWTZOfE3>LER~Pp38~mVpfL`@BssLwxqZdBR z>Oj)*7sF%Jl*)r$7##gdYDjj;s1s_^_g!k=ahq7O} zyHuz8sCl&vG~Z3GT!|~T>|}>f)LensNiLkI+WB5}Unvou&0jBl@rIdw=pvUI^Nf^n zzJ2tSa^V4T4(p{ab$YpHM<==H%qrGSIEqZ-FRV?c{Bp`=W>!7l2XCZYW@eb94?ZLp zPbgZ(wRQ3v;!621nY~_e8DFXb`oxuKhq>RZ3|J?9Fn?6H$SgbeeSNqXT8{vv1QUgIlQjLJRq z$$wcfnhZy;JWyeTl#D8=ZJj41n2-#(LqX zto!O!FKT=lT_Q`qqtp1p8|Es+PVsW0DEm6fs&vwqoC7XEMO!Di$fbhAv7203 zQ`xJo5{u0ns_^aQH|*z_$hQ}cQhn%`5IXsJ_6KiRgV-&vC$p^LMy%2WN12ahD`Ada z`?P2D#V2;+4>RAu1~UrlV~4pyim&f9f8`Elc+sqq>Z32%FstgqUs>VMZQoQGJu8W@0e#{WRbT3d)Qfh?JIvKK z{JeRrkKf2yS&Y)F?#rwULt*Fp{~d7o{V+ii7Qp;<^0vjZ{V=udHbdG ziN6x}Cg$i=kIfE~K{U#G@$+=X&v)`0S-AoIX7;=bj!HYstezD`edNMwBBiZ{NpnU^hsW$fJu~fS!7Yh`t@bsPd4K=XIv$4oNcq4g@ ztkdho1E`=k!g}F(sL=eC+1M_2XkO0W@X5M>E^_{1|^Zb}vM)!Pkp%CwgyZd^8Nr1*^@FiD?vH2VU_2NX z#^G?ryW^eleB2vHV?RE98qc4^Cojgg)+ z{`lHXxsqqgtjcElRBWqUst#Eol(-3xy{&T1y5g@k$uDoKT-K)W@|*He+bUN^Ub)J0 z6CQhex84sT-Hgc7u_WO+RpKnRigQO9Gm4GwpA|qFW!*<*o4R4Ub%ArLo7!pJCx6(FZ@QI z`3s|9w=diVq}a*N<9b!+2a?6|UW45hwnYF}}>TdeXU!pGTp;~~=_N&}heUV0{ zLsdX0+$%;&&c-+Ts zhNSA`H?oqAx8JNDU|Z)ec_M3Way3U6J2ZDCFDk3j2S;VK^85=Aop_tv#VqTVJI!CP zeERs!>e;r{Z}9REy*R8>T$wxfCVSY4@6E`I=jg?Y@`$iuVyQm!mmO0@m3V+Y_%PRh znVr*r^wC$Y)Ps6A+b3{a{e0Ga$vUXy=%cUv1y!u5jAb9W?6+(!y@= zGX65kHd{}$oqi+v4NNw#(I-yH>LEK~nU(wCUbPK&#qx~x?c@(>3zp(R`p89FY|!}c zGrll~{Kd@D_t97WGEuToo$!XSg$ZDVo$OGx4NfYV#6EVY8X$9fJboX!a*Z!9->c3X z(y)_**{jp|%8VWi1uOQk!+Ez96*ryaVtr)GUn}mTFIggUmppT)@s$w@A5Uw2^p(4q z(N-sYnYVaES<6oGmw0l{6Twnl>@fKaXkO-_Q(P(bqBGj+lmALy!w9nrOecM1_RARS z<>##m$ltFt#`^e;#PZg|LWo`DnzOL*;b!~JZ6}XyR^|&u&DhaC`kGfr^{O+^T`Fk| z-`0s|6QShVFVzlx>`=Dc`mb*5+El=*-oQt7T1RhG<=&2Jr25!lt_mPsuRIZHa-Su& zUw!mtR3#%NV(B6m9w681<<25q%VYpD!wqI zedgulY{XD_(LOv|vZCf>^K+kio2(Y4Ewghcx$G)pgc)aj=4D=7y@e=#4LPSf4zRXqjYmw%P9mY%7nQ zzfY9wzdG@v_(OO@zNn8s%zWjHe*56Vtipz*WGp+?AI_OCd7?i0q9a*Yb=gK)pZQBf z32Q)8op>9(C_JAPqkZ&6TYQFb)+w$OO{yl!-2(gQOa3+YGV8P-ta;h&rEVGS=%TNz zKg34mFBEl=E4>BJp_-@@?#)`Z+-m(*asBRh7Pz}&2{(_D3#*|n1;fz^ADWr*hve&oH{im|!gk76TAP9s zopVK9=H=W?1e)g;`{bA5d3hpZjBH)>1x00@9yY3zT)YJo1^4!uzvwG>f`Ots;X_DX zT$#H%bQxdCN2vsW4?FQ2S#gt$RHu9jJ`~3RPi?|un<3sGKaTfKJg&#Pi||O|gbVR3 zv#}U=!ZvosN7M1+mDeA}XDr5QlJn7?tK(qojr-$SJU@u*4~H|}9q)|iW|lc`?2n=f>^nZn>X`@9Zg;ezek@KkTG0s~qSa^4Mfv-c~yl|0ido7WU`+`R$a8 zH-tCL&%NvrhEm(hOKrk$Y^%OzW>h8YCOB$a<>D`lF?@EP@s+4IcQjWi)x~d!mt`DO zaP;!?Y5_=xkJ78+TJ+>AHMerAF_zI#fQ0T%lYWzOMekQG;f`h zYY_iJC^I6>a`RhA+;p_YTVZ4l% z?Zw?7Z})f-{~b;5_#XXo5dH9I`VOf4Uf}V^@&1F?_qa8kwd_4tIpcoZXFtC2X?*j3 z+{^dt9e39ny?hwQ2l4+R{_aiRLrY|w+q#JN$+d{<9>rPiCVDuxcDMW(JcOmAgER&-8vuux#mkKP@|qc4j{> zLL&Fwk3LI%&HXXU^ynx1UxcKONG%dNj8=Kf*1!&D{jBzwnCwbOb1weY?c({=o}*Jf znQdy-Vpz2HAHDw0v_g>^Ss%yuFSWd?d1oxfzr{Jv;|%eiTIUP#C(6;<)iJlUNcaD{ z_)IS^R;)6|;lIcD$e8djzWK@2y1jVM7~P4Ux$$Z|Y1_kbcWU30=zZGHvo6M2J8Ngr z(eu#Se2zO^e09!~$%5`qwz(hYJ&3#h5O?2?`#*{{&UY2Teh@9X7oOqnWJBbB^y>57 zS9hb)$05P8?RTSfFXK!tUX|q9_>b3pewLo~=Qis3^8f4k7uECbmg;%uSJ(5kwUNr- z93Rid>Xo&#e*N)$`Bxdw#?3a4XY$iV7J!7Y5Uu^Dwljoj2^I4vIzS}SBxvy-g zrnRezrhnNP`Y#*NH%}n7Ma=zYW9ucFIm_;rcKJ3|Jom*L4)U{lI##>8InLBbN9wVw+DUoGR8$6zY?& z;W-UguR3O2YVYz}Y0p^6(Q?vr`rU%FwI_#=dS8w`>xO=M?71p6bW(fTlH+ny?N!Ht zOYMnU-t0MzpLupW?m4YJ&VD|9-pk$cj*mT_FYk8N*i$iew&(mgWA7hhUc}n1qLw$y zD?fZauK8WyYHS?5Btkj8540A3IDWQThqUUOffC?|PviMtU(epMwd>cYPhan=)6Sdk z4S{?Xcl|uL?au?Je6e!(lkGR>xO-mn>G+3uERn*=^Q|_V@xzyK_giu8@xEIG3${1m zLHOe=-t=@NWi?s$&vA>=%QMD0Z7$ZTymHka=p}bM+oI;HPV)yd4tpDaa8^&;i+-3H z_|y60U%nsKXSK}Wy{RAas^k5T{Hn~5tIz5OpR923% z+86Qu>=)LrI88#=k$?S~^$e6YFY|Y)o0qR5G5Nc+pgH%(?G)5#l1O!)uye#JuNd231q%$fGYNO)tXxyT-LD z-yB^l$LZ)=^%0J)Wj~IrMBSfna;>bWqigLrlYJ=i^S|ratE21X@f}?+=k4fvc~wW> zZN7GNEg$UY+MMysEF^QSmD!H2l|y%Qt$D=JwJNr=O;DQuUC&25y54N>=z6Hk(e?bi zqwiKZ=jdAAKWh{nT?=D4x>hx{qiaQ1j;@6V^}lP)#@Qwq-~X;xUGC_5wTzB)RomBL zTR+4*>lPrZc{}o&Ea7Gx{}?$gGc6BuJN~;ljkEjlEDOcI88HvzNP9H%$FNX&BDuw< zF}95KWftk&%6aG4z8wRaxz)Y6OL_<+m(TcP9AyZNnDiER*RDPc2Gtoqga^T9d_VO_ z&W_LHlV_9vIE+uyfAx}_gff`xa<2A;?A)C1?$`&bSzYq2%KrD0HSWcG#)-fG15)ZS A%m4rY literal 0 HcmV?d00001 From 8c0b6586cf0a9e6cd66f5a4e2f364621c1f9f447 Mon Sep 17 00:00:00 2001 From: rolopu1 Date: Tue, 2 Jun 2026 11:37:59 +0200 Subject: [PATCH 90/93] Add new functions for slow wave analysis and related processing --- src/lib/swa/mat_to_dict.py | 24 +++ src/lib/swa/swa_CalculateReference.py | 233 ++++++++++++++++++++++++ src/lib/swa/swa_FindSWChannels.py | 234 ++++++++++++++++++++++++ src/lib/swa/swa_FindSWRef.py | 253 ++++++++++++++++++++++++++ src/lib/swa/swa_FindSWTravelling.py | 201 ++++++++++++++++++++ src/lib/swa/swa_channelNeighbours.py | 121 ++++++++++++ src/lib/swa/swa_cluster_test.py | 80 ++++++++ src/lib/swa/swa_filter_data.py | 66 +++++++ src/lib/swa/swa_getInfoDefaults.py | 140 ++++++++++++++ src/lib/swa/swa_get_peaks.py | 92 ++++++++++ src/lib/swa/swa_saveOutput.py | 88 +++++++++ src/lib/swa/swa_xcorr.py | 154 ++++++++++++++++ 12 files changed, 1686 insertions(+) create mode 100644 src/lib/swa/mat_to_dict.py create mode 100644 src/lib/swa/swa_CalculateReference.py create mode 100644 src/lib/swa/swa_FindSWChannels.py create mode 100644 src/lib/swa/swa_FindSWRef.py create mode 100644 src/lib/swa/swa_FindSWTravelling.py create mode 100644 src/lib/swa/swa_channelNeighbours.py create mode 100644 src/lib/swa/swa_cluster_test.py create mode 100644 src/lib/swa/swa_filter_data.py create mode 100644 src/lib/swa/swa_getInfoDefaults.py create mode 100644 src/lib/swa/swa_get_peaks.py create mode 100644 src/lib/swa/swa_saveOutput.py create mode 100644 src/lib/swa/swa_xcorr.py diff --git a/src/lib/swa/mat_to_dict.py b/src/lib/swa/mat_to_dict.py new file mode 100644 index 0000000..a2a9f98 --- /dev/null +++ b/src/lib/swa/mat_to_dict.py @@ -0,0 +1,24 @@ +import scipy.io as sio +import numpy as np + +def mat_to_dict(mat_element): + """ + Convierte de forma recursiva estructuras de MATLAB (mat-objects) + en diccionarios nativos de Python, limpiando las dimensiones vacías. + """ + if isinstance(mat_element, sio.matlab.mat_struct): + # Es una estructura de MATLAB -> La convertimos a diccionario + return {field: mat_to_dict(getattr(mat_element, field)) for field in mat_element._fieldnames} + + elif isinstance(mat_element, np.ndarray): + # Si es un array de objetos, seguimos buscando estructuras dentro + if mat_element.dtype == object: + return [mat_to_dict(element) for element in mat_element] + # Si es un array normal pero está envuelto en dimensiones extra (ej: [[valor]]) + elif mat_element.ndim <= 2 and mat_element.size == 1: + return mat_element.item() + else: + return mat_element + + return mat_element + diff --git a/src/lib/swa/swa_CalculateReference.py b/src/lib/swa/swa_CalculateReference.py new file mode 100644 index 0000000..e88ef39 --- /dev/null +++ b/src/lib/swa/swa_CalculateReference.py @@ -0,0 +1,233 @@ +import numpy as np +import matplotlib.pyplot as plt +import matplotlib.cm as cm + +def swa_CalculateReference(data, Info, display_plot=False): + """ + Calculates the canonical reference wave(s) for slow wave detection + and applies the bandpass filter. + + Parameters: + ----------- + data : numpy.ndarray + Original EEG data matrix of shape (n_channels, n_samples). + Info : dict + Configuration dictionary. + display_plot : bool, optional + If True, plots electrode selections and a 15-second data sample. + + Returns: + -------- + filtData : numpy.ndarray + Filtered reference wave(s). Shape depends on method (e.g., 1 row for envelope, 4 for diamond). + Info : dict + Updated Info dictionary with chosen parameters and reference electrodes. + """ + # 1. Validaciones de estructura iniciales + if 'Electrodes' not in Info: + raise ValueError("Error: No electrode information found in Info") + if 'Recording' not in Info or 'sRate' not in Info['Recording']: + raise ValueError("Error: No sampling rate information found in Info['Recording']") + if 'Parameters' not in Info: + from swa_getInfoDefaults import swa_getInfoDefaults # Evitar importación cíclica + Info = swa_getInfoDefaults(Info, 'SW', 'envelope') + print("Warning: No parameters specified; using defaults.") + + # Compatibilidad de minúsculas + Info['Parameters']['Ref_Method'] = Info['Parameters']['Ref_Method'].lower() + ref_method = Info['Parameters']['Ref_Method'] + print(f"Calculating: Canonical wave ({ref_method})") + + # 2. Ajuste y proyección 2D de las coordenadas de los electrodos + # MATLAB: Th = pi/180*[Info.Electrodes.theta]; Rd = [Info.Electrodes.radius]; + e_locs = Info['Electrodes'] + # Soporta si e_locs es una lista de objetos o una lista de dicts (desde el JSON) + theta = np.array([getattr(el, 'theta', el.get('theta')) for el in e_locs], dtype=float) + radius = np.array([getattr(el, 'radius', el.get('radius')) for el in e_locs], dtype=float) + + Th = (np.pi / 180.0) * theta + x = radius * np.cos(Th) + y = radius * np.sin(Th) + + # Encajonar las coordenadas en un rango de -0.5 a 0.5 + intrad = min(1.0, max(np.abs(radius))) + intrad = max(intrad, 0.5) + squeezefac = 0.5 / intrad + x = x * squeezefac + y = y * squeezefac + + # Inicializar figura si flag_plot está activo + fig_topo, ax_topo = None, None + if display_plot: + fig_topo, ax_topo = plt.subplots(figsize=(6, 6)) + ax_topo.scatter(y, x, s=30, edgecolors=[0.5, 0.5, 0.5], facecolors=[0.5, 0.5, 0.5], label='All Electrodes') + ax_topo.set_aspect('equal') + ax_topo.axis('off') + + n_samples = data.shape[1] + n_total_ch = len(x) + + # ========================================================================= + # SELECCIÓN POR MÉTODOS + # ========================================================================= + + if ref_method == 'envelope': + # Evitar canales externos/periféricos si está configurado + if Info['Parameters'].get('Ref_UseInside', True): + distances = np.sqrt(x**2 + y**2) + # Guardar máscara (1 fila, n_canales) + Info['Parameters']['Ref_Electrodes'] = distances < 0.35 + working_data = data.iloc[Info['Parameters']['Ref_Electrodes'], :] + else: + Info['Parameters']['Ref_Electrodes'] = np.ones(n_total_ch, dtype=bool) + working_data = data + + # Ordenar muestras para obtener el percentil de negatividad + rData = np.sort(working_data, axis=0) + nCh = max(3, int(np.floor(n_total_ch * 0.025))) + + # Si hay más de 3 canales, saltamos el más negativo para evitar artefactos + if nCh > 3: + nData = np.mean(rData[1:nCh, :], axis=0, keepdims=True) + else: + nData = np.mean(rData[0:nCh, :], axis=0, keepdims=True) + + elif ref_method in ['square', 'diamond']: + distance_from_center = 0.2 + circle_radius = 0.175 + + if ref_method == 'square': + RegionCenters = np.array([ + [-distance_from_center, -distance_from_center, distance_from_center, distance_from_center], + [ distance_from_center, -distance_from_center, -distance_from_center, distance_from_center] + ]) + else: # diamond + RegionCenters = np.array([ + [distance_from_center, 0, 0, -distance_from_center], + [ 0, distance_from_center, -distance_from_center, 0] + ]) + + nData = np.zeros((4, n_samples)) + ref_electrodes_mask = np.zeros((4, n_total_ch), dtype=bool) + + for n in range(4): + distances = np.sqrt((x + RegionCenters[0, n])**2 + (y + RegionCenters[1, n])**2) + ref_electrodes_mask[n, :] = distances < circle_radius + nData[n, :] = np.mean(data[ref_electrodes_mask[n, :], :], axis=0) + + Info['Parameters']['Ref_Electrodes'] = ref_electrodes_mask + + elif ref_method == 'grid': + distance_from_center = 0.225 + circle_radius = 0.10 + + # Replicar meshgrid (-1 a 1) de MATLAB + grid_x, grid_y = np.meshgrid([-distance_from_center, 0, distance_from_center], + [-distance_from_center, 0, distance_from_center]) + centers_x = grid_x.flatten() + centers_y = grid_y.flatten() + + nData = np.zeros((9, n_samples)) + ref_electrodes_mask = np.zeros((9, n_total_ch), dtype=bool) + + for n in range(9): + distances = np.sqrt((x + centers_x[n])**2 + (y + centers_y[n])**2) + ref_electrodes_mask[n, :] = distances < circle_radius + nData[n, :] = np.mean(data[ref_electrodes_mask[n, :], :], axis=0) + + Info['Parameters']['Ref_Electrodes'] = ref_electrodes_mask + + elif ref_method == 'central': + circle_radius = 0.175 + distances = np.sqrt(x**2 + y**2) + + Info['Parameters']['Ref_Electrodes'] = distances < circle_radius + print(f"Information: Central using {np.sum(Info['Parameters']['Ref_Electrodes'])} channels for reference") + nData = np.mean(data[Info['Parameters']['Ref_Electrodes'], :], axis=0, keepdims=True) + + elif ref_method == 'midline': + distance_from_center = 0.25 + circle_radius = 0.125 + RegionCenters = np.array([ + [-distance_from_center, 0, distance_from_center], + [ 0, 0, 0] + ]) + + nData = np.zeros((3, n_samples)) + ref_electrodes_mask = np.zeros((3, n_total_ch), dtype=bool) + + for n in range(3): + distances = np.sqrt((x + RegionCenters[0, n])**2 + (y + RegionCenters[1, n])**2) + ref_electrodes_mask[n, :] = distances < circle_radius + nData[n, :] = np.mean(data[ref_electrodes_mask[n, :], :], axis=0) + + Info['Parameters']['Ref_Electrodes'] = ref_electrodes_mask + + else: + raise ValueError(f"Unrecognised reference method type: {ref_method}") + + # ========================================================================= + # PLOT REGIONES (Si corresponde) + # ========================================================================= + if display_plot and ax_topo is not None: + mask_matrix = np.atleast_2d(Info['Parameters']['Ref_Electrodes']) + no_regions = mask_matrix.shape[0] + colors = cm.get_cmap('tab10', no_regions) + + for n in range(no_regions): + region_idx = mask_matrix[n, :] + ax_topo.scatter(y[region_idx], x[region_idx], s=90, + edgecolors=[0.3, 0.3, 0.3], facecolors=colors(n), + label=f'Region {n+1}') + plt.title(f"Selected Electrodes: {ref_method}") + plt.show() + + # ========================================================================= + # FILTRADO DE LA SEÑAL CANÓNICA + # ========================================================================= + if Info['Parameters'].get('Filter_Apply', True): + if 'Filter_Method' not in Info['Parameters']: + print("Information: No filter parameters given, using defaults.") + Info['Parameters']['Filter_Method'] = 'Chebyshev' + Info['Parameters']['Filter_hPass'] = 0.2 + Info['Parameters']['Filter_lPass'] = 4.0 + Info['Parameters']['Filter_order'] = 2 + + print(f"Calculation: Applying {Info['Parameters']['Filter_Method']} filter for [{Info['Parameters']['Filter_hPass']:.1f}, {Info['Parameters']['Filter_lPass']:.1f}] Hz...") + + # Llamar a la función previamente traducida + from lib.swa.swa_filter_data import swa_filter_data + filtData = swa_filter_data(nData, Info) + print("Done") + else: + filtData = nData + + # ========================================================================= + # PLOT 15 SEGUNDOS DE SEÑAL + # ========================================================================= + if display_plot: + sRate = Info['Recording']['sRate'] + # Tomar un punto aleatorio asegurando espacio para 15 segundos + max_start = max(1, n_samples - int(15 * sRate)) + random_sample = np.random.randint(0, max_start) + sample_range = np.arange(random_sample, random_sample + int(15 * sRate)) + time_range = np.arange(len(sample_range)) / sRate + + plt.figure(figsize=(10, 4)) + no_waves = filtData.shape[0] + colors = cm.get_cmap('tab10', no_waves) + + for n in range(no_waves): + if ref_method == 'envelope': + plt.plot(time_range, filtData[n, sample_range], color=[0.2, 0.2, 0.2], linewidth=2.5, label='Envelope Ref') + else: + # Separar las ondas verticalmente como lo hace MATLAB si hay múltiples regiones + plt.plot(time_range, filtData[n, sample_range] - (n * 60), color=colors(n), linewidth=1.5, label=f'Region {n+1}') + + plt.xlabel('Time (seconds)') + plt.ylabel('Amplitude') + plt.title('15 Second Sample of Calculated Reference Wave(s)') + plt.grid(True, alpha=0.3) + plt.show() + + return filtData, Info \ No newline at end of file diff --git a/src/lib/swa/swa_FindSWChannels.py b/src/lib/swa/swa_FindSWChannels.py new file mode 100644 index 0000000..4b8c29c --- /dev/null +++ b/src/lib/swa/swa_FindSWChannels.py @@ -0,0 +1,234 @@ +import numpy as np +from tqdm import tqdm +from lib.swa.swa_channelNeighbours import swa_channelNeighbours +from lib.swa.swa_filter_data import swa_filter_data +from lib.swa.swa_xcorr import swa_xcorr, swa_xcorr2, swa_xcorr_ultra +from lib.swa.swa_cluster_test import swa_cluster_test + +def swa_FindSWChannels(Data, Info, SW, flag_progress=True): + """ + Finds the slow waves present at each channel given the parameters + already calculated for the reference wave. + Translated from swa-Matlab. + """ + # 1. Check inputs & structures + if len(SW) == 0: + print("Warning: Wave structure is empty") + return Data, Info, SW + + + + # 2. Cluster test parameter check + if getattr(Info['Parameters'], 'Channels_ClusterTest', True): + if not hasattr(Info['Recording'], 'ChannelNeighbours'): + print("Calculating: Channel Neighbours...", end="") + # Asume la existencia de la función externa swa_channelNeighbours + Info['Recording']['ChannelNeighbours'] = swa_channelNeighbours(Info['Electrodes']) + print(" done.") + else: + print("Information: Using channels neighbourhood in 'Info'.") + + # 3. Parameter default settings for Envelope method + if getattr(Info['Parameters'], 'Ref_Method', '') == 'Envelope': + if not hasattr(Info['Parameters'], 'Channels_Threshold'): + print("Warning: No further SW parameters found in Info; using defaults") + Info['Parameters']['Channels_Threshold'] = 0.9 + Info['Parameters']['Channels_WinSize'] = 0.2 + + # 4. Filter data if not already done + if hasattr(Data, 'Filtered') and Data.Filtered is not None: + if Data.Filtered.size == 0: + Data.Filtered = swa_filter_data(Data['Raw'], Info) + else: + print("Calculation: Filtering Data.") + Data['Filtered'] = swa_filter_data(Data['Raw'], Info) + + # Calculate window size in samples + win = int(round(Info['Parameters']['Channels_WinSize'] * Info['Recording']['sRate'])) + + # Dimensiones del set de datos (0: canales, 1: muestras de tiempo) + try: + n_channels = Data['Filtered'].shape[0] + except: + n_channels = 1 + n_samples = Data['Filtered'].shape[1] + + to_delete = set() + + # 5. Switch between detection methods + detection_method = Info['Parameters']['Channels_Detection'].lower() + + # ========================================================================= + # CORRELATION METHOD + # ========================================================================= + if detection_method == 'correlation': + + # Iteración con barra de progreso tqdm + for nSW, sw in enumerate(tqdm(SW, disable=not flag_progress, desc="Finding Slow Waves (Correlation)")): + + # Ajuste de índice de MATLAB (1-based) a Python (0-based) + ref_peak_ind = int(sw['Ref_PeakInd']) - 1 + + # Check search window bounds + if ref_peak_ind - win * 2 < 0 or ref_peak_ind + win * 2 >= n_samples: + to_delete.add(nSW) + continue + + # Extract portion of data around reference peak + shortData = Data['Filtered'][:,ref_peak_ind - win * 2 : ref_peak_ind + win * 2 + 1] + + # Get canonical wave reference data + correlate_type = getattr(Info['Parameters'], 'Channels_Correlate2', 'all') + + if correlate_type == 'mean': + # En Python los índices de regiones deben ser enteros o booleanos + regions = np.array(sw['Ref_Region'], dtype=int) - 1 + refData = np.mean(Data['SWRef'][regions, ref_peak_ind - win : ref_peak_ind + win + 1], axis=0) + elif correlate_type == 'main': + main_region = int(sw['Ref_Region'][0]) - 1 + refData = np.mean(Data['SWRef'][[main_region], ref_peak_ind - win : ref_peak_ind + win + 1], axis=0) + else: + # if n_channels == 1: + refData = Data['SWRef'][0][ref_peak_ind - win : ref_peak_ind + win + 1] + # else: + # refData = np.mean(Data['SWRef'][0,ref_peak_ind - win : ref_peak_ind + win + 1], axis=0) + + # Cross correlate (Asume la existencia de la función externa swa_xcorr) + cc = swa_xcorr_ultra(refData, shortData, win) + + # Find max correlation and its index location + maxCC = np.max(cc, axis=1) + maxID = np.argmax(cc, axis=1) + + # 1. Forzamos a que maxCC sea un vector plano 1D (elimina cualquier dimensión extra como (N, 1)) + maxCC_1d = np.squeeze(np.asarray(maxCC)) + + # 2. Creamos channels_active como un vector booleano puramente 1D de tamaño (n_channels,) + channels_active = maxCC_1d > Info['Parameters']['Channels_Threshold'] + + # Si ningún canal correlaciona bien, se elimina la SW y se continúa + if np.sum(channels_active) == 0: + to_delete.add(nSW) + continue + + # 3. Inicializar matriz de amplitudes negativas con NaNs (canales x 1) + sw['Channels_NegAmp'] = np.full((n_channels, 1), np.nan) + + # 4. Extracción segura de amplitudes para canales activos + if np.any(channels_active): + if n_channels == 1: + sw['Channels_NegAmp'][channels_active, 0] = np.min(shortData) + else: + # shortData es (n_channels, n_samples). channels_active al ser 1D filtra las filas perfectamente. + sw['Channels_NegAmp'][channels_active, 0] = np.min(shortData[channels_active, :], axis=1) + + # 5. Desactivar canales que no superen el umbral de amplitud absoluta mínimo + amp_thresh = np.mean(Info['Parameters']['Ref_AmplitudeAbsolute']) / 10.0 + + # Forzamos que la máscara de desactivación sea 1D para que no altere las dimensiones + deactivate_mask = (sw['Channels_NegAmp'][:, 0] > amp_thresh).flatten() + channels_active[deactivate_mask] = False + + if getattr(Info['Parameters'], 'Channels_ClusterTest', True): + clusters = swa_cluster_test(channels_active.astype(float), Info['Recording']['ChannelNeighbours'], 0.01) + clusters[np.isnan(clusters)] = 0 + n_clusters = np.unique(clusters) + + if len(n_clusters) > 2: # Más allá del fondo (0) y un cluster válido + max_cluster_size = 0 + keep_cluster_val = n_clusters[1] + for val in n_clusters: + if val == 0: + continue + s_cluster = np.sum(clusters == val) + if s_cluster > max_cluster_size: + max_cluster_size = s_cluster + keep_cluster_val = val + channels_active = (clusters == keep_cluster_val) + + # Recalcular con el canal prototípico si la correlación de la referencia no es muy alta + negative_peak_index = np.nanargmin(sw['Channels_NegAmp'][:, 0]) + + if maxCC[negative_peak_index] < (Info['Parameters']['Channels_Threshold'] + 1) / 2: + # maxID es 0-based, compensamos el comportamiento de MATLAB (maxID_matlab - win) + max_delay = int((maxID[negative_peak_index] + 1) - win) + + maxData = Data['Filtered'][ + negative_peak_index, + ref_peak_ind - win + max_delay : ref_peak_ind + win + max_delay + 1 + ] + + cc = swa_xcorr_ultra(maxData, shortData, win) + maxCC = np.max(cc, axis=1) + maxID = np.argmax(cc, axis=1) + + channels_active[maxCC > Info['Parameters']['Channels_Threshold']] = True + + # Limpieza final de Amplitudes Negativas basada en los canales descartados + sw['Channels_NegAmp'][~channels_active, 0] = np.nan + + # Delay Calculation + sw['Travelling_Delays'] = np.full((n_channels, 1), np.nan) + # maxID de Python ya es nativamente relativo a 0 + sw['Travelling_Delays'][channels_active, 0] = maxID[channels_active] - np.min(maxID[channels_active]) + + sw['Channels_Globality'] = (np.sum(channels_active) / n_channels) * 100.0 + sw['Channels_Active'] = channels_active + + # ========================================================================= + # THRESHOLD METHOD + # ========================================================================= + elif detection_method == 'threshold': + if not getattr(Info['Parameters'], 'Ref_Peak2Peak', None): + Info['Parameters']['Ref_Peak2Peak'] = abs(Info['Parameters']['Ref_NegAmpMin']) * 1.75 + + for nSW, sw in enumerate(tqdm(SW, disable=not flag_progress, desc="Finding Slow Waves (Threshold)")): + + ref_peak_ind = int(sw['Ref_PeakInd']) - 1 + + if ref_peak_ind - win < 0 or ref_peak_ind + win * 3 >= n_samples: + to_delete.add(nSW) + continue + + # Short window for negative peak + shortData = Data['Filtered'][:, ref_peak_ind - win : ref_peak_ind + win + 1] + + sw.Channels_NegAmp = np.min(shortData, axis=1) + minChId = np.argmax(shortData, axis=1) # El índice relativo del mínimo dentro de este bloque + + thresh_val = -np.mean(Info['Parameters']['Ref_AmplitudeAbsolute']) * Info['Parameters']['Channels_Threshold'] + sw.Channels_Active = sw.Channels_NegAmp < thresh_val + + # Peak to Peak Check (ventana más larga hacia adelante) + shortData_long = Data['Filtered'][:, ref_peak_ind - win : ref_peak_ind + win * 3 + 1] + + posPeakAmp = np.full(n_channels, np.nan) + active_indices = np.where(sw.Channels_Active)[0] + + for nCh in active_indices: + # Buscamos el pico positivo después del pico negativo detectado + posPeakAmp[nCh] = np.max(shortData_long[nCh, minChId[nCh]:]) + + # Aplicar filtro de umbral pico a pico + p2p_thresh = Info['Parameters']['Ref_Peak2Peak'] * Info['Parameters']['Channels_Threshold'] + sw.Channels_Active[posPeakAmp - sw.Channels_NegAmp < p2p_thresh] = False + + # Eliminar canales sub-umbral de las amplitudes negativas + sw.Channels_NegAmp[~sw.Channels_Active] = np.nan + + if np.sum(sw.Channels_Active) == 0: + to_delete.add(nSW) + continue + + # Delay Calculation usando el identificador del pico negativo + sw.Travelling_Delays = np.full((len(Info.Electrodes), 1), np.nan) + active_delays = minChId[sw.Channels_Active] + sw.Travelling_Delays[sw.Channels_Active, 0] = active_delays - np.min(active_delays) + + sw.Channels_Globality = (np.sum(sw.Channels_Active) / len(sw.Channels_Active)) * 100.0 + + # 6. Delete the bad waves from list + print(f"Information: {len(to_delete)} slow waves were removed due to insufficient criteria.") + SW = [sw for idx, sw in enumerate(SW) if idx not in to_delete] + + return Data, Info, SW \ No newline at end of file diff --git a/src/lib/swa/swa_FindSWRef.py b/src/lib/swa/swa_FindSWRef.py new file mode 100644 index 0000000..683182d --- /dev/null +++ b/src/lib/swa/swa_FindSWRef.py @@ -0,0 +1,253 @@ +import numpy as np +from scipy.signal import find_peaks + +def swa_FindSWRef(Data, Info, SW=None): + """ + Finds slow waves in the reference signal based on amplitude, wavelength, + and slope criteria. Handles multiple reference regions and removes duplicates. + Translated from swa-Matlab. + + Parameters: + ----------- + Data : dict + Dictionary containing 'SWRef' (n_refs x n_samples) and optionally 'sleep_stages'. + Info : dict + Configuration dictionary. + SW : list of dicts, optional + Existing list of detected slow waves. Defaults to empty list. + + Returns: + -------- + Data : dict + Info : dict + SW : list of dicts + """ + if 'SWRef' not in Data: + raise ValueError("Error: Data dictionary must contain 'SWRef' array.") + + p = Info.get('Parameters', {}) + if 'Ref_Method' not in p: + print("Error: No detection parameters found in the 'Info' structure") + return Data, Info, SW + + # Inicializar la lista de ondas si no existe + if SW is None: + SW = [] + + OSWCount = len(SW) + SWCount = len(SW) + + number_ref_waves = Data['SWRef'].shape[0] + + # Inicializar vectores de umbrales según el criterio + if p.get('Ref_AmplitudeCriteria') == 'relative': + p['Ref_AmplitudeAbsolute'] = np.zeros(number_ref_waves) + elif p.get('Ref_AmplitudeCriteria') == 'absolute': + # Asegurar que Ref_AmplitudeRelative exista como lista/array + p['Ref_AmplitudeRelative'] = np.zeros(number_ref_waves) + + if 'Recording' not in Info: + Info['Recording'] = {} + Info['Recording']['Data_Deviation'] = np.zeros(number_ref_waves) + + # Asegurar que el umbral absoluto tenga el tamaño adecuado si es un solo valor + if p.get('Ref_AmplitudeCriteria') == 'absolute': + abs_val = np.atleast_1d(p['Ref_AmplitudeAbsolute']) + if len(abs_val) < number_ref_waves: + p['Ref_AmplitudeAbsolute'] = np.repeat(abs_val[0], number_ref_waves) + + sRate = Info['Recording']['sRate'] + + # Bucle por cada onda de referencia (ej. regiones diamond, central, envelope) + for ref_wave in range(number_ref_waves): + if ref_wave > 0: + OSWCount = len(SW) + + ref_signal = Data['SWRef'][ref_wave, :] + + # Calcular la derivada de la señal (añadiendo un 0 inicial para mantener longitud) + slopeData = np.concatenate(([0], np.diff(ref_signal))) + + # Extraer Mínimos (MNP - Maximum Negative Peaks) y Máximos (MPP) + from lib.swa.swa_get_peaks import swa_get_peaks + MNP, MPP = swa_get_peaks(slopeData, Info, True) + + MNP = MNP[1:] + MPP = MPP[1:] + # Eliminar picos fuera de las fases de sueño de interés + if p.get('Ref_UseStages') is not None and 'sleep_stages' in Data: + stages = Data['sleep_stages'] + valid_MNP_mask = np.isin(stages[MNP], p['Ref_UseStages']) + relevant_MNP = MNP[valid_MNP_mask] + else: + relevant_MNP = MNP + + # Control de seguridad si no hay picos + if len(relevant_MNP) == 0: + continue + + # ===================================================================== + # CÁLCULO DEL UMBRAL DE AMPLITUD (Amplitude Threshold Criteria) + # ===================================================================== + # MAD (Median Absolute Deviation respecto a la mediana) + median_act = np.median(ref_signal[relevant_MNP]) + mad_val = np.median(np.abs(ref_signal[relevant_MNP] - median_act)) + Info['Recording']['Data_Deviation'][ref_wave] = mad_val + + if p['Ref_AmplitudeCriteria'] == 'relative': + p['Ref_AmplitudeAbsolute'][ref_wave] = (mad_val * p['Ref_AmplitudeRelative']) + abs(median_act) + print(f"Calculation: Amplitude threshold set to {p['Ref_AmplitudeAbsolute'][ref_wave]:.1f}uV for canonical wave {ref_wave + 1}") + elif p['Ref_AmplitudeCriteria'] == 'absolute': + deviation = (p['Ref_AmplitudeAbsolute'][ref_wave] - abs(median_act)) / mad_val if mad_val != 0 else 0 + p['Ref_AmplitudeRelative'][ref_wave] = deviation + print(f"Information: Threshold is {deviation:.1f} deviations from median activity") + + if p.get('Ref_Peak2Peak') is None: + p['Ref_Peak2Peak'] = abs(p['Ref_AmplitudeAbsolute'][ref_wave]) * 1.75 + + # ===================================================================== + # MÉTODOS DE INSPECCIÓN (MNP vs ZC) + # ===================================================================== + if p['Ref_InspectionPoint'] == 'MNP': + # ----- MÉTODO MNP (Maximum Negative Peak) ----- + badWaves = np.zeros(len(MNP), dtype=bool) + + if p.get('Ref_UseStages') is not None and 'sleep_stages' in Data: + badWaves[~valid_MNP_mask] = True + + # (El código MNP de MATLAB requiere alinear los MPP con MNP, + # asumiendo que un MNP está rodeado por dos MPP. Esta lógica + # se adapta mejor con Zero-Crossings a menos que los picos estén + # estrictamente pareados). + # Por simplicidad y robustez frente al código MATLAB original: + pass # En Python el método más seguro y estandarizado es ZC. + + elif p['Ref_InspectionPoint'] == 'ZC': + # ----- MÉTODO ZC (Zero Crossings) ----- + signData = np.sign(ref_signal) + signData[signData == 0] = 1 # Evitar pendiente 0 exacta + + DZC = np.where(np.diff(signData) < 0)[0] + UZC = np.where(np.diff(signData) > 0)[0] + 1 # +1 para ser la muestra tras el cruce + + # Umbral de pendiente (percentil) + pos_slopes = slopeData[slopeData > 0] + slopeThresh = np.percentile(pos_slopes, p['Ref_SlopeMin'] * 100) if len(pos_slopes) > 0 else 0 + + # Alinear DZC y UZC + if len(DZC) > 0 and len(UZC) > 0 and DZC[0] >= UZC[0]: + UZC = UZC[1:] + if len(DZC) != len(UZC): + DZC = DZC[:-1] + if len(DZC) > len(UZC): + DZC = DZC[:-1] + + # Filtrar por fase de sueño + if p.get('Ref_UseStages') is not None and 'sleep_stages' in Data: + valid_DZC = np.isin(Data['sleep_stages'][DZC], p['Ref_UseStages']) + DZC = DZC[valid_DZC] + UZC = UZC[valid_DZC] + + # Criterio Wavelength + SWLengths = UZC - DZC + valid_len = (SWLengths >= p['Ref_WaveLength'][0] * sRate) & (SWLengths <= p['Ref_WaveLength'][1] * sRate) + DZC = DZC[valid_len] + UZC = UZC[valid_len] + + AllPeaks = np.array([sw['Ref_PeakInd'] for sw in SW]) if len(SW) > 0 else np.array([]) + + # Analizar cada candidato DZC + for n in range(len(DZC)): + start, end = DZC[n], UZC[n] + segment = ref_signal[start:end] + + if len(segment) == 0: + continue + + # Amplitud Negativa + NegPeakAmp = np.min(segment) + NegPeakId = start + np.argmin(segment) + + if NegPeakAmp > -p['Ref_AmplitudeAbsolute'][ref_wave] or NegPeakAmp < -p.get('Ref_AmplitudeMax', 250): + continue + + # Amplitud Peak2Peak + search_end = min(end + int(1 * sRate), len(ref_signal)) + PosPeakAmp = np.max(ref_signal[end:search_end]) if search_end > end else 0 + + if p['Ref_Method'] in ['diamond', 'square']: + if (PosPeakAmp - NegPeakAmp) < p['Ref_Peak2Peak']: + continue + + # Pendiente positiva máxima + MaxPosSlope = np.max(slopeData[start:end]) + if MaxPosSlope < slopeThresh: + continue + + # Evitar duplicados inter-regiones + if ref_wave > 0 and len(AllPeaks) > 0: + # Comprobar si el pico de una onda ya guardada cae dentro de este nuevo DZC-UZC + overlap_mask = (AllPeaks > start) & (AllPeaks < end) + if np.any(overlap_mask): + SWid = np.argmax(overlap_mask) # Encontrar el índice conflictivo + + # Guardar la región que tenga la mayor amplitud (más negativo) + if ref_signal[NegPeakId] < SW[SWid]['Ref_PeakAmp']: + # Sobrescribir con la nueva onda que es mejor + old_regions = SW[SWid]['Ref_Region'] if isinstance(SW[SWid]['Ref_Region'], list) else [SW[SWid]['Ref_Region']] + SW[SWid]['Ref_Region'] = [ref_wave + 1] + old_regions # +1 para mantener ID visual (1-based index concept) + SW[SWid]['Ref_DownInd'] = start + SW[SWid]['Ref_PeakInd'] = NegPeakId + SW[SWid]['Ref_UpInd'] = end + SW[SWid]['Ref_PeakAmp'] = ref_signal[NegPeakId] + SW[SWid]['Ref_P2PAmp'] = PosPeakAmp - NegPeakAmp + SW[SWid]['Ref_NegSlope'] = np.min(slopeData[start:end]) + SW[SWid]['Ref_PosSlope'] = MaxPosSlope + else: + # Añadir esta región a la onda existente + if isinstance(SW[SWid]['Ref_Region'], list): + SW[SWid]['Ref_Region'].append(ref_wave + 1) + else: + SW[SWid]['Ref_Region'] = [SW[SWid]['Ref_Region'], ref_wave + 1] + continue + + # Añadir nueva onda detectada + SWCount += 1 + new_wave = { + 'Ref_Region': [ref_wave + 1], + 'Ref_DownInd': start, + 'Ref_PeakInd': NegPeakId, + 'Ref_UpInd': end, + 'Ref_PeakAmp': ref_signal[NegPeakId], + 'Ref_P2PAmp': PosPeakAmp - NegPeakAmp, + 'Ref_NegSlope': np.min(slopeData[start:end]), + 'Ref_PosSlope': MaxPosSlope, + # Campos vacíos que se llenarán más adelante + 'Channels_Active': [], 'Channels_NegAmp': [], + 'Channels_Globality': [], 'Travelling_Delays': [], + 'Travelling_DelayMap': [], 'Travelling_Streams': [], 'Code': [] + } + SW.append(new_wave) + + else: + print("Error: Unrecognised detection method") + return Data, Info, SW + + # Ordenar las ondas detectadas cronológicamente + if len(SW) > 0: + SW.sort(key=lambda x: x['Ref_DownInd']) + + # Limpieza final de estadios de sueño no válidos (por si acaso quedaron en las fusiones) + if p.get('Ref_UseStages') is not None and 'sleep_stages' in Data: + valid_indices = [] + for i, wave in enumerate(SW): + if Data['sleep_stages'][wave['Ref_PeakInd']] in p['Ref_UseStages']: + valid_indices.append(i) + + removed_count = len(SW) - len(valid_indices) + SW = [SW[i] for i in valid_indices] + + if removed_count > 0: + print(f"Information: {removed_count} waves were found in non-specified stages and removed") + + return Data, Info, SW \ No newline at end of file diff --git a/src/lib/swa/swa_FindSWTravelling.py b/src/lib/swa/swa_FindSWTravelling.py new file mode 100644 index 0000000..2dca0eb --- /dev/null +++ b/src/lib/swa/swa_FindSWTravelling.py @@ -0,0 +1,201 @@ +import numpy as np +from scipy.interpolate import griddata, RegularGridInterpolator + +def swa_FindSWTravelling(Info, SW, indSW=None, flag_wait=True): + """ + Calculates the travelling parameters for each slow wave given the + delay maps, defining the streamlines of propagation. + Translated from swa-Matlab. + + Parameters: + ----------- + Info : dict + Configuration dictionary. + SW : list of dicts + List containing all detected slow waves. + indSW : list or numpy.ndarray, optional + Indices of the SW list to process. If None, processes all. + flag_wait : bool + If True, prints progress. + + Returns: + -------- + Info : dict + SW : list of dicts + """ + if len(SW) == 0 or ('Ref_Region' not in SW[0] and len(SW) < 2): + print("Warning: Wave structure is empty, you must find waves in the reference first.") + return Info, SW + + p = Info.setdefault('Parameters', {}) + + # Defaults + if 'Travelling_GS' not in p: + p['Travelling_GS'] = 40 + print("Information: Interpolation grid set at 40x40 by default.") + p['Travelling_MinDelay'] = 40.0 + if 'Travelling_RecalculateDelay' not in p: + p['Travelling_RecalculateDelay'] = True + + GS = p['Travelling_GS'] + sRate = Info['Recording']['sRate'] + + # --- Check Electrodes and 2D Locations --- + if p['Travelling_RecalculateDelay']: + # Ensure 2D locations are scaled to grid size [1, GS] + # (Assuming 'x' and 'y' are present in Info['Electrodes']) + xloc = np.array([el.get('x', 0) for el in Info['Electrodes']], dtype=float) + yloc = np.array([el.get('y', 0) for el in Info['Electrodes']], dtype=float) + + # Auto-scale coordinates to grid if they are between -0.5 and 0.5 (from Reference func) + if np.max(np.abs(xloc)) <= 1.0: + xloc = (xloc - np.min(xloc)) / (np.max(xloc) - np.min(xloc) + 1e-9) * (GS - 1) + 1 + yloc = (yloc - np.min(yloc)) / (np.max(yloc) - np.min(yloc) + 1e-9) * (GS - 1) + 1 + # Update dictionary coordinates for consistency + for i, el in enumerate(Info['Electrodes']): + el['x'], el['y'] = xloc[i], yloc[i] + + print("Calculation: 2D electrode projections created.") + + # Grid definition + XYrange = np.linspace(1, GS, GS) + grid_x, grid_y = np.meshgrid(XYrange, XYrange) + else: + if 'Travelling_DelayMap' not in SW[0] or len(SW[0]['Travelling_DelayMap']) == 0: + raise ValueError("User requested no map calculation, but no maps were found.") + XYrange = np.linspace(1, SW[0]['Travelling_DelayMap'].shape[1], SW[0]['Travelling_DelayMap'].shape[1]) + xloc = np.array([el['x'] for el in Info['Electrodes']]) + yloc = np.array([el['y'] for el in Info['Electrodes']]) + + loopRange = indSW if indSW is not None else range(len(SW)) + total_sw = len(loopRange) + + for idx, nSW in enumerate(loopRange): + if p['Travelling_RecalculateDelay']: + Delays = np.array(SW[nSW].get('Travelling_Delays', [])) + + if len(Delays) == 0 or np.max(Delays) < (p['Travelling_MinDelay'] * sRate / 1000.0): + continue + + # Interpolate Delays onto 2D Grid + # method='cubic' acts similarly to 'natural' interpolant in MATLAB + delay_map = griddata((xloc, yloc), Delays, (grid_x, grid_y), method='cubic') + delay_map = np.nan_to_num(delay_map) # Replace NaNs with 0 + SW[nSW]['Travelling_DelayMap'] = delay_map + + # Starting points + active_ch = SW[nSW].get('Channels_Active', []) + if len(active_ch) == 0: + continue + sx = xloc[active_ch] + sy = yloc[active_ch] + + else: + delay_map = SW[nSW]['Travelling_DelayMap'] + active_ch = SW[nSW].get('Channels_Active', []) + sx = xloc[active_ch] + sy = yloc[active_ch] + + # Calculate gradients (MATLAB: [u, v] = gradient(Map)) + # np.gradient returns [row_diff (Y), col_diff (X)] + grad_y, grad_x = np.gradient(delay_map) + u, v = grad_x, grad_y + + Streams = [] + Distances = [] + + for n in range(len(sx)): + # Trace backwards + path_b, dist_b = _trace_streamline(XYrange, XYrange, -u, -v, sx[n], sy[n], step=0.1, max_steps=1000) + # Trace forwards + path_f, dist_f = _trace_streamline(XYrange, XYrange, u, v, sx[n], sy[n], step=0.1, max_steps=1000) + + # Combine paths (Reverse backward path, skip the overlapping start point, append forward) + if path_b.shape[1] > 0 and path_f.shape[1] > 0: + path = np.hstack((path_b[:, ::-1], path_f[:, 1:])) + dist = np.concatenate((dist_b[::-1], dist_f)) + Streams.append(path) + Distances.append(dist) + + if len(Streams) == 0: + continue + + # Filter Streams by minimum distance threshold (25% of longest path) + tDist = np.array([np.sum(d) for d in Distances]) + valid_streams = tDist >= (np.max(tDist) / 4.0) + + Streams = [s for i, s in enumerate(Streams) if valid_streams[i]] + tDist = tDist[valid_streams] + + if len(Streams) == 0: + continue + + SW[nSW]['Travelling_Streams'] = [] + + # 1. Longest Displacement (Straight line distance from start to end of stream) + tDisp = np.array([np.hypot(s[0, 0] - s[0, -1], s[1, 0] - s[1, -1]) for s in Streams]) + maxDispId = np.argmax(tDisp) + SW[nSW]['Travelling_Streams'].append(Streams[maxDispId]) + + # 2. Longest Travelled Distance (Sum of all step distances) + maxDistId = np.argmax(tDist) + if maxDistId != maxDispId: + SW[nSW]['Travelling_Streams'].append(Streams[maxDistId]) + + # 3. Most different displacement angle compared to longest displacement stream (> 45 degrees) + def get_angle(path): + return np.degrees(np.arctan2(path[1, -1] - path[1, 0], path[0, -1] - path[0, 0])) + + base_angle = get_angle(Streams[maxDispId]) + angles = np.array([get_angle(s) for s in Streams]) + angle_diff = np.abs(angles - base_angle) + # Normalize to 0-180 + angle_diff = np.where(angle_diff > 180, 360 - angle_diff, angle_diff) + + maxAngleId = np.argmax(angle_diff) + if angle_diff[maxAngleId] > 45.0 and maxAngleId not in [maxDispId, maxDistId]: + SW[nSW]['Travelling_Streams'].append(Streams[maxAngleId]) + + if flag_wait and (idx + 1) % max(1, int(total_sw / 10)) == 0: + print(f"Processing Slow Wave {idx + 1} of {total_sw}...") + + return Info, SW + + +# --- HELPER FUNCTION: Vector Field Streamline Tracer --- +def _trace_streamline(X, Y, U, V, start_x, start_y, step=0.1, max_steps=1000): + """ + Simulates MATLAB's adstream2b by tracing paths through a 2D vector field. + """ + interp_U = RegularGridInterpolator((Y, X), U, bounds_error=False, fill_value=0) + interp_V = RegularGridInterpolator((Y, X), V, bounds_error=False, fill_value=0) + + path = [[start_x], [start_y]] + dists = [] + + curr_x, curr_y = start_x, start_y + + for _ in range(max_steps): + u_val = interp_U((curr_y, curr_x)) + v_val = interp_V((curr_y, curr_x)) + + norm = np.hypot(u_val, v_val) + if norm < 1e-5: # Vector is practically zero, stop tracing + break + + # Normalize direction and apply step size + dx = step * (u_val / norm) + dy = step * (v_val / norm) + + curr_x += dx + curr_y += dy + + # Check Boundaries + if curr_x < X[0] or curr_x > X[-1] or curr_y < Y[0] or curr_y > Y[-1]: + break + + path[0].append(float(curr_x)) + path[1].append(float(curr_y)) + dists.append(float(np.hypot(dx, dy))) + + return np.array(path), np.array(dists) \ No newline at end of file diff --git a/src/lib/swa/swa_channelNeighbours.py b/src/lib/swa/swa_channelNeighbours.py new file mode 100644 index 0000000..74b204b --- /dev/null +++ b/src/lib/swa/swa_channelNeighbours.py @@ -0,0 +1,121 @@ +import numpy as np +from scipy.spatial import ConvexHull +from scipy.optimize import minimize +import matplotlib.pyplot as plt +from mpl_toolkits.mplot3d.art3d import Poly3DCollection + +def swa_channelNeighbours(eLoc, displayNet=False): + """ + Searches for neighbouring channels using triangulation and reports back + each channel's neighbours. + Translated from swa-Matlab / ept_TFCE. + """ + nCh = len(eLoc) + + # 1. Extraer coordenadas X, Y, Z de forma flexible (soporta objetos o dicts) + x = np.array([getattr(el, 'X', el.get('X') if isinstance(el, dict) else None) for el in eLoc], dtype=float) + y = np.array([getattr(el, 'Y', el.get('Y') if isinstance(el, dict) else None) for el in eLoc], dtype=float) + z = np.array([getattr(el, 'Z', el.get('Z') if isinstance(el, dict) else None) for el in eLoc], dtype=float) + + vertices = np.column_stack((x, y, z)) + + # 2. Proyección plana 2D (Algoritmo simplificado de Brainstorm) + z2 = z - np.max(z) + hypotxy = np.hypot(x, y) + R = np.hypot(hypotxy, z2) + PHI = np.arctan2(z2, hypotxy) + TH = np.arctan2(y, x) + + # Prevenir valores excesivamente pequeños para PHI + PHI[PHI < 0.001] = 0.001 + + # Proyección achatada (Flat projection) + R2 = R / (np.cos(PHI) ** 0.2) + X_2d = R2 * np.cos(TH) + Y_2d = R2 * np.sin(TH) + + # 3. Ajuste de Esfera (Equivalente a bst_bfs / fminsearch) + mass = np.mean(vertices, axis=0) + diffvert = vertices - mass + R0 = np.mean(np.sqrt(np.sum(diffvert**2, axis=1))) + vec0 = np.append(mass, R0) + + # Función de optimización interna para encontrar el centro de la cabeza + def dist_sph(vec, sensloc): + radius = vec[-1] + center = vec[:-1] + diff = sensloc - center + return np.mean(np.abs(np.sqrt(np.sum(diff**2, axis=1)) - radius)) + + res = minimize(dist_sph, vec0, args=(vertices,), method='Nelder-Mead') + HeadCenter = res.x[:-1] + + # Normalización sobre el centro estimado de la cabeza + coordC = vertices - HeadCenter + coordC = coordC / np.sqrt(np.sum(coordC**2, axis=1))[:, np.newaxis] + + # Teselación (Convex Hull 3D) -> Reemplaza a convhulln + hull_3d = ConvexHull(coordC) + faces = hull_3d.simplices # Matriz de Nx3 (índices de los triángulos) + + # 4. Eliminar triángulos innecesarios / externos + # Obtener el borde exterior en 2D + points_2d = np.column_stack((X_2d, Y_2d)) + hull_2d = ConvexHull(points_2d) + border = set(hull_2d.vertices) + + # Conservar caras que NO tengan todos sus 3 vértices en el borde exterior + iInside = ~np.all(np.isin(faces, list(border)), axis=1) + faces = faces[iInside] + + # Calcular perímetros para eliminar triángulos desproporcionados (outliers) + v0 = vertices[faces[:, 0]] + v1 = vertices[faces[:, 1]] + v2 = vertices[faces[:, 2]] + + side0 = np.sqrt(np.sum((v0 - v1)**2, axis=1)) + side1 = np.sqrt(np.sum((v0 - v2)**2, axis=1)) + side2 = np.sqrt(np.sum((v1 - v2)**2, axis=1)) + + triPerimeter = side0 + side1 + side2 + thresholdPerim = np.mean(triPerimeter) + 3 * np.std(triPerimeter) + + # Aplicar umbral de distancia + faces = faces[triPerimeter <= thresholdPerim] + + # 5. Visualización de la Red (Opcional) + if displayNet: + fig = plt.figure(figsize=(7, 7)) + ax = fig.add_subplot(111, projection='3d') + + # Dibujar las caras del mallado + mesh = Poly3DCollection(vertices[faces], alpha=0.6, facecolor=[0.5, 0.5, 0.5], edgecolor='k', linewidths=0.5) + ax.add_collection3d(mesh) + + # Dibujar los electrodos como puntos + ax.scatter(x, y, z, color='r', s=40, depthshade=False) + + ax.set_axis_off() + ax.set_box_aspect([1,1,1]) + plt.show() + + # 6. Buscar vecinos para cada canal + output = [] + for n in range(nCh): + # Encontrar filas donde aparezca el canal actual 'n' + rows_with_n = np.any(faces == n, axis=1) + # Extraer todos los nodos únicos de esos triángulos + neighbours = np.unique(faces[rows_with_n]) + + # OJO: MATLAB guarda el propio nodo dentro de sus vecinos. + # Convertimos a Base-1 para mantener idéntica paridad con MATLAB + output.append(neighbours + 1) + + # Generar matriz final acolchada con ceros (como cell2mat en MATLAB) + max_neighbors = max(len(ch) for ch in output) + ChN = np.zeros((nCh, max_neighbors), dtype=int) + + for idx, neighbours in enumerate(output): + ChN[idx, :len(neighbours)] = neighbours + + return ChN \ No newline at end of file diff --git a/src/lib/swa/swa_cluster_test.py b/src/lib/swa/swa_cluster_test.py new file mode 100644 index 0000000..c4f6a49 --- /dev/null +++ b/src/lib/swa/swa_cluster_test.py @@ -0,0 +1,80 @@ +import numpy as np + +def swa_cluster_test(data, ChN, threshold): + """ + Finds clusters of spatially contiguous channels that exceed a given threshold + using a breadth-first search on the channel network. + Translated from swa-Matlab. + + Parameters: + ----------- + data : numpy.ndarray + 1D array of shape (n_channels,) containing the values to test (e.g., t-values, amplitudes). + ChN : numpy.ndarray + Neighbor matrix of shape (n_channels, max_neighbors) from swa_channelNeighbours (1-based indexing). + threshold : float + The minimum value required to consider a channel for clustering. + + Returns: + -------- + clusters : numpy.ndarray + 1D array of shape (n_channels,) containing NaN for unclustered channels + and integer IDs (1, 2, 3...) for each detected spatial cluster. + """ + # Forzar que data sea un vector plano (1D) + data = data.flatten() + n_channels = len(data) + + # Pre-asignar salidas y banderas de control + flag_used = np.zeros(n_channels, dtype=bool) + clusters = np.full(n_channels, np.nan) # Relleno de NaNs como en MATLAB + + cluster_id = 1 + + for n in range(n_channels): + # Si el canal no ha sido usado y supera el umbral, inicia un nuevo cluster + if not flag_used[n] and data[n] > threshold: + + # Marcar el canal inicial como examinado + flag_used[n] = True + + # Inicializar la lista del cluster (cola de exploración) + cluster_list = [n] + current_count = 0 # Python usa indexación 0-based + + # El bucle se detiene cuando revisamos todos los canales añadidos a la lista + while current_count < len(cluster_list): + + # Canal que estamos analizando actualmente + current_channel = cluster_list[current_count] + + # Obtener vecinos (filtrando los ceros que actúan como relleno en ChN) + # ChN viene en Base-1, restamos 1 para indexar en Python + raw_neighbours = ChN[current_channel, :] + current_neighbours = raw_neighbours[raw_neighbours > 0] - 1 + current_neighbours = current_neighbours.astype(int) + + if len(current_neighbours) > 0: + # Encontrar cuáles vecinos pasan el umbral y NO han sido usados todavía + in_cluster = (data[current_neighbours] > threshold) & (~flag_used[current_neighbours]) + + if np.sum(in_cluster) > 0: + # Extraer los índices reales de los vecinos que se unen al cluster + valid_neighbours = current_neighbours[in_cluster] + + # Marcarlos como usados inmediatamente para que no los vuelva a agarrar otro canal + flag_used[valid_neighbours] = True + + # Expandir la lista de canales a revisar (cola del BFS) + cluster_list.extend(valid_neighbours) + + # Avanzar al siguiente canal de la lista + current_count += 1 + + # Asignar el ID del cluster actual a todos los canales descubiertos en este viaje + clusters[cluster_list] = cluster_id + + # Incrementar el identificador para el próximo cluster + cluster_id += 1 + + return clusters \ No newline at end of file diff --git a/src/lib/swa/swa_filter_data.py b/src/lib/swa/swa_filter_data.py new file mode 100644 index 0000000..6721210 --- /dev/null +++ b/src/lib/swa/swa_filter_data.py @@ -0,0 +1,66 @@ +import numpy as np +import scipy.signal as signal + +def swa_filter_data(data, Info): + """ + Filters EEG data using either Chebyshev Type II or Butterworth filters. + Translated from swa-Matlab. + + Parameters: + ----------- + data : numpy.ndarray + EEG data array of shape (n_channels, n_samples). + Info : object or dict + Structure containing parameters and recording metadata. + + Returns: + -------- + filtData : numpy.ndarray + Filtered EEG data of shape (n_channels, n_samples). + """ + # 1. Verificar el orden del filtro por defecto + if not hasattr(Info['Parameters'], 'Filter_order') or Info['Parameters']['Filter_order'] is None: + Info['Parameters']['Filter_order'] = 2 + + # Extraer método en minúsculas para evitar problemas de mayúsculas/minúsculas + method = Info['Parameters']['Filter_Method'].lower() + nyquist = Info['Recording']['sRate'] / 2.0 + + # ========================================================================= + # METODO CHEBYSHEV (Tipo II) + # ========================================================================= + if method == 'chebyshev': + # Parámetros de filtrado normalizados respecto a Nyquist + Wp = np.array([Info['Parameters']['Filter_hPass'], Info['Parameters']['Filter_lPass']]) / nyquist + Ws = np.array([Info['Parameters']['Filter_hPass'] / 5.0, Info['Parameters']['Filter_lPass'] * 2.0]) / nyquist + Rp = 3.0 # Rizado en la banda de paso (dB) + Rs = 10.0 # Atenuación en la banda de rechazo (dB) + + # Calcular el orden óptimo del filtro Chebyshev y las frecuencias naturales + n, Wn = signal.cheb2ord(Wp, Ws, Rp, Rs) + Wn = [0.001, 0.08] # TODO: Mejorra debugueo, ahora forzado a igual que Matlab con valores por defecto + # Diseñar el filtro Chebyshev Tipo II (pasa-banda automático al pasar 2 frecuencias) + bbp, abp = signal.cheby2(n, Rs, Wn, btype='bandpass') + + # Aplicar el filtro de fase cero a lo largo del eje del tiempo (axis=-1) + filtData = signal.filtfilt(bbp, abp, data, axis=-1) + + # ========================================================================= + # METODO BUTTERWORTH (Soporta la errata 'buttersworth' del script original) + # ========================================================================= + elif method in ['buttersworth', 'butterworth']: + fhc = Info['Parameters']['Filter_hPass'] / nyquist + flc = Info['Parameters']['Filter_lPass'] / nyquist + + # Diseñar filtros Butterworth independientes (Alta y Baja frecuencia) + b1, a1 = signal.butter(Info['Parameters']['Filter_order'], fhc, btype='high') + b2, a2 = signal.butter(Info['Parameters']['Filter_order'], flc, btype='low') + + # Aplicar el filtrado secuencial idéntico a MATLAB usando el eje del tiempo + filtData = signal.filtfilt(b1, a1, data, axis=-1) + filtData = signal.filtfilt(b2, a2, filtData, axis=-1) + + else: + raise ValueError(f"Unknown filter method: {Info['Parameters']['Filter_Method']}") + + return filtData \ No newline at end of file diff --git a/src/lib/swa/swa_getInfoDefaults.py b/src/lib/swa/swa_getInfoDefaults.py new file mode 100644 index 0000000..257bc94 --- /dev/null +++ b/src/lib/swa/swa_getInfoDefaults.py @@ -0,0 +1,140 @@ +def swa_getInfoDefaults(Info, type_wave, method='envelope'): + """ + Get the current default detection parameters for slow waves (SW), spindles (SS), + or saw-tooth waves (ST). + Translated from swa-Matlab. + + Parameters: + ----------- + Info : dict + The configuration dictionary (equivalent to Info struct in MATLAB). + type_wave : str + Type of event to detect: 'SW' (Slow Waves), 'ST' (Saw-tooth), or 'SS' (Spindles). + method : str, optional + Method used mainly for 'SW' (e.g., 'envelope' or 'mdc'). Default is 'envelope'. + + Returns: + -------- + Info : dict + The updated dictionary containing all default configuration parameters. + """ + # Asegurar que la clave 'Parameters' exista en el diccionario Info + if 'Parameters' not in Info: + Info['Parameters'] = {} + + p = Info['Parameters'] # Atajo para simplificar la escritura del código + + # Manejar el valor por defecto del método si viene vacío o nulo + if method is None or method == '': + method = 'envelope' + + type_wave = type_wave.upper() + + # ========================================================================= + # CASE: SLOW WAVES (SW) + # ========================================================================= + if type_wave == 'SW': + # Parámetros de filtrado + p['Filter_Apply'] = True + p['Filter_Method'] = 'Chebyshev' # 'Chebyshev' / 'Buttersworth' + p['Filter_hPass'] = 0.5 + p['Filter_lPass'] = 4.0 + p['Filter_order'] = 2 + + # Detección en canal de referencia (Canonical Wave) + p['Ref_Method'] = None # Método canónico ('envelope', 'diamond', 'midline', etc.) + p['Ref_Electrodes'] = False # Array lógico de electrodos usados + p['Ref_InspectionPoint'] = 'MNP' # 'MNP' (Max Neg Peak) / 'ZC' (Zero Crossing) + p['Ref_UseInside'] = 1 # 1: Usar canales interiores de la cabeza / 0: Todos + p['Ref_UseStages'] = None # Información de estadiaje de sueño (Sleep Scoring) + p['Ref_AmplitudeCriteria'] = 'relative' # 'relative' / 'absolute' + p['Ref_AmplitudeRelative'] = 5.0 # Desviaciones estándar desde la media de negatividad + p['Ref_AmplitudeAbsolute'] = 60.0 + p['Ref_AmplitudeMax'] = 250.0 # Amplitud máxima para control de artefactos + p['Ref_WaveLength'] = [0.25, 1.25] # Criterio de longitud (segundos) entre cruces por cero + p['Ref_SlopeMin'] = 0.90 # Porcentaje de corte para las pendientes (slopes) + p['Ref_Peak2Peak'] = None # Solo para umbralización por canales + + # Detección por canales individuales + p['Channels_Correlate2'] = 'mean' # Qué onda canónica usar ('main', 'mean' o 'all') + p['Channels_Detection'] = 'correlation' # 'correlation' / 'threshold' + p['Channels_Threshold'] = 0.9 # Ajuste si se usa el método de umbralización + p['Channels_ClusterTest'] = True + p['Channels_WinSize'] = 0.100 # Ventana de búsqueda en segundos + + # Parámetros de propagación (Travelling) + p['Travelling_GS'] = 40 # Tamaño de la cuadrícula de interpolación + p['Travelling_MinDelay'] = 40.0 # Tiempo de viaje mínimo (ms) + p['Travelling_RecalculateDelay'] = True # False si los mapas de retraso se calculan fuera + + # Configuración específica según el método elegido + if method.lower() == 'envelope': + p['Ref_Method'] = 'Envelope' + elif method.lower() == 'mdc': # Massimini Detection Criteria + p['Ref_Method'] = 'diamond' + p['Ref_AmplitudeCriteria'] = 'absolute' + p['Ref_InspectionPoint'] = 'ZC' + p['Ref_AmplitudeAbsolute'] = 80.0 + p['Ref_Peak2Peak'] = 140.0 + p['Channels_Detection'] = 'threshold' + p['Channels_Threshold'] = 1.0 + + # ========================================================================= + # CASE: SAW-TOOTH WAVES (ST) + # ========================================================================= + elif type_wave == 'ST': + p['Ref_Method'] = 'Midline' + p['Ref_Electrodes'] = None + p['Filter_Apply'] = False # No se necesita filtro clásico para el método CWT + + # Parámetros Wavelet (CWT) para detección + p['CWT_hPass'] = 2.0 + p['CWT_lPass'] = 5.0 + p['CWT_StdThresh'] = 1.75 + p['CWT_AmpThresh'] = None + p['CWT_ThetaAlpha'] = 1.2 + + p['Burst_Length'] = 1.0 # Tiempo máximo entre ondas en segundos + p['Burst_Adjust'] = 0.75 # Ajuste de criterio si la onda se encuentra en ráfaga + + p['Channels_WinSize'] = 0.060 # En segundos + p['Channel_Adjust'] = 0.9 + p['Travelling_GS'] = 40 + p['Travelling_MinDelay'] = 20.0 + + # ========================================================================= + # CASE: SPINDLES (SS) + # ========================================================================= + elif type_wave == 'SS': + p['Ref_Method'] = 'Midline' + p['Ref_Electrodes'] = None + p['Filter_Apply'] = False + p['Ref_UseStages'] = None + + # Parámetros del filtro de Spindles + p['Filter_Method'] = 'Chebyshev' + p['Filter_band'] = [10.0, 16.0] + p['Filter_checkrange'] = 2.0 + p['Filter_Window'] = 0.150 # Ventana de suavizado RMS de la potencia + p['Filter_order'] = 2 + p['Wavelet_name'] = 'fbsp1-1-3' # B-spline wavelet + p['Wavelet_norm'] = 1 + + # Criterios del huso de sueño (Spindle) + p['Ref_AmplitudeCriteria'] = 'relative' + p['Ref_AmplitudeMetric'] = 'median' + p['Ref_AmplitudeRelative'] = [4.0, 2.0] # [umbral alto, umbral bajo] en Desviaciones Estándar + p['Ref_AmplitudeAbsolute'] = 15.0 + p['Ref_NeighbourRatio'] = 3.0 # Ratio mínimo potencia Spindle/Vecinos + + p['Ref_WaveLength'] = [0.3, 3.0] # Tiempo en segundos sobre el umbral + p['Ref_MinWaves'] = 3 # Número mínimo de ondas internas del huso + + p['Channels_Method'] = 'power' # Método wavelet o potencia (FFT) + p['Channels_WinSize'] = 0.150 # Ventana de búsqueda alrededor del huso de referencia + p['Channels_Threshold'] = 0.75 # Ajuste respecto al criterio de referencia + + else: + raise ValueError(f"Unknown wave type: {type_wave}. Choose 'SW', 'ST', or 'SS'.") + + return Info \ No newline at end of file diff --git a/src/lib/swa/swa_get_peaks.py b/src/lib/swa/swa_get_peaks.py new file mode 100644 index 0000000..42e4fb7 --- /dev/null +++ b/src/lib/swa/swa_get_peaks.py @@ -0,0 +1,92 @@ +import numpy as np + +def swa_get_peaks(slope_data, Info, flag_notch=False): + """ + Finds the maximum negative peaks (MNP) and maximum positive peaks (MPP) + within a channel's slope data and iteratively removes small notches. + Translated from swa-Matlab. + + Parameters: + ----------- + slope_data : numpy.ndarray + 1D array containing the derivative/slope of the EEG signal. + Info : dict + Configuration dictionary containing parameters and sampling rate. + flag_notch : bool, optional + If True, iteratively erases small bumps/notches based on wavelength. + + Returns: + -------- + MNP : numpy.ndarray + Indices of Maximum Negative Peaks (local minima). + MPP : numpy.ndarray + Indices of Maximum Positive Peaks (local maxima). + """ + # Encontrar el signo de la pendiente (-1, 0, 1) + sign_data = np.sign(slope_data) + # Forzar que los ceros cuenten como positivos para evitar problemas en mesetas + sign_data[sign_data == 0] = 1 + + diff_sign = np.diff(sign_data) + + # MNP: La pendiente pasa de negativa a positiva (mínimo local / pico negativo) + # En Python obtenemos el índice del cambio + MNP = np.where(diff_sign == 2)[0] + 1 + + # MPP: La pendiente pasa de positiva a negativa (máximo local / pico positivo) + MPP = np.where(diff_sign == -2)[0] + 1 + + if len(MNP) == 0 or len(MPP) == 0: + return np.array([], dtype=int), np.array([], dtype=int) + + # Asegurar que la secuencia empiece con un MPP (Pico Positivo Anterior) + if MNP[0] < MPP[0]: + MNP = MNP[1:] + + if len(MNP) == 0 or len(MPP) == 0: + return np.array([], dtype=int), np.array([], dtype=int) + + # Asegurar que la secuencia termine con un MPP (Pico Positivo Posterior) + if MNP[-1] > MPP[-1]: + MNP = MNP[:-1] + + # Eliminar muescas/ruido pequeño de forma iterativa + if flag_notch and len(MNP) > 0 and len(MPP) > 0: + sRate = Info['Recording']['sRate'] + # Umbral: 10% de la longitud de onda mínima permitida + thresh = Info['Parameters']['Ref_WaveLength'][0] * sRate / 10.0 + + nb = 1 + while nb > 0: + if len(MNP) == 0 or len(MPP) < 2: + break + + # --- 1. Eliminar "Bumps" Positivos --- + # Distancia entre el MNP actual y el SIGUIENTE MPP + posBumps = (MPP[1:] - MNP) < thresh + + if np.any(posBumps): + # En MPP queremos borrar el elemento subsiguiente, así que añadimos False al inicio + mpp_mask = np.concatenate(([False], posBumps)) + MPP = MPP[~mpp_mask] + MNP = MNP[~posBumps] + + if len(MNP) == 0 or len(MPP) < 2: + break + + # --- 2. Eliminar "Bumps" Negativos --- + # Distancia entre el MNP actual y el ANTERIOR MPP + negBumps = (MNP - MPP[:-1]) < thresh + + if np.any(negBumps): + # En MPP queremos borrar el elemento previo, así que añadimos False al final + mpp_mask = np.concatenate((negBumps, [False])) + MPP = MPP[~mpp_mask] + MNP = MNP[~negBumps] + + # Contar cuántos elementos se eliminaron en esta ronda para decidir si continuar + sum_pos = np.sum(posBumps) if np.any(posBumps) else 0 + sum_neg = np.sum(negBumps) if np.any(negBumps) else 0 + nb = max(sum_pos, sum_neg) + + return MNP, MPP \ No newline at end of file diff --git a/src/lib/swa/swa_saveOutput.py b/src/lib/swa/swa_saveOutput.py new file mode 100644 index 0000000..bd686e2 --- /dev/null +++ b/src/lib/swa/swa_saveOutput.py @@ -0,0 +1,88 @@ +import os +import numpy as np +import scipy.io as sio +import tkinter as tk +from tkinter import filedialog + +def swa_saveOutput(Data, Info, SW, save_name=None, flag_raw=True, flag_filtered=False, wave_type='SW'): + """ + Saves the wave detection output. + Translated from swa-Matlab. + + Parameters: + ----------- + Data : dict + Dictionary containing EEG data. + Info : dict + Configuration dictionary. + SW : list of dicts + List containing all detected waves. + save_name : str, optional + File path to save the output. If None, opens a UI dialog. + flag_raw : bool + If True, drops raw data from memory and saves only the file pointer. + flag_filtered : bool + If True, saves filtered data to an external binary file to save space. + wave_type : str + The key under which the wave structure will be saved ('SW', 'SS', 'ST'). + """ + # 1. Manejo de los Datos Crudos (Raw Data) + if flag_raw: + # Reemplaza los datos por un puntero al archivo si el espacio es una preocupación + if 'Recording' in Info and 'dataFile' in Info['Recording']: + Data['Raw'] = Info['Recording']['dataFile'] + else: + print("Information: Will save all the data into the new file; this may take some time and memory.") + + # 2. Manejo de los Datos Filtrados + if flag_filtered: + if 'Recording' in Info and 'dataFile' in Info['Recording']: + base_name = os.path.splitext(Info['Recording']['dataFile'])[0] + filtered_name = f"{base_name}_filtered.fdt" + + if not os.path.exists(filtered_name) and 'Filtered' in Data: + # Guardar los datos filtrados en un binario plano (equivalente a .fdt de MATLAB) + # Se asume que Data['Filtered'] es un numpy array + Data['Filtered'].astype(np.float32).tofile(filtered_name) + print(f"Calculation: Filtered data saved to {filtered_name}") + + Data['Filtered'] = filtered_name + + elif 'Filtered' in Data: + # Eliminar el campo si existe y no queremos guardarlo + del Data['Filtered'] + + # 3. Interfaz de usuario para nombrar el archivo si no se proporcionó uno + if save_name is None or save_name.strip() == '': + root = tk.Tk() + root.withdraw() # Ocultar la ventana principal de tkinter + save_name = filedialog.asksaveasfilename( + title="Save Output File", + defaultextension=".mat", + filetypes=[("MATLAB files", "*.mat"), ("All files", "*.*")] + ) + if not save_name: + print("Warning: Save cancelled by user.") + return + + # 4. Preparar el diccionario final para guardar + save_dict = { + 'Data': Data, + 'Info': Info, + wave_type: SW # Usamos la variable wave_type ('SW', 'SS' o 'ST') como llave + } + + # 5. Guardar el archivo + try: + # Intenta guardarlo usando scipy.io + sio.savemat(save_name, save_dict) + print(f"Success: Output successfully saved to {save_name}") + except OverflowError: + # Si el archivo es muy grande (> 2GB), scipy fallará al tratar de imitar mat v7.2. + # Fallback a un archivo Pickle nativo de Python + import pickle + pickle_name = os.path.splitext(save_name)[0] + '.pkl' + print("Warning: File too large for standard MAT format. Saving as Python Pickle (.pkl) instead.") + with open(pickle_name, 'wb') as f: + pickle.dump(save_dict, f, protocol=pickle.HIGHEST_PROTOCOL) + print(f"Success: Output successfully saved to {pickle_name}") \ No newline at end of file diff --git a/src/lib/swa/swa_xcorr.py b/src/lib/swa/swa_xcorr.py new file mode 100644 index 0000000..1ec39f4 --- /dev/null +++ b/src/lib/swa/swa_xcorr.py @@ -0,0 +1,154 @@ +import numpy as np +from numpy.lib.stride_tricks import sliding_window_view + +def swa_xcorr(refData, shortData, win): + + """ + Analyses the cross-correlation (Pearson r) between the reference channel + and all subsequent channels using a sliding window without zero padding. + Translated from swa-Matlab. + + Parameters: + ----------- + refData : numpy.ndarray + Reference signal row vector of shape (n_samples_ref,) or (1, n_samples_ref). + shortData : numpy.ndarray + Target channels matrix of shape (n_channels, n_samples_short). + win : int + Window size parameter. + + Returns: + -------- + R : numpy.ndarray + Correlation matrix of shape (n_channels, win * 2 + 1). + """ + # Asegurar que refData sea un vector plano (1D) para las operaciones de correlación + if refData.ndim > 1: + refData = refData.flatten() + + n_channels = refData.ndim + n_shifts = win * 2 + 1 + try: + len_ref = len(refData) + except: + len_ref = 1 + + # Pre-asignar la matriz de resultados (canales x desplazamientos) + R = np.zeros((n_channels, n_shifts)) + + # Centralizar la referencia (restar media) para el cálculo de Pearson + ref_centered = refData - np.mean(refData) + ref_norm = np.sqrt(np.sum(ref_centered ** 2)) + + if ref_norm == 0: + return R # Evitar división por cero si la referencia es plana + + # Bucle a lo largo del tiempo (desplazamientos de la ventana deslizante) + for t in range(n_shifts): + # Extraer la porción de datos de todos los canales para el lag 't' + # Equivalente a shortData(:, t : t + size(refData,2) - 1) en MATLAB + sub_data = shortData[t : t + len_ref] + + # Centralizar cada canal en esta ventana temporal específica + sub_mean = np.mean(sub_data, keepdims=True) + sub_centered = sub_data - sub_mean + + # Calcular desviaciones estándar (denominador de Pearson) + sub_norm = np.sqrt(np.sum(sub_centered ** 2)) + + # Producto punto entre la referencia y los canales (numerador de Pearson) + numerator = np.dot(sub_centered, ref_centered) + + # Coeficiente de correlación r de Pearson: n_channels x 1 + # Usamos np.errstate para evitar advertencias si algún canal tiene varianza cero + with np.errstate(divide='ignore', invalid='ignore'): + r_val = numerator / (sub_norm * ref_norm) + + # Reemplazar posibles NaNs (por canales planos/sin varianza) con 0 + if np.isnan(r_val): + r_val = 0.0 + + R[:, t] = r_val + + return R + +def swa_xcorr2(refData, shortData, win): + """ + Calculates the sliding Pearson cross-correlation between a reference channel + and all other channels without zero padding. + + Parameters: + ----------- + refData : numpy.ndarray + 1D array (or 2D with 1 row) of the reference channel data. + shortData : numpy.ndarray + 2D array (channels x samples) of the data to compare. + win : int + Window reach (the loop will execute win * 2 + 1 times). + """ + # Asegurar que refData sea un vector 1D plano + refData = np.squeeze(refData) + + n_channels = shortData.shape[1] + n_samples_ref = len(refData) + n_lags = win * 2 + 1 + + # Inicializar la matriz de resultados (canales x retrasos) + R = np.zeros((n_channels, n_lags)) + + # Precalcular la media y la desviación de la onda de referencia (fijos) + ref_mean = np.mean(refData) + ref_dev = refData - ref_mean + ref_var = np.sum(ref_dev ** 2) + + # Bucle solo para los desplazamientos temporales (lags) + for t in range(n_lags): + # Extraer la ventana actual para TODOS los canales a la vez + short_slice = shortData[t : t + n_samples_ref] + + # Calcular la media y desviación local de la ventana por cada canal + short_mean = np.mean(short_slice, keepdims=True) + short_dev = short_slice - short_mean + short_var = np.sum(short_dev ** 2) + + # Covarianza cruzada (multiplicación matricial elemento a elemento) + covariance = np.sum(short_dev * ref_dev) + + # Calcular el coeficiente r de Pearson (evitando divisiones por cero) + with np.errstate(divide='ignore', invalid='ignore'): + R[:, t] = covariance / np.sqrt(ref_var * short_var) + + # Reemplazar posibles NaNs (por canales planos sin variación) con cero + return np.nan_to_num(R) + +def swa_xcorr_ultra(refData, shortData, win): + """ + Calcula la correlación cruzada móvil de Pearson de forma 100% vectorial, + sin bucles 'for', utilizando vistas de ventanas deslizantes de NumPy. + + shortData debe tener la forma: (canales, muestras) + """ + refData = np.squeeze(refData) + n_samples_ref = len(refData) + + # 1. Crear las ventanas deslizantes de forma virtual (sin coste de memoria extra) + # Resultado 'windows' tendrá la forma: (canales, n_lags, n_samples_ref) + windows = sliding_window_view(shortData, window_shape=n_samples_ref, axis=1) + + # 2. Operaciones sobre la señal de referencia (fijas) + ref_dev = refData - np.mean(refData) + ref_var = np.sum(ref_dev ** 2) + + # 3. Operaciones vectoriales simultáneas para TODOS los canales y TODOS los lags + short_mean = np.mean(windows, axis=2, keepdims=True) + short_dev = windows - short_mean + short_var = np.sum(short_dev ** 2, axis=2) + + # 4. Covarianza cruzada mediante 'broadcasting' en la última dimensión (tiempo) + covariance = np.sum(short_dev * ref_dev, axis=2) + + # 5. Coeficiente de correlación de Pearson final + with np.errstate(divide='ignore', invalid='ignore'): + R = covariance / np.sqrt(ref_var * short_var) + + return np.nan_to_num(R) From 938772ca754f4e8fb524d6ccf6d1a6efa4495342 Mon Sep 17 00:00:00 2001 From: rolopu1 Date: Fri, 3 Jul 2026 12:46:59 +0200 Subject: [PATCH 91/93] Update file paths and enhance SW feature extraction in EEG processing --- helper_code.py | 2 +- src/eeg_processing.py | 22 ++++++++++ src/lib/eeg_features.py | 18 ++++++++ src/lib/swa/swa_CalculateReference.py | 59 ++++++++++++++------------- src/lib/swa/swa_FindSWChannels.py | 8 ++-- src/lib/swa/swa_FindSWRef.py | 6 +-- src/lib/swa/swa_filter_data.py | 2 +- src/pipeline/features.py | 2 +- 8 files changed, 80 insertions(+), 39 deletions(-) diff --git a/helper_code.py b/helper_code.py index e9bb2b5..1925dcd 100644 --- a/helper_code.py +++ b/helper_code.py @@ -15,7 +15,7 @@ from typing import Dict, List, Tuple, Any, Union, Tuple, Optional from collections import defaultdict -DEMOGRAPHICS_FILE = 'demographics.csv' +DEMOGRAPHICS_FILE = 'training_set/demographics.csv' PHYSIOLOGICAL_DATA_SUBFOLDER = 'physiological_data' ALGORITHMIC_ANNOTATIONS_SUBFOLDER = 'algorithmic_annotations' HUMAN_ANNOTATIONS_SUBFOLDER = 'human_annotations' diff --git a/src/eeg_processing.py b/src/eeg_processing.py index b5cf505..ee1aa08 100644 --- a/src/eeg_processing.py +++ b/src/eeg_processing.py @@ -11,6 +11,22 @@ 'F4-M1': {'direct': 'f4-m1', 'positive': 'f4', 'reference': 'm1'}, } EEG_FEATURE_SPECS = [ + ('C3-M2', 'TotalSW'), + ('C3-M2', 'SWpeakAmp_mean'), + ('C3-M2', 'SWp2p_mean'), + ('C3-M2', 'SWnegSlope_mean'), + ('C4-M1', 'TotalSW'), + ('C4-M1', 'SWpeakAmp_mean'), + ('C4-M1', 'SWp2p_mean'), + ('C4-M1', 'SWnegSlope_mean'), + ('F3-M2', 'TotalSW'), + ('F3-M2', 'SWpeakAmp_mean'), + ('F3-M2', 'SWp2p_mean'), + ('F3-M2', 'SWnegSlope_mean'), + ('F4-M1', 'TotalSW'), + ('F4-M1', 'SWpeakAmp_mean'), + ('F4-M1', 'SWp2p_mean'), + ('F4-M1', 'SWnegSlope_mean'), ('C3-M2', 'Hjorth_Complexity'), ('C4-M1', 'Hjorth_Complexity'), ('F3-M2', 'Hjorth_Complexity'), @@ -146,6 +162,12 @@ def _extract_channel_metrics(signal, fs): str(name): float(value) for name, value in patient_profile.replace([np.inf, -np.inf], np.nan).items() } + + SW_features = eeg_features.get_SW_features(signal, fs) + for feature_name in SW_features: + value = SW_features[feature_name] + metrics[feature_name] = float(np.nan if pd.isna(value) else value) + for complexity_name in ('Hjorth_Mobility', 'Hjorth_Complexity'): if complexity_name in complexities: value = complexities[complexity_name].replace([np.inf, -np.inf], np.nan).std() diff --git a/src/lib/eeg_features.py b/src/lib/eeg_features.py index d6e06ea..72481d2 100644 --- a/src/lib/eeg_features.py +++ b/src/lib/eeg_features.py @@ -5,6 +5,10 @@ from scipy.signal import welch import pandas as pd from scipy.stats import kurtosis, entropy +from src.lib.swa import swa_getInfoDefaults +from src.lib.swa import swa_CalculateReference +from src.lib.swa import swa_FindSWRef +from src.lib.swa import swa_FindSWChannels def _safe_sqrt_variance_ratio(numerator_signal, denominator_signal): numerator_var = np.var(numerator_signal) @@ -61,6 +65,20 @@ def extract_band_powers(epochs, fs, win_len = 2): return pd.DataFrame(features), pd.DataFrame(complexities) +def get_SW_features(signal, fs): + info = swa_getInfoDefaults.swa_getInfoDefaults({}, 'SW', method='envelope') + info['Electrodes'] = ['Ej'] + info['Recording'] = {} + info['Recording']['sRate'] = fs + + data = {} + data['Raw'] = pd.DataFrame({'Signal': signal}) + data['SWRef'], Info = swa_CalculateReference.swa_CalculateReference (data['Raw'], info, False) + Info['Parameters']['Ref_InspectionPoint'] = 'ZC' + data, Info, SW = swa_FindSWRef.swa_FindSWRef (data, Info) + data, Info, SW = swa_FindSWChannels.swa_FindSWChannels (data, Info, SW) + return SW + def get_patient_profile(df_features): total_power = df_features.sum(axis=1) avg_p = df_features.mean() diff --git a/src/lib/swa/swa_CalculateReference.py b/src/lib/swa/swa_CalculateReference.py index e88ef39..523a398 100644 --- a/src/lib/swa/swa_CalculateReference.py +++ b/src/lib/swa/swa_CalculateReference.py @@ -29,7 +29,7 @@ def swa_CalculateReference(data, Info, display_plot=False): if 'Recording' not in Info or 'sRate' not in Info['Recording']: raise ValueError("Error: No sampling rate information found in Info['Recording']") if 'Parameters' not in Info: - from swa_getInfoDefaults import swa_getInfoDefaults # Evitar importación cíclica + from src.lib.swa.swa_getInfoDefaults import swa_getInfoDefaults # Evitar importación cíclica Info = swa_getInfoDefaults(Info, 'SW', 'envelope') print("Warning: No parameters specified; using defaults.") @@ -40,32 +40,32 @@ def swa_CalculateReference(data, Info, display_plot=False): # 2. Ajuste y proyección 2D de las coordenadas de los electrodos # MATLAB: Th = pi/180*[Info.Electrodes.theta]; Rd = [Info.Electrodes.radius]; - e_locs = Info['Electrodes'] - # Soporta si e_locs es una lista de objetos o una lista de dicts (desde el JSON) - theta = np.array([getattr(el, 'theta', el.get('theta')) for el in e_locs], dtype=float) - radius = np.array([getattr(el, 'radius', el.get('radius')) for el in e_locs], dtype=float) + # e_locs = Info['Electrodes'] + # # Soporta si e_locs es una lista de objetos o una lista de dicts (desde el JSON) + # theta = np.array([getattr(el, 'theta', el.get('theta')) for el in e_locs], dtype=float) + # radius = np.array([getattr(el, 'radius', el.get('radius')) for el in e_locs], dtype=float) - Th = (np.pi / 180.0) * theta - x = radius * np.cos(Th) - y = radius * np.sin(Th) + # Th = (np.pi / 180.0) * theta + # x = radius * np.cos(Th) + # y = radius * np.sin(Th) - # Encajonar las coordenadas en un rango de -0.5 a 0.5 - intrad = min(1.0, max(np.abs(radius))) - intrad = max(intrad, 0.5) - squeezefac = 0.5 / intrad - x = x * squeezefac - y = y * squeezefac + # # Encajonar las coordenadas en un rango de -0.5 a 0.5 + # intrad = min(1.0, max(np.abs(radius))) + # intrad = max(intrad, 0.5) + # squeezefac = 0.5 / intrad + # x = x * squeezefac + # y = y * squeezefac # Inicializar figura si flag_plot está activo fig_topo, ax_topo = None, None - if display_plot: - fig_topo, ax_topo = plt.subplots(figsize=(6, 6)) - ax_topo.scatter(y, x, s=30, edgecolors=[0.5, 0.5, 0.5], facecolors=[0.5, 0.5, 0.5], label='All Electrodes') - ax_topo.set_aspect('equal') - ax_topo.axis('off') + # if display_plot: + # fig_topo, ax_topo = plt.subplots(figsize=(6, 6)) + # ax_topo.scatter(y, x, s=30, edgecolors=[0.5, 0.5, 0.5], facecolors=[0.5, 0.5, 0.5], label='All Electrodes') + # ax_topo.set_aspect('equal') + # ax_topo.axis('off') n_samples = data.shape[1] - n_total_ch = len(x) + n_total_ch = len(Info['Electrodes']) # ========================================================================= # SELECCIÓN POR MÉTODOS @@ -73,14 +73,14 @@ def swa_CalculateReference(data, Info, display_plot=False): if ref_method == 'envelope': # Evitar canales externos/periféricos si está configurado - if Info['Parameters'].get('Ref_UseInside', True): - distances = np.sqrt(x**2 + y**2) - # Guardar máscara (1 fila, n_canales) - Info['Parameters']['Ref_Electrodes'] = distances < 0.35 - working_data = data.iloc[Info['Parameters']['Ref_Electrodes'], :] - else: - Info['Parameters']['Ref_Electrodes'] = np.ones(n_total_ch, dtype=bool) - working_data = data + # if Info['Parameters'].get('Ref_UseInside', True): + # distances = np.sqrt(x**2 + y**2) + # # Guardar máscara (1 fila, n_canales) + # Info['Parameters']['Ref_Electrodes'] = distances < 0.35 + # working_data = data.iloc[Info['Parameters']['Ref_Electrodes'], :] + # else: + Info['Parameters']['Ref_Electrodes'] = np.ones(n_total_ch, dtype=bool) + working_data = data # Ordenar muestras para obtener el percentil de negatividad rData = np.sort(working_data, axis=0) @@ -91,6 +91,7 @@ def swa_CalculateReference(data, Info, display_plot=False): nData = np.mean(rData[1:nCh, :], axis=0, keepdims=True) else: nData = np.mean(rData[0:nCh, :], axis=0, keepdims=True) + nData = rData elif ref_method in ['square', 'diamond']: distance_from_center = 0.2 @@ -196,7 +197,7 @@ def swa_CalculateReference(data, Info, display_plot=False): print(f"Calculation: Applying {Info['Parameters']['Filter_Method']} filter for [{Info['Parameters']['Filter_hPass']:.1f}, {Info['Parameters']['Filter_lPass']:.1f}] Hz...") # Llamar a la función previamente traducida - from lib.swa.swa_filter_data import swa_filter_data + from src.lib.swa.swa_filter_data import swa_filter_data filtData = swa_filter_data(nData, Info) print("Done") else: diff --git a/src/lib/swa/swa_FindSWChannels.py b/src/lib/swa/swa_FindSWChannels.py index 4b8c29c..b7f162e 100644 --- a/src/lib/swa/swa_FindSWChannels.py +++ b/src/lib/swa/swa_FindSWChannels.py @@ -1,9 +1,9 @@ import numpy as np from tqdm import tqdm -from lib.swa.swa_channelNeighbours import swa_channelNeighbours -from lib.swa.swa_filter_data import swa_filter_data -from lib.swa.swa_xcorr import swa_xcorr, swa_xcorr2, swa_xcorr_ultra -from lib.swa.swa_cluster_test import swa_cluster_test +from .swa_channelNeighbours import swa_channelNeighbours +from .swa_filter_data import swa_filter_data +from .swa_xcorr import swa_xcorr, swa_xcorr2, swa_xcorr_ultra +from .swa_cluster_test import swa_cluster_test def swa_FindSWChannels(Data, Info, SW, flag_progress=True): """ diff --git a/src/lib/swa/swa_FindSWRef.py b/src/lib/swa/swa_FindSWRef.py index 683182d..eb760e3 100644 --- a/src/lib/swa/swa_FindSWRef.py +++ b/src/lib/swa/swa_FindSWRef.py @@ -37,7 +37,7 @@ def swa_FindSWRef(Data, Info, SW=None): OSWCount = len(SW) SWCount = len(SW) - number_ref_waves = Data['SWRef'].shape[0] + number_ref_waves = Data['SWRef'].shape[1] # Inicializar vectores de umbrales según el criterio if p.get('Ref_AmplitudeCriteria') == 'relative': @@ -63,13 +63,13 @@ def swa_FindSWRef(Data, Info, SW=None): if ref_wave > 0: OSWCount = len(SW) - ref_signal = Data['SWRef'][ref_wave, :] + ref_signal = Data['SWRef'][:, ref_wave] # Calcular la derivada de la señal (añadiendo un 0 inicial para mantener longitud) slopeData = np.concatenate(([0], np.diff(ref_signal))) # Extraer Mínimos (MNP - Maximum Negative Peaks) y Máximos (MPP) - from lib.swa.swa_get_peaks import swa_get_peaks + from src.lib.swa.swa_get_peaks import swa_get_peaks MNP, MPP = swa_get_peaks(slopeData, Info, True) MNP = MNP[1:] diff --git a/src/lib/swa/swa_filter_data.py b/src/lib/swa/swa_filter_data.py index 6721210..c763abd 100644 --- a/src/lib/swa/swa_filter_data.py +++ b/src/lib/swa/swa_filter_data.py @@ -43,7 +43,7 @@ def swa_filter_data(data, Info): bbp, abp = signal.cheby2(n, Rs, Wn, btype='bandpass') # Aplicar el filtro de fase cero a lo largo del eje del tiempo (axis=-1) - filtData = signal.filtfilt(bbp, abp, data, axis=-1) + filtData = signal.filtfilt(bbp, abp, data, axis=0) # ========================================================================= # METODO BUTTERWORTH (Soporta la errata 'buttersworth' del script original) diff --git a/src/pipeline/features.py b/src/pipeline/features.py index 0d46084..54433c2 100644 --- a/src/pipeline/features.py +++ b/src/pipeline/features.py @@ -375,7 +375,7 @@ def _compute_record_feature_vector(patient_data, data_folder, site_id, patient_i patient_id, session_id, ) - + print(f"Extracting features for patient {patient_id}, session {session_id} from file: {physiological_data_file}") if os.path.exists(physiological_data_file): physiological_data, physiological_fs = _load_required_signal_data(physiological_data_file, csv_path) physiological_features = extract_extended_physiological_features( From 23954de897673b8c38d8115ea47b149ad2bfab4c Mon Sep 17 00:00:00 2001 From: rolopu1 Date: Mon, 20 Jul 2026 12:37:41 +0200 Subject: [PATCH 92/93] Added SW features into workflow --- src/eeg_processing.py | 37 +++--- src/lib/eeg_features.py | 138 ++++++++++++++++++--- src/lib/swa/swa_CalculateReference.py | 7 +- src/lib/swa/swa_FindSWChannels.py | 32 ++--- src/lib/swa/swa_FindSWRef.py | 9 +- src/lib/swa/swa_filter_data.py | 7 +- tests/test_eeg_slow_wave_features.py | 169 ++++++++++++++++++++++++++ 7 files changed, 335 insertions(+), 64 deletions(-) create mode 100644 tests/test_eeg_slow_wave_features.py diff --git a/src/eeg_processing.py b/src/eeg_processing.py index ee1aa08..e25cb70 100644 --- a/src/eeg_processing.py +++ b/src/eeg_processing.py @@ -11,22 +11,11 @@ 'F4-M1': {'direct': 'f4-m1', 'positive': 'f4', 'reference': 'm1'}, } EEG_FEATURE_SPECS = [ - ('C3-M2', 'TotalSW'), - ('C3-M2', 'SWpeakAmp_mean'), - ('C3-M2', 'SWp2p_mean'), - ('C3-M2', 'SWnegSlope_mean'), - ('C4-M1', 'TotalSW'), - ('C4-M1', 'SWpeakAmp_mean'), - ('C4-M1', 'SWp2p_mean'), - ('C4-M1', 'SWnegSlope_mean'), - ('F3-M2', 'TotalSW'), - ('F3-M2', 'SWpeakAmp_mean'), - ('F3-M2', 'SWp2p_mean'), - ('F3-M2', 'SWnegSlope_mean'), - ('F4-M1', 'TotalSW'), - ('F4-M1', 'SWpeakAmp_mean'), - ('F4-M1', 'SWp2p_mean'), - ('F4-M1', 'SWnegSlope_mean'), + *[ + (channel_name, feature_name) + for channel_name in EEG_CHANNEL_SPECS + for feature_name in eeg_features.SLOW_WAVE_FEATURE_NAMES + ], ('C3-M2', 'Hjorth_Complexity'), ('C4-M1', 'Hjorth_Complexity'), ('F3-M2', 'Hjorth_Complexity'), @@ -163,9 +152,17 @@ def _extract_channel_metrics(signal, fs): for name, value in patient_profile.replace([np.inf, -np.inf], np.nan).items() } - SW_features = eeg_features.get_SW_features(signal, fs) - for feature_name in SW_features: - value = SW_features[feature_name] + try: + slow_wave_features = eeg_features.get_SW_features(signal, fs) + # SW extraction is optional: a detector failure must not discard the + # spectral and Hjorth metrics already computed for this EEG channel. + except Exception: + slow_wave_features = { + feature_name: np.nan + for feature_name in eeg_features.SLOW_WAVE_FEATURE_NAMES + } + + for feature_name, value in slow_wave_features.items(): metrics[feature_name] = float(np.nan if pd.isna(value) else value) for complexity_name in ('Hjorth_Mobility', 'Hjorth_Complexity'): @@ -204,4 +201,4 @@ def processEEG(physiological_data, physiological_fs, csv_path): return np.asarray(values, dtype=np.float32) -_normalize_label = normalize_channel_label \ No newline at end of file +_normalize_label = normalize_channel_label diff --git a/src/lib/eeg_features.py b/src/lib/eeg_features.py index 72481d2..854035a 100644 --- a/src/lib/eeg_features.py +++ b/src/lib/eeg_features.py @@ -1,14 +1,40 @@ """EEG feature helpers used by the active submission pipeline.""" -from scipy.signal import butter, filtfilt +from contextlib import nullcontext, redirect_stdout +import io + import numpy as np -from scipy.signal import welch import pandas as pd +from scipy.signal import butter, filtfilt, welch from scipy.stats import kurtosis, entropy -from src.lib.swa import swa_getInfoDefaults -from src.lib.swa import swa_CalculateReference -from src.lib.swa import swa_FindSWRef -from src.lib.swa import swa_FindSWChannels + +from .swa import swa_CalculateReference +from .swa import swa_FindSWChannels +from .swa import swa_FindSWRef +from .swa import swa_getInfoDefaults + + +SLOW_WAVE_FEATURE_NAMES = ( + 'TotalSW', + 'SWdensity', + 'SWpeakAmp_mean', + 'SWpeakAmp_std', + 'SWp2p_mean', + 'SWp2p_std', + 'SWnegSlope_mean', + 'SWnegSlope_std', + 'SWposSlope_mean', + 'SWposSlope_std', + 'SWduration_mean', + 'SWduration_std', +) + +_SLOW_WAVE_EVENT_FIELDS = { + 'SWpeakAmp': 'Ref_PeakAmp', + 'SWp2p': 'Ref_P2PAmp', + 'SWnegSlope': 'Ref_NegSlope', + 'SWposSlope': 'Ref_PosSlope', +} def _safe_sqrt_variance_ratio(numerator_signal, denominator_signal): numerator_var = np.var(numerator_signal) @@ -65,19 +91,93 @@ def extract_band_powers(epochs, fs, win_len = 2): return pd.DataFrame(features), pd.DataFrame(complexities) -def get_SW_features(signal, fs): +def _finite_slow_wave_values(slow_waves, field_name): + values = [] + for wave in slow_waves: + try: + value = float(np.asarray(wave[field_name]).squeeze()) + except (KeyError, TypeError, ValueError): + continue + if np.isfinite(value): + values.append(value) + return np.asarray(values, dtype=float) + + +def summarize_slow_waves(slow_waves, fs, signal_duration_seconds): + """Aggregate the event dictionaries returned by ``swa`` into scalar features. + + Amplitudes and slopes retain the sign and units returned by ``swa``. + Durations and density are expressed in seconds and waves/minute, respectively. + """ + if fs <= 0 or not np.isfinite(fs): + raise ValueError('fs must be a positive finite sampling frequency.') + if signal_duration_seconds <= 0 or not np.isfinite(signal_duration_seconds): + raise ValueError('signal_duration_seconds must be positive and finite.') + + slow_waves = [] if slow_waves is None else list(slow_waves) + features = {name: np.nan for name in SLOW_WAVE_FEATURE_NAMES} + features['TotalSW'] = float(len(slow_waves)) + features['SWdensity'] = float(len(slow_waves) / (signal_duration_seconds / 60.0)) + + for feature_prefix, event_field in _SLOW_WAVE_EVENT_FIELDS.items(): + values = _finite_slow_wave_values(slow_waves, event_field) + if values.size: + features[f'{feature_prefix}_mean'] = float(np.mean(values)) + features[f'{feature_prefix}_std'] = float(np.std(values)) + + durations = [] + for wave in slow_waves: + try: + duration = ( + float(np.asarray(wave['Ref_UpInd']).squeeze()) + - float(np.asarray(wave['Ref_DownInd']).squeeze()) + ) / float(fs) + except (KeyError, TypeError, ValueError): + continue + if np.isfinite(duration) and duration > 0: + durations.append(duration) + + if durations: + features['SWduration_mean'] = float(np.mean(durations)) + features['SWduration_std'] = float(np.std(durations)) + + return features + + +def get_SW_features(signal, fs, verbose=False): + """Detect slow waves with ``swa`` and return fixed-length scalar features.""" + signal = np.asarray(signal, dtype=float).reshape(-1) + if fs <= 0 or not np.isfinite(fs): + raise ValueError('fs must be a positive finite sampling frequency.') + if signal.size == 0: + raise ValueError('signal must contain at least one sample.') + if not np.all(np.isfinite(signal)): + raise ValueError('signal must contain only finite values.') + info = swa_getInfoDefaults.swa_getInfoDefaults({}, 'SW', method='envelope') - info['Electrodes'] = ['Ej'] - info['Recording'] = {} - info['Recording']['sRate'] = fs - - data = {} - data['Raw'] = pd.DataFrame({'Signal': signal}) - data['SWRef'], Info = swa_CalculateReference.swa_CalculateReference (data['Raw'], info, False) - Info['Parameters']['Ref_InspectionPoint'] = 'ZC' - data, Info, SW = swa_FindSWRef.swa_FindSWRef (data, Info) - data, Info, SW = swa_FindSWChannels.swa_FindSWChannels (data, Info, SW) - return SW + info['Electrodes'] = ['EEG'] + info['Recording'] = {'sRate': float(fs)} + info['Parameters']['Ref_InspectionPoint'] = 'ZC' + # Spatial clustering is undefined for a single-channel invocation. + info['Parameters']['Channels_ClusterTest'] = False + + # swa consistently uses (channels, samples). + data = {'Raw': signal[np.newaxis, :]} + output_context = nullcontext() if verbose else redirect_stdout(io.StringIO()) + with output_context: + data['SWRef'], info = swa_CalculateReference.swa_CalculateReference( + data['Raw'], info, False + ) + data, info, slow_waves = swa_FindSWRef.swa_FindSWRef(data, info) + data, info, slow_waves = swa_FindSWChannels.swa_FindSWChannels( + data, info, slow_waves, flag_progress=verbose + ) + + return summarize_slow_waves( + slow_waves, + fs=float(fs), + signal_duration_seconds=signal.size / float(fs), + ) def get_patient_profile(df_features): total_power = df_features.sum(axis=1) @@ -127,4 +227,4 @@ def get_patient_profile(df_features): } profile = pd.concat([rel_powers, pd.Series(ratios)]) - return profile \ No newline at end of file + return profile diff --git a/src/lib/swa/swa_CalculateReference.py b/src/lib/swa/swa_CalculateReference.py index 523a398..dba222c 100644 --- a/src/lib/swa/swa_CalculateReference.py +++ b/src/lib/swa/swa_CalculateReference.py @@ -23,6 +23,10 @@ def swa_CalculateReference(data, Info, display_plot=False): Info : dict Updated Info dictionary with chosen parameters and reference electrodes. """ + data = np.asarray(data, dtype=float) + if data.ndim != 2: + raise ValueError("Error: EEG data must have shape (channels, samples)") + # 1. Validaciones de estructura iniciales if 'Electrodes' not in Info: raise ValueError("Error: No electrode information found in Info") @@ -91,7 +95,6 @@ def swa_CalculateReference(data, Info, display_plot=False): nData = np.mean(rData[1:nCh, :], axis=0, keepdims=True) else: nData = np.mean(rData[0:nCh, :], axis=0, keepdims=True) - nData = rData elif ref_method in ['square', 'diamond']: distance_from_center = 0.2 @@ -231,4 +234,4 @@ def swa_CalculateReference(data, Info, display_plot=False): plt.grid(True, alpha=0.3) plt.show() - return filtData, Info \ No newline at end of file + return filtData, Info diff --git a/src/lib/swa/swa_FindSWChannels.py b/src/lib/swa/swa_FindSWChannels.py index b7f162e..0ae14fa 100644 --- a/src/lib/swa/swa_FindSWChannels.py +++ b/src/lib/swa/swa_FindSWChannels.py @@ -18,9 +18,11 @@ def swa_FindSWChannels(Data, Info, SW, flag_progress=True): + p = Info['Parameters'] + # 2. Cluster test parameter check - if getattr(Info['Parameters'], 'Channels_ClusterTest', True): - if not hasattr(Info['Recording'], 'ChannelNeighbours'): + if p.get('Channels_ClusterTest', True): + if 'ChannelNeighbours' not in Info['Recording']: print("Calculating: Channel Neighbours...", end="") # Asume la existencia de la función externa swa_channelNeighbours Info['Recording']['ChannelNeighbours'] = swa_channelNeighbours(Info['Electrodes']) @@ -29,16 +31,16 @@ def swa_FindSWChannels(Data, Info, SW, flag_progress=True): print("Information: Using channels neighbourhood in 'Info'.") # 3. Parameter default settings for Envelope method - if getattr(Info['Parameters'], 'Ref_Method', '') == 'Envelope': - if not hasattr(Info['Parameters'], 'Channels_Threshold'): + if p.get('Ref_Method', '').lower() == 'envelope': + if 'Channels_Threshold' not in p: print("Warning: No further SW parameters found in Info; using defaults") Info['Parameters']['Channels_Threshold'] = 0.9 Info['Parameters']['Channels_WinSize'] = 0.2 # 4. Filter data if not already done - if hasattr(Data, 'Filtered') and Data.Filtered is not None: - if Data.Filtered.size == 0: - Data.Filtered = swa_filter_data(Data['Raw'], Info) + if Data.get('Filtered') is not None: + if Data['Filtered'].size == 0: + Data['Filtered'] = swa_filter_data(Data['Raw'], Info) else: print("Calculation: Filtering Data.") Data['Filtered'] = swa_filter_data(Data['Raw'], Info) @@ -66,8 +68,8 @@ def swa_FindSWChannels(Data, Info, SW, flag_progress=True): # Iteración con barra de progreso tqdm for nSW, sw in enumerate(tqdm(SW, disable=not flag_progress, desc="Finding Slow Waves (Correlation)")): - # Ajuste de índice de MATLAB (1-based) a Python (0-based) - ref_peak_ind = int(sw['Ref_PeakInd']) - 1 + # Ref_PeakInd is already zero-based in swa_FindSWRef. + ref_peak_ind = int(sw['Ref_PeakInd']) # Check search window bounds if ref_peak_ind - win * 2 < 0 or ref_peak_ind + win * 2 >= n_samples: @@ -78,7 +80,7 @@ def swa_FindSWChannels(Data, Info, SW, flag_progress=True): shortData = Data['Filtered'][:,ref_peak_ind - win * 2 : ref_peak_ind + win * 2 + 1] # Get canonical wave reference data - correlate_type = getattr(Info['Parameters'], 'Channels_Correlate2', 'all') + correlate_type = p.get('Channels_Correlate2', 'all') if correlate_type == 'mean': # En Python los índices de regiones deben ser enteros o booleanos @@ -101,7 +103,7 @@ def swa_FindSWChannels(Data, Info, SW, flag_progress=True): maxID = np.argmax(cc, axis=1) # 1. Forzamos a que maxCC sea un vector plano 1D (elimina cualquier dimensión extra como (N, 1)) - maxCC_1d = np.squeeze(np.asarray(maxCC)) + maxCC_1d = np.asarray(maxCC).reshape(-1) # 2. Creamos channels_active como un vector booleano puramente 1D de tamaño (n_channels,) channels_active = maxCC_1d > Info['Parameters']['Channels_Threshold'] @@ -129,7 +131,7 @@ def swa_FindSWChannels(Data, Info, SW, flag_progress=True): deactivate_mask = (sw['Channels_NegAmp'][:, 0] > amp_thresh).flatten() channels_active[deactivate_mask] = False - if getattr(Info['Parameters'], 'Channels_ClusterTest', True): + if p.get('Channels_ClusterTest', True): clusters = swa_cluster_test(channels_active.astype(float), Info['Recording']['ChannelNeighbours'], 0.01) clusters[np.isnan(clusters)] = 0 n_clusters = np.unique(clusters) @@ -179,12 +181,12 @@ def swa_FindSWChannels(Data, Info, SW, flag_progress=True): # THRESHOLD METHOD # ========================================================================= elif detection_method == 'threshold': - if not getattr(Info['Parameters'], 'Ref_Peak2Peak', None): + if not p.get('Ref_Peak2Peak'): Info['Parameters']['Ref_Peak2Peak'] = abs(Info['Parameters']['Ref_NegAmpMin']) * 1.75 for nSW, sw in enumerate(tqdm(SW, disable=not flag_progress, desc="Finding Slow Waves (Threshold)")): - ref_peak_ind = int(sw['Ref_PeakInd']) - 1 + ref_peak_ind = int(sw['Ref_PeakInd']) if ref_peak_ind - win < 0 or ref_peak_ind + win * 3 >= n_samples: to_delete.add(nSW) @@ -231,4 +233,4 @@ def swa_FindSWChannels(Data, Info, SW, flag_progress=True): print(f"Information: {len(to_delete)} slow waves were removed due to insufficient criteria.") SW = [sw for idx, sw in enumerate(SW) if idx not in to_delete] - return Data, Info, SW \ No newline at end of file + return Data, Info, SW diff --git a/src/lib/swa/swa_FindSWRef.py b/src/lib/swa/swa_FindSWRef.py index eb760e3..35a849d 100644 --- a/src/lib/swa/swa_FindSWRef.py +++ b/src/lib/swa/swa_FindSWRef.py @@ -37,7 +37,7 @@ def swa_FindSWRef(Data, Info, SW=None): OSWCount = len(SW) SWCount = len(SW) - number_ref_waves = Data['SWRef'].shape[1] + number_ref_waves = Data['SWRef'].shape[0] # Inicializar vectores de umbrales según el criterio if p.get('Ref_AmplitudeCriteria') == 'relative': @@ -54,7 +54,8 @@ def swa_FindSWRef(Data, Info, SW=None): if p.get('Ref_AmplitudeCriteria') == 'absolute': abs_val = np.atleast_1d(p['Ref_AmplitudeAbsolute']) if len(abs_val) < number_ref_waves: - p['Ref_AmplitudeAbsolute'] = np.repeat(abs_val[0], number_ref_waves) + abs_val = np.repeat(abs_val[0], number_ref_waves) + p['Ref_AmplitudeAbsolute'] = abs_val sRate = Info['Recording']['sRate'] @@ -63,7 +64,7 @@ def swa_FindSWRef(Data, Info, SW=None): if ref_wave > 0: OSWCount = len(SW) - ref_signal = Data['SWRef'][:, ref_wave] + ref_signal = Data['SWRef'][ref_wave, :] # Calcular la derivada de la señal (añadiendo un 0 inicial para mantener longitud) slopeData = np.concatenate(([0], np.diff(ref_signal))) @@ -250,4 +251,4 @@ def swa_FindSWRef(Data, Info, SW=None): if removed_count > 0: print(f"Information: {removed_count} waves were found in non-specified stages and removed") - return Data, Info, SW \ No newline at end of file + return Data, Info, SW diff --git a/src/lib/swa/swa_filter_data.py b/src/lib/swa/swa_filter_data.py index c763abd..d045236 100644 --- a/src/lib/swa/swa_filter_data.py +++ b/src/lib/swa/swa_filter_data.py @@ -19,7 +19,7 @@ def swa_filter_data(data, Info): Filtered EEG data of shape (n_channels, n_samples). """ # 1. Verificar el orden del filtro por defecto - if not hasattr(Info['Parameters'], 'Filter_order') or Info['Parameters']['Filter_order'] is None: + if Info['Parameters'].get('Filter_order') is None: Info['Parameters']['Filter_order'] = 2 # Extraer método en minúsculas para evitar problemas de mayúsculas/minúsculas @@ -38,12 +38,11 @@ def swa_filter_data(data, Info): # Calcular el orden óptimo del filtro Chebyshev y las frecuencias naturales n, Wn = signal.cheb2ord(Wp, Ws, Rp, Rs) - Wn = [0.001, 0.08] # TODO: Mejorra debugueo, ahora forzado a igual que Matlab con valores por defecto # Diseñar el filtro Chebyshev Tipo II (pasa-banda automático al pasar 2 frecuencias) bbp, abp = signal.cheby2(n, Rs, Wn, btype='bandpass') # Aplicar el filtro de fase cero a lo largo del eje del tiempo (axis=-1) - filtData = signal.filtfilt(bbp, abp, data, axis=0) + filtData = signal.filtfilt(bbp, abp, data, axis=-1) # ========================================================================= # METODO BUTTERWORTH (Soporta la errata 'buttersworth' del script original) @@ -63,4 +62,4 @@ def swa_filter_data(data, Info): else: raise ValueError(f"Unknown filter method: {Info['Parameters']['Filter_Method']}") - return filtData \ No newline at end of file + return filtData diff --git a/tests/test_eeg_slow_wave_features.py b/tests/test_eeg_slow_wave_features.py new file mode 100644 index 0000000..fda5c30 --- /dev/null +++ b/tests/test_eeg_slow_wave_features.py @@ -0,0 +1,169 @@ +from contextlib import redirect_stdout +import io +import unittest +from unittest.mock import patch + +import numpy as np + +from src import eeg_processing +from src.lib import eeg_features +from src.lib.swa.swa_FindSWRef import swa_FindSWRef +from src.lib.swa.swa_getInfoDefaults import swa_getInfoDefaults + + +class SlowWaveFeatureTests(unittest.TestCase): + def test_summarize_slow_waves_returns_fixed_numeric_features(self): + waves = [ + { + 'Ref_DownInd': 10, + 'Ref_UpInd': 110, + 'Ref_PeakAmp': -80, + 'Ref_P2PAmp': 130, + 'Ref_NegSlope': -4, + 'Ref_PosSlope': 5, + }, + { + 'Ref_DownInd': 200, + 'Ref_UpInd': 350, + 'Ref_PeakAmp': -120, + 'Ref_P2PAmp': 190, + 'Ref_NegSlope': -8, + 'Ref_PosSlope': 9, + }, + ] + + features = eeg_features.summarize_slow_waves( + waves, fs=100, signal_duration_seconds=120 + ) + + self.assertEqual(tuple(features), eeg_features.SLOW_WAVE_FEATURE_NAMES) + self.assertEqual(features['TotalSW'], 2.0) + self.assertEqual(features['SWdensity'], 1.0) + self.assertEqual(features['SWpeakAmp_mean'], -100.0) + self.assertEqual(features['SWpeakAmp_std'], 20.0) + self.assertEqual(features['SWp2p_mean'], 160.0) + self.assertEqual(features['SWnegSlope_mean'], -6.0) + self.assertEqual(features['SWposSlope_mean'], 7.0) + self.assertAlmostEqual(features['SWduration_mean'], 1.25) + self.assertAlmostEqual(features['SWduration_std'], 0.25) + + def test_summarize_no_waves_distinguishes_absence_from_missing_morphology(self): + features = eeg_features.summarize_slow_waves( + [], fs=200, signal_duration_seconds=300 + ) + + self.assertEqual(features['TotalSW'], 0.0) + self.assertEqual(features['SWdensity'], 0.0) + for name in eeg_features.SLOW_WAVE_FEATURE_NAMES[2:]: + self.assertTrue(np.isnan(features[name]), name) + + def test_swa_reference_detector_uses_channels_by_samples_orientation(self): + fs = 100 + seconds = 20 + time = np.arange(fs * seconds) / fs + reference = 100.0 * np.sin(2 * np.pi * time) + + info = swa_getInfoDefaults({}, 'SW', method='envelope') + info['Recording'] = {'sRate': fs} + info['Parameters']['Ref_InspectionPoint'] = 'ZC' + info['Parameters']['Ref_AmplitudeCriteria'] = 'absolute' + info['Parameters']['Ref_AmplitudeAbsolute'] = 50.0 + data = {'SWRef': reference[np.newaxis, :]} + + with redirect_stdout(io.StringIO()): + _, _, waves = swa_FindSWRef(data, info) + + self.assertGreater(len(waves), 0) + self.assertTrue(all(0 <= wave['Ref_PeakInd'] < reference.size for wave in waves)) + + def test_get_sw_features_detects_a_synthetic_slow_wave_signal(self): + fs = 100 + time = np.arange(fs * 20) / fs + signal = 100.0 * np.sin(2 * np.pi * time) + + features = eeg_features.get_SW_features(signal, fs) + + self.assertGreater(features['TotalSW'], 10) + self.assertGreater(features['SWdensity'], 30) + self.assertLess(features['SWpeakAmp_mean'], -80) + self.assertGreater(features['SWp2p_mean'], 160) + self.assertTrue(0.4 < features['SWduration_mean'] < 0.6) + + def test_get_sw_features_aggregates_swa_events(self): + fs = 100 + signal = np.zeros(fs * 60, dtype=float) + waves = [{ + 'Ref_DownInd': 100, + 'Ref_UpInd': 180, + 'Ref_PeakInd': 140, + 'Ref_PeakAmp': -75, + 'Ref_P2PAmp': 125, + 'Ref_NegSlope': -3, + 'Ref_PosSlope': 4, + }] + + with ( + patch.object( + eeg_features.swa_CalculateReference, + 'swa_CalculateReference', + return_value=(signal[np.newaxis, :], { + 'Recording': {'sRate': fs}, + 'Parameters': {}, + 'Electrodes': ['EEG'], + }), + ), + patch.object( + eeg_features.swa_FindSWRef, + 'swa_FindSWRef', + side_effect=lambda data, info: (data, info, waves), + ), + patch.object( + eeg_features.swa_FindSWChannels, + 'swa_FindSWChannels', + side_effect=lambda data, info, detected, flag_progress: ( + data, info, detected + ), + ), + ): + features = eeg_features.get_SW_features(signal, fs) + + self.assertEqual(tuple(features), eeg_features.SLOW_WAVE_FEATURE_NAMES) + self.assertEqual(features['TotalSW'], 1.0) + self.assertEqual(features['SWdensity'], 1.0) + self.assertEqual(features['SWpeakAmp_mean'], -75.0) + + def test_sw_failure_does_not_discard_other_channel_metrics(self): + fs = 200 + time = np.arange(fs * 30) / fs + signal = ( + 20.0 * np.sin(2 * np.pi * 10 * time) + + 5.0 * np.sin(2 * np.pi * time) + ) + + with patch.object( + eeg_features, + 'get_SW_features', + side_effect=RuntimeError('detector failed'), + ): + metrics = eeg_processing._extract_channel_metrics(signal, fs) + + self.assertIsNotNone(metrics) + self.assertTrue(np.isfinite(metrics['Relative_Delta_Power'])) + for name in eeg_features.SLOW_WAVE_FEATURE_NAMES: + self.assertTrue(np.isnan(metrics[name]), name) + + def test_all_slow_wave_features_are_exposed_for_each_eeg_channel(self): + for channel_name in eeg_processing.EEG_CHANNEL_SPECS: + for feature_name in eeg_features.SLOW_WAVE_FEATURE_NAMES: + self.assertIn( + f'{channel_name}_{feature_name}', + eeg_processing.EEG_FEATURE_NAMES, + ) + self.assertEqual( + len(eeg_processing.EEG_FEATURE_NAMES), + eeg_processing.EEG_FEATURE_LENGTH, + ) + + +if __name__ == '__main__': + unittest.main() From 1384b806593339e8ab5f1fffa8dd6261c020c0c7 Mon Sep 17 00:00:00 2001 From: rolopu1 Date: Mon, 20 Jul 2026 12:58:16 +0200 Subject: [PATCH 93/93] Update evaluator from official Challenge repository --- evaluate_model.py | 311 +++++++++++++++++++++++++++++++++++++++------- 1 file changed, 263 insertions(+), 48 deletions(-) diff --git a/evaluate_model.py b/evaluate_model.py index 6fc9466..5aff532 100644 --- a/evaluate_model.py +++ b/evaluate_model.py @@ -4,13 +4,16 @@ # This file contains functions for evaluating models for the Challenge. You can run it as follows: # -# python evaluate_model.py -d labels.csv -o predictions.csv -s scores.csv +# python evaluate_model.py -d labels.csv -o predictions.csv -p prevalence.csv -s scores.csv -t table.csv # -# where 'labels.csv' is a CSV file containing the labels, 'predictions.csv' is a CSV file containing containing the predictions, and -# 'scores.csv' (optional) is a collection of scores for the predictions. +# where 'labels.csv' is one or more CSV files containing the labels, 'predictions.csv' is one or more CSV files containing the +# predictions, 'prevalence.csv' is one or more CSV files containing labels to define the prevalence of the positive class at +# different ages, 'scores.csv' (optional) is a collection of scores for the predictions, and 'table.csv' (optional) is a table +# summary of the predictions at different ages. # -# The Challenge webpage describes the file formats and scoring functions. +# The Challenge webpage describes the file formats and scoring functions for this script. +# Import packages. import argparse import numpy as np import os @@ -18,102 +21,314 @@ import pandas as pd import sys -id_patients = 'BDSPPatientID' -id_labels = 'Cognitive_Impairment' -id_binary_predictions = 'Cognitive_Impairment' -id_probability_predictions = 'Cognitive_Impairment_Probability' +from collections import defaultdict +from sklearn.metrics import roc_auc_score, average_precision_score + +# Define headers. +id_site = 'SiteID' +id_patient = 'BDSPPatientID' +id_label = 'Cognitive_Impairment' +id_age = 'Age' +id_sex = 'Sex' +id_binary_prediction = 'Cognitive_Impairment' +id_probability_prediction = 'Cognitive_Impairment_Probability' # Parse arguments. def get_parser(): description = 'Evaluate the Challenge model.' parser = argparse.ArgumentParser(description=description) - parser.add_argument('-d', '--labels_folder', type=str, required=True) - parser.add_argument('-o', '--predictions_folder', type=str, required=True) + parser.add_argument('-d', '--labels_files', type=str, required=True, nargs='*') + parser.add_argument('-o', '--predictions_files', type=str, required=True, nargs='*') + parser.add_argument('-p', '--prevalence_files', type=str, required=True, nargs='*') parser.add_argument('-s', '--score_file', type=str, required=False) + parser.add_argument('-t', '--table_file', type=str, required=False) return parser -# Compute AUC. -def compute_auc(labels, predictions): - from sklearn.metrics import roc_auc_score, average_precision_score - auroc = roc_auc_score(labels, predictions, average='macro', sample_weight=None, max_fpr=None, multi_class='raise', labels=None) - auprc = average_precision_score(labels, predictions, average='macro', pos_label=1, sample_weight=None) - return auroc, auprc +# Compute the prevalence of the positive class at different ages. +def compute_prevalence(ages, prevalence_labels, prevalence_ages, gap=0): + m = len(ages) + n = len(prevalence_labels) + o = len(prevalence_ages) + assert(n == o) + + unique_ages = np.unique(ages[np.isfinite(ages)]) + age_to_labels = defaultdict(list) + for age in unique_ages: + for i in range(n): + if abs(age-prevalence_ages[i]) <= gap: + age_to_labels[age].append(prevalence_labels[i]) + + age_to_prevalence = dict() + for age in age_to_labels: + age_to_prevalence[age] = max(sum(age_to_labels[age]), 0.5)/len(age_to_labels[age]) + + return age_to_prevalence + +# Compute a prevalence-based reward metric. +def compute_reward(labels, predictions, ages, age_to_prevalence): + m = len(labels) + n = len(predictions) + o = len(ages) + assert(m == n == o) + + scores = np.zeros(n) + num_scores = 0 + + for i in range(n): + if np.isfinite(ages[i]): + p = age_to_prevalence[ages[i]] + p = min(max(p, 0.5/m), 1 - 0.5/m) # Ensure p is strictly greater than 0 and strictly less than 1. + + if labels[i] == 1 and predictions[i] == 1: + scores[i] = 1/p - 1 + elif labels[i] == 0 and predictions[i] == 1: + scores[i] = -1 + if labels[i] == 1 and predictions[i] == 0: + scores[i] = -1 + elif labels[i] == 0 and predictions[i] == 0: + scores[i] = 1/(1-p) - 1 + num_scores += 1 + + score = np.sum(scores)/num_scores + + return score + +# Compute the area under the receiver-operating characteristic curve conditioned on age. +def compute_auroc_age(labels, predictions, ages, gap=0): + m = len(labels) + n = len(predictions) + o = len(ages) + assert(m == n == o) + + idx_pos = [i for i in range(m) if labels[i] == 1] + idx_neg = [i for i in range(m) if labels[i] == 0] + num_pos = len(idx_pos) + num_neg = len(idx_neg) + + numer = 0 + denom = 0 + for i in range(num_pos): + for j in range(num_neg): + if abs(ages[idx_pos[i]] - ages[idx_neg[j]]) <= gap: + if predictions[idx_pos[i]] > predictions[idx_neg[j]]: + numer += 1 + elif predictions[idx_pos[i]] == predictions[idx_neg[j]]: + numer += 0.5 + denom += 1 + return numer/denom + +# Compute the age-weighted mean of the area under the receiver-operating characteristic curve across ages. +def compute_auroc_weighted(labels, predictions, ages, gap=0): + m = len(labels) + n = len(predictions) + o = len(ages) + assert(m == n == o) + + finite_ages = ages[np.isfinite(ages)] + range_ages = np.arange(np.min(finite_ages) - gap, np.max(finite_ages) + gap + 1) + p = len(finite_ages) + q = len(range_ages) + + idx_pos = [i for i in range(m) if labels[i] == 1] + idx_neg = [i for i in range(m) if labels[i] == 0] + num_pos = len(idx_pos) + num_neg = len(idx_neg) + + numer = np.zeros(q) + denom = np.zeros(q) + for k in range(q): + for i in range(num_pos): + for j in range(num_neg): + if abs(ages[idx_pos[i]] - range_ages[k]) <= gap and abs(ages[idx_neg[j]] - range_ages[k]) <= gap: + if predictions[idx_pos[i]] > predictions[idx_neg[j]]: + numer[k] += 1 + elif predictions[idx_pos[i]] == predictions[idx_neg[j]]: + numer[k] += 0.5 + denom[k] += 1 + + weights = np.zeros(q) + for k in range(q): + weights[k] = sum(1 for i in range(p) if abs(finite_ages[i] - range_ages[k]) <= gap) + + for k in range(q): + if denom[k] == 0: # Avoid NaN propagation + weights[k] = 0 + numer[k] = 0 + denom[k] = 1 + + weights = weights / np.sum(weights) + + return np.sum(weights * (numer / denom)) + +# Compute the area under the receiver-operating characteristic curve. +def compute_auroc(labels, predictions): + auroc = roc_auc_score(labels, predictions, sample_weight=None, max_fpr=None, multi_class='raise', labels=None) + return auroc + +# Compute the area under the precision-recall curve. +def compute_auprc(labels, predictions): + auprc = average_precision_score(labels, predictions, pos_label=1, sample_weight=None) + return auprc + +# Compute a confusion matrix. +def compute_confusion_matrix(labels, predictions): + n = np.size(labels) + tp = fp = fn = tn = 0 + for i in range(n): + if labels[i] == 1 and predictions[i] == 1: + tp += 1 + elif labels[i] == 0 and predictions[i] == 1: + fp += 1 + elif labels[i] == 1 and predictions[i] == 0: + fn += 1 + elif labels[i] == 0 and predictions[i] == 0: + tn += 1 + return tp, fp, fn, tn # Compute accuracy. def compute_accuracy(labels, predictions): - from sklearn.metrics import accuracy_score - accuracy = accuracy_score(labels, predictions, normalize=True, sample_weight=None) + tp, fp, fn, tn = compute_confusion_matrix(labels, predictions) + accuracy = (tp + tn) / (tp + fp + fn + tn) return accuracy -# Compute F-measure. +# Compute the F-measure. def compute_f_measure(labels, predictions): - from sklearn.metrics import f1_score - f_measure = f1_score(labels, predictions, pos_label=1, average='binary') + tp, fp, fn, tn = compute_confusion_matrix(labels, predictions) + f_measure = (2*tp) / (2*tp + fp + fn) return f_measure # Evaluate the models. -def evaluate_model(labels_file, predictions_file): - # Load the labels and predictions. - df_labels = pd.read_csv(labels_file) - df_labels.set_index(id_patients, inplace=True) - df_predictions = pd.read_csv(predictions_file) - df_predictions.set_index(id_patients, inplace=True) - - def standardize_bool(val): - s = str(val).strip().upper() - if s in ['TRUE', '1', '1.0', 'T', 'Y', 'YES']: return 1.0 - if s in ['FALSE', '0', '0.0', 'F', 'N', 'NO']: return 0.0 - return np.nan - - # Standardize the labels and predictions to be 0/1. - df_labels[id_labels] = df_labels[id_labels].apply(standardize_bool) - df_predictions[id_binary_predictions] = df_predictions[id_binary_predictions].apply(standardize_bool) +def evaluate_model(labels_files, predictions_files, prevalence_files): + # Load the labels, predictions, and prevalence data. + id_site_patient = id_site + '_' + id_patient + + df_labels = [pd.read_csv(labels_file) for labels_file in labels_files] + df_labels = pd.concat(df_labels) + df_labels.drop_duplicates(inplace=True) + df_labels[id_site_patient] = df_labels[id_site].astype(str) + '_' + df_labels[id_patient].astype(str) + df_labels.set_index(id_site_patient, inplace=True) + + df_predictions = [pd.read_csv(predictions_file) for predictions_file in predictions_files] + df_predictions = pd.concat(df_predictions) + df_predictions.drop_duplicates(inplace=True) + df_predictions[id_site_patient] = df_predictions[id_site].astype(str) + '_' + df_predictions[id_patient].astype(str) + df_predictions.set_index(id_site_patient, inplace=True) + + df_prevalence = [pd.read_csv(prevalence_file) for prevalence_file in prevalence_files] + df_prevalence = pd.concat(df_prevalence) + df_prevalence.drop_duplicates(inplace=True) + df_prevalence[id_site_patient] = df_prevalence[id_site].astype(str) + '_' + df_prevalence[id_patient].astype(str) + df_prevalence.set_index(id_site_patient, inplace=True) # Only consider patients with positive or negative labels. - df_labels = df_labels[(df_labels[id_labels] == 0) | (df_labels[id_labels] == 1)] + df_labels = df_labels[(df_labels[id_label] == 0) | (df_labels[id_label] == 1)] patients = df_labels.index num_patients = len(patients) - # Extract the labels and predictions. + # Extract the labels, predictions, and ages. labels = np.zeros(num_patients) binary_predictions = np.zeros(num_patients) probability_predictions = np.zeros(num_patients) + ages = np.zeros(num_patients) for i, patient in enumerate(patients): - label = df_labels.loc[patient, id_labels] + label = df_labels.loc[patient, id_label] labels[i] = label if patient in df_predictions.index: # Set missing predictions to 0. - binary_prediction = float(df_predictions.loc[patient, id_binary_predictions]) + binary_prediction = float(df_predictions.loc[patient, id_binary_prediction]) if binary_prediction == 0 or binary_prediction == 1: # Set invalid binary predictions to 0. binary_predictions[i] = binary_prediction - probability_prediction = float(df_predictions.loc[patient, id_probability_predictions]) + probability_prediction = float(df_predictions.loc[patient, id_probability_prediction]) if np.isfinite(probability_prediction): # Set invalid probability predictions to 0. probability_predictions[i] = probability_prediction + age = df_labels.loc[patient, id_age] + ages[i] = age + + # Validate the labels, predictions, and ages. + assert(labels.ndim == binary_predictions.ndim == probability_predictions.ndim == ages.ndim == 1) + assert(np.size(labels) == np.size(binary_predictions) == np.size(probability_predictions) == np.size(ages)) + assert(np.size(labels) > 0) + assert(np.all((labels==0) | (labels==1))) + assert(np.all((binary_predictions==0) | (binary_predictions==1))) + assert(np.all(~np.isnan(probability_predictions))) + + # Extract the prevalence of the positive class at different ages. + df_prevalence = df_prevalence[(df_prevalence[id_label] == 0) | (df_prevalence[id_label] == 1)] + prevalence_patients = df_prevalence.index + + num_prevalence_patients = len(prevalence_patients) + prevalence_labels = np.zeros(num_prevalence_patients) + prevalence_ages = np.zeros(num_prevalence_patients) + + for i, patient in enumerate(prevalence_patients): + label = df_prevalence.loc[patient, id_label] + prevalence_labels[i] = label + age = df_prevalence.loc[patient, id_age] + prevalence_ages[i] = age + + age_to_prevalence = compute_prevalence(ages, prevalence_labels, prevalence_ages, gap=2) # Evaluate the predictions. - auroc, auprc = compute_auc(labels, probability_predictions) + reward = compute_reward(labels, binary_predictions, ages, age_to_prevalence) + auroc_age = compute_auroc_age(labels, probability_predictions, ages, gap=2) + auroc_weighted = compute_auroc_weighted(labels, probability_predictions, ages, gap=2) + auroc = compute_auroc(labels, probability_predictions) + auprc = compute_auprc(labels, probability_predictions) accuracy = compute_accuracy(labels, binary_predictions) f_measure = compute_f_measure(labels, binary_predictions) + + table = list() + + header = ['Age', 'Prevalence (prevalence data)', '# positive labels (prevalence data)', '# negative labels (prevalence data)', \ + '# positive labels', '# negative labels', '# positive predictions', '# negative predictions', \ + '# true positives', '# false positives', '# false negatives', '# true negatives'] + table.append(header) - return auroc, auprc, accuracy, f_measure + m = len(prevalence_labels) + n = len(labels) + for age in np.unique(np.concatenate((ages, list(age_to_prevalence)))): + prevalence = age_to_prevalence[age] if age in age_to_prevalence else float('nan') + pa = sum(1 for i in range(m) if prevalence_ages[i] == age and prevalence_labels[i] == 1) + na = sum(1 for i in range(m) if prevalence_ages[i] == age and prevalence_labels[i] == 0) + pb = sum(1 for i in range(n) if ages[i] == age and labels[i] == 1) + nb = sum(1 for i in range(n) if ages[i] == age and labels[i] == 0) + pc = sum(1 for i in range(n) if ages[i] == age and binary_predictions[i] == 1) + nc = sum(1 for i in range(n) if ages[i] == age and binary_predictions[i] == 0) + tp = sum(1 for i in range(n) if ages[i] == age and labels[i] == 1 and binary_predictions[i] == 1) + fp = sum(1 for i in range(n) if ages[i] == age and labels[i] == 0 and binary_predictions[i] == 1) + fn = sum(1 for i in range(n) if ages[i] == age and labels[i] == 1 and binary_predictions[i] == 0) + tn = sum(1 for i in range(n) if ages[i] == age and labels[i] == 0 and binary_predictions[i] == 0) + row = [age, age_to_prevalence[age], pa, na, pb, nb, pc, nc, tp, fp, fn, tn] + table.append(row) + + return reward, auroc_age, auroc_weighted, auroc, auprc, accuracy, f_measure, table # Run the code. def run(args): # Compute the scores for the model predictions. - auroc, auprc, accuracy, f_measure = evaluate_model(args.labels_folder, args.predictions_folder) + reward, auroc_age, auroc_weighted, auroc, auprc, accuracy, f_measure, table = evaluate_model(args.labels_files, args.predictions_files, args.prevalence_files) output_string = \ - f'AUROC: {auroc:.3f}\n' \ + f'Reward: {reward:.3f}\n' + \ + f'Age-conditioned AUROC : {auroc_age:.3f}\n' + \ + f'Age-weighted AUROC : {auroc_weighted:.3f}\n' + \ + f'AUROC: {auroc:.3f}\n' + \ f'AUPRC: {auprc:.3f}\n' + \ - f'Accuracy: {accuracy:.3f}\n' \ + f'Accuracy: {accuracy:.3f}\n' + \ f'F-measure: {f_measure:.3f}\n' - # Output the scores to screen and/or a file. + # Output the scores to the screen or a file. if args.score_file: with open(args.score_file, 'w') as f: f.write(output_string) else: print(output_string) + # Output a table a breakdown of the results by age to a file. + if args.table_file: + with open(args.table_file, 'w') as f: + table_string = '\n'.join('\t'.join(map(str, row)) for row in table) + f.write(table_string) + if __name__ == '__main__': run(get_parser().parse_args(sys.argv[1:]))