diff --git a/README.rst b/README.rst index 0db98e4..01b1d1a 100644 --- a/README.rst +++ b/README.rst @@ -59,8 +59,8 @@ Storage backends: HDF5 and TileDB ModelArrayIO supports two on-disk backends for the subject-by-element matrix: -* HDF5 (default), implemented in ``modelarrayio/h5_storage.py`` -* TileDB, implemented in ``modelarrayio/tiledb_storage.py`` +* HDF5 (default), implemented in ``src/modelarrayio/storage/h5_storage.py`` +* TileDB, implemented in ``src/modelarrayio/storage/tiledb_storage.py`` Both backends expose a similar API: @@ -71,6 +71,6 @@ Both backends expose a similar API: Notes and minor differences: * Chunking vs tiling: HDF5 uses chunks; TileDB uses tiles. We compute tile sizes analogous to chunk sizes to keep write/read patterns similar. -* Compression: HDF5 uses ``gzip`` by default; TileDB defaults to ``zstd`` with shuffle for better speed/ratio. You can switch to ``gzip`` for parity. +* Compression: HDF5 uses ``gzip`` by default; TileDB defaults to ``gzip`` with shuffle for better speed/ratio. You can switch to ``gzip`` for parity. * Metadata: HDF5 stores ``column_names`` as a dataset attribute; TileDB stores names as JSON metadata on the array/group. * Layout: Both backends keep dimensions in the same order and use zero-based indices. diff --git a/docs/examples/README.rst b/docs/examples/README.rst index 792b21c..acce9b9 100644 --- a/docs/examples/README.rst +++ b/docs/examples/README.rst @@ -2,4 +2,5 @@ Walkthroughs ============ Step-by-step guides for converting neuroimaging data to and from the HDF5 -format used by `ModelArray `_. +format used by `ModelArray `_. Each +workflow includes cohort CSV examples in both long and wide formats. diff --git a/docs/examples/plot_cifti_workflow.py b/docs/examples/plot_cifti_workflow.py index 397aa4a..6ac8e55 100644 --- a/docs/examples/plot_cifti_workflow.py +++ b/docs/examples/plot_cifti_workflow.py @@ -15,12 +15,16 @@ # # To convert a list of CIFTI files to ``.h5`` format, you need: # -# 1. **A cohort CSV** describing every CIFTI file to include (one CSV per scalar recommended). +# 1. **A cohort CSV** describing every CIFTI file to include. # -# Cohort CSV columns (names are fixed, not user-defined): +# The cohort CSV may use either of these layouts: # -# * ``scalar_name`` — which metric is being analysed (e.g., ``FA``) -# * ``source_file`` — path to the subject's CIFTI file +# * **Long format:** one row per subject and scalar, with the fixed columns ``scalar_name`` and +# ``source_file``. +# * **Wide format:** one row per subject, with a separate file-path column for each scalar +# (for example, ``FA`` and ``MD``). Pass those column names to ``--scalar-columns``. +# +# Other columns, such as subject IDs and demographics, may appear alongside the path columns. # %% # Example folder structure @@ -30,7 +34,8 @@ # # /home/username/myProject/data # | -# ├── cohort_FA.csv +# ├── cohort_long.csv +# ├── cohort_wide.csv # │ # ├── FA # │ ├── sub-01_FA.dscalar.nii @@ -38,9 +43,18 @@ # │ ├── sub-03_FA.dscalar.nii # │ └── ... # │ +# ├── MD +# │ ├── sub-01_MD.dscalar.nii +# │ ├── sub-02_MD.dscalar.nii +# │ ├── sub-03_MD.dscalar.nii +# │ └── ... +# │ # └── ... # -# Corresponding ``cohort_FA.csv`` for scalar FA: +# Long-format cohort CSV +# ---------------------- +# +# Corresponding ``cohort_long.csv`` for scalars FA and MD: # # .. list-table:: # :header-rows: 1 @@ -66,6 +80,21 @@ # - sub-03 # - 15 # - F +# * - MD +# - /home/username/myProject/data/MD/sub-01_MD.dscalar.nii +# - sub-01 +# - 10 +# - F +# * - MD +# - /home/username/myProject/data/MD/sub-02_MD.dscalar.nii +# - sub-02 +# - 20 +# - M +# * - MD +# - /home/username/myProject/data/MD/sub-03_MD.dscalar.nii +# - sub-03 +# - 15 +# - F # * - ... # - ... # - ... @@ -79,10 +108,46 @@ # between the CSV and disk. # %% -# Convert CIFTI files to HDF5 -# --------------------------- +# Wide-format cohort CSV +# ---------------------- +# +# A wide CSV stores all scalar paths for a subject on one row. For example, +# ``cohort_wide.csv`` can contain both FA and MD: # -# Using the FA dataset from the example above: +# .. list-table:: +# :header-rows: 1 +# :widths: auto +# +# * - subject_id +# - **FA** +# - **MD** +# - age +# - sex +# * - sub-01 +# - /home/username/myProject/data/FA/sub-01_FA.dscalar.nii +# - /home/username/myProject/data/MD/sub-01_MD.dscalar.nii +# - 10 +# - F +# * - sub-02 +# - /home/username/myProject/data/FA/sub-02_FA.dscalar.nii +# - /home/username/myProject/data/MD/sub-02_MD.dscalar.nii +# - 20 +# - M +# * - ... +# - ... +# - ... +# - ... +# - ... +# +# ``FA`` and ``MD`` are user-defined scalar column names. All files for a scalar must use +# compatible CIFTI axes. + +# %% +# Convert a long-format cohort +# ---------------------------- +# +# Long-format cohorts combine all scalars into one output by default. The +# ``--no-split-files`` flag below makes that choice explicit: # # .. code-block:: console # @@ -90,18 +155,58 @@ # conda activate # # modelarrayio to-modelarray \ -# --cohort-file /home/username/myProject/data/cohort_FA.csv \ -# --output /home/username/myProject/data/FA.h5 +# --cohort-file /home/username/myProject/data/cohort_long.csv \ +# --no-split-files \ +# --output /home/username/myProject/data/modelarray.h5 +# +# This creates one ``modelarray.h5`` containing both ``scalars/FA`` and ``scalars/MD``. +# To write one file per scalar instead, use ``--split-files`` with the same output basename: +# +# .. code-block:: console +# +# modelarrayio to-modelarray \ +# --cohort-file /home/username/myProject/data/cohort_long.csv \ +# --split-files \ +# --output /home/username/myProject/data/modelarray.h5 +# +# The split command creates ``FA_modelarray.h5`` and ``MD_modelarray.h5``. You can then use +# `ModelArray `_ to run statistical analyses on either +# the combined output or the scalar-specific outputs. + +# %% +# Convert a wide-format cohort +# ---------------------------- +# +# Name each scalar path column with ``--scalar-columns``: +# +# .. code-block:: console +# +# modelarrayio to-modelarray \ +# --cohort-file /home/username/myProject/data/cohort_wide.csv \ +# --scalar-columns FA MD \ +# --output /home/username/myProject/data/modelarray.h5 +# +# Wide cohorts write one output per scalar by default. This command creates +# ``FA_modelarray.h5`` and ``MD_modelarray.h5``. To override that default, add +# ``--no-split-files``: +# +# .. code-block:: console +# +# modelarrayio to-modelarray \ +# --cohort-file /home/username/myProject/data/cohort_wide.csv \ +# --scalar-columns FA MD \ +# --no-split-files \ +# --output /home/username/myProject/data/modelarray.h5 # -# This produces ``FA.h5`` in ``/home/username/myProject/data``. You can then use -# `ModelArray `_ to run statistical analyses on it. +# This creates one ``modelarray.h5`` containing both scalar groups. # %% # Convert result .h5 back to CIFTI # -------------------------------- # -# After running **ModelArray** and obtaining statistical results inside ``FA.h5`` (suppose the -# analysis name is ``"mylm"``), use ``modelarrayio export-results`` to export them as CIFTI files. +# After running **ModelArray** and obtaining statistical results inside ``FA_modelarray.h5`` +# (suppose the analysis name is ``"mylm"``), use ``modelarrayio export-results`` to export them +# as CIFTI files. # # Supply either ``--cohort-file`` (the first ``source_file`` entry is used as a header template) # or ``--example-file`` (an explicit template path) — these two flags are mutually exclusive. @@ -109,9 +214,9 @@ # .. code-block:: console # # modelarrayio export-results \ -# --cohort-file /home/username/myProject/data/cohort_FA.csv \ +# --cohort-file /home/username/myProject/data/cohort_long.csv \ # --analysis-name mylm \ -# --input-hdf5 /home/username/myProject/data/FA.h5 \ +# --input-hdf5 /home/username/myProject/data/FA_modelarray.h5 \ # --output-dir /home/username/myProject/data/FA_stats # # All converted volume data are saved as ``float32``. Results in ``FA_stats`` can be viewed diff --git a/docs/examples/plot_fixel_workflow.py b/docs/examples/plot_fixel_workflow.py index dcef1f7..a91ce5a 100644 --- a/docs/examples/plot_fixel_workflow.py +++ b/docs/examples/plot_fixel_workflow.py @@ -14,13 +14,16 @@ # ------------ # # To convert a list of fixel-wise data from ``.mif`` to ``.h5`` format, prepare a cohort CSV -# file that describes every ``.mif`` file you want to include. We recommend one CSV per scalar -# (e.g. FD, FC, FDC), yielding one ``.h5`` file per scalar. +# file that describes every ``.mif`` file you want to include. # -# Cohort CSV columns (names are fixed, not user-defined): +# The cohort CSV may use either of these layouts: # -# * ``scalar_name`` — which metric is being analysed (e.g. ``FD``, ``FC``, ``FDC``) -# * ``source_file`` — path to the ``.mif`` file for this subject +# * **Long format:** one row per subject and scalar, with the fixed columns ``scalar_name`` and +# ``source_file``. +# * **Wide format:** one row per subject, with a separate file-path column for each scalar +# (for example, ``FD`` and ``FC``). Pass those column names to ``--scalar-columns``. +# +# Other columns, such as subject IDs and demographics, may appear alongside the path columns. # %% # Example folder structure @@ -30,7 +33,8 @@ # # /home/username/myProject/data # | -# ├── cohort_FD.csv +# ├── cohort_long.csv +# ├── cohort_wide.csv # │ # ├── FD # │ ├── index.mif @@ -51,7 +55,10 @@ # # These ``.mif`` files are generated by MRtrix. # -# Corresponding ``cohort_FD.csv`` for scalar FD: +# Long-format cohort CSV +# ---------------------- +# +# Corresponding ``cohort_long.csv`` for scalars FD and FC: # # .. list-table:: # :header-rows: 1 @@ -77,6 +84,21 @@ # - sub-03 # - 15 # - F +# * - FC +# - /home/username/myProject/data/FC/sub-01_fc.mif +# - sub-01 +# - 10 +# - F +# * - FC +# - /home/username/myProject/data/FC/sub-02_fc.mif +# - sub-02 +# - 20 +# - M +# * - FC +# - /home/username/myProject/data/FC/sub-03_fc.mif +# - sub-03 +# - 15 +# - F # * - ... # - ... # - ... @@ -90,10 +112,46 @@ # between the CSV and disk. # %% -# Convert .mif files to HDF5 -# -------------------------- +# Wide-format cohort CSV +# ---------------------- +# +# A wide CSV stores all scalar paths for a subject on one row. For example, +# ``cohort_wide.csv`` can contain both FD and FC: +# +# .. list-table:: +# :header-rows: 1 +# :widths: auto +# +# * - subject_id +# - **FD** +# - **FC** +# - age +# - sex +# * - sub-01 +# - /home/username/myProject/data/FD/sub-01_fd.mif +# - /home/username/myProject/data/FC/sub-01_fc.mif +# - 10 +# - F +# * - sub-02 +# - /home/username/myProject/data/FD/sub-02_fd.mif +# - /home/username/myProject/data/FC/sub-02_fc.mif +# - 20 +# - M +# * - ... +# - ... +# - ... +# - ... +# - ... +# +# ``FD`` and ``FC`` are user-defined scalar column names. The files must use the fixel layout +# described by the supplied ``index.mif`` and ``directions.mif`` files. + +# %% +# Convert a long-format cohort +# ---------------------------- # -# Using the FD dataset from the example above: +# Long-format cohorts combine all scalars into one output by default. The +# ``--no-split-files`` flag below makes that choice explicit: # # .. code-block:: console # @@ -103,29 +161,74 @@ # modelarrayio to-modelarray \ # --index-file /home/username/myProject/data/FD/index.mif \ # --directions-file /home/username/myProject/data/FD/directions.mif \ -# --cohort-file /home/username/myProject/data/cohort_FD.csv \ -# --output /home/username/myProject/data/FD.h5 +# --cohort-file /home/username/myProject/data/cohort_long.csv \ +# --no-split-files \ +# --output /home/username/myProject/data/modelarray.h5 +# +# This creates one ``modelarray.h5`` containing both ``scalars/FD`` and ``scalars/FC``. +# To write one file per scalar instead, use ``--split-files`` with the same output basename: +# +# .. code-block:: console +# +# modelarrayio to-modelarray \ +# --index-file /home/username/myProject/data/FD/index.mif \ +# --directions-file /home/username/myProject/data/FD/directions.mif \ +# --cohort-file /home/username/myProject/data/cohort_long.csv \ +# --split-files \ +# --output /home/username/myProject/data/modelarray.h5 # -# This produces ``FD.h5`` in ``/home/username/myProject/data``. You can then use -# `ModelArray `_ to run statistical analyses on it. +# The split command creates ``FD_modelarray.h5`` and ``FC_modelarray.h5``. You can then use +# `ModelArray `_ to run statistical analyses on either +# the combined output or the scalar-specific outputs. + +# %% +# Convert a wide-format cohort +# ---------------------------- +# +# Name each scalar path column with ``--scalar-columns``: +# +# .. code-block:: console +# +# modelarrayio to-modelarray \ +# --index-file /home/username/myProject/data/FD/index.mif \ +# --directions-file /home/username/myProject/data/FD/directions.mif \ +# --cohort-file /home/username/myProject/data/cohort_wide.csv \ +# --scalar-columns FD FC \ +# --output /home/username/myProject/data/modelarray.h5 +# +# Wide cohorts write one output per scalar by default. This command creates +# ``FD_modelarray.h5`` and ``FC_modelarray.h5``. To override that default, add +# ``--no-split-files``: +# +# .. code-block:: console +# +# modelarrayio to-modelarray \ +# --index-file /home/username/myProject/data/FD/index.mif \ +# --directions-file /home/username/myProject/data/FD/directions.mif \ +# --cohort-file /home/username/myProject/data/cohort_wide.csv \ +# --scalar-columns FD FC \ +# --no-split-files \ +# --output /home/username/myProject/data/modelarray.h5 +# +# This creates one ``modelarray.h5`` containing both scalar groups. # %% # Convert result .h5 back to .mif files # -------------------------------------- # -# After running ModelArray and obtaining statistical results inside ``FD.h5`` (suppose the -# analysis name is ``"mylm"``), use ``modelarrayio export-results`` to export them as ``.mif`` files. -# The command also copies the original ``index.mif`` and ``directions.mif`` into the output -# folder. +# After running ModelArray and obtaining statistical results inside ``FD_modelarray.h5`` +# (suppose the analysis name is ``"mylm"``), use ``modelarrayio export-results`` to export them +# as ``.mif`` files. The command also copies the original ``index.mif`` and ``directions.mif`` +# into the output folder. # # .. code-block:: console # # modelarrayio export-results \ # --index-file /home/username/myProject/data/FD/index.mif \ # --directions-file /home/username/myProject/data/FD/directions.mif \ -# --cohort-file /home/username/myProject/data/cohort_FD.csv \ +# --cohort-file /home/username/myProject/data/cohort_long.csv \ # --analysis-name mylm \ -# --input-hdf5 /home/username/myProject/data/FD.h5 \ +# --input-hdf5 /home/username/myProject/data/FD_modelarray.h5 \ # --output-dir /home/username/myProject/data/FD_stats # # The results in ``FD_stats`` can now be viewed in ``mrview``. diff --git a/docs/examples/plot_voxel_workflow.py b/docs/examples/plot_voxel_workflow.py index 290ddd0..2155bce 100644 --- a/docs/examples/plot_voxel_workflow.py +++ b/docs/examples/plot_voxel_workflow.py @@ -15,17 +15,22 @@ # # To convert a list of NIfTI files to ``.h5`` format, you need: # -# 1. **A cohort CSV** describing every NIfTI file to include (one CSV per scalar recommended). +# 1. **A cohort CSV** describing every NIfTI file to include. # 2. **A group mask** — only voxels inside the group mask are kept during conversion. -# 3. **Subject-specific masks** *(optional)* — voxels outside each subject's mask are set to -# ``NaN`` after conversion. If you do not have per-subject masks, supply the group mask for -# every subject (see the CSV example below). +# 3. **Subject-specific masks** — voxels outside each subject's mask are set to ``NaN`` after +# conversion. Every cohort row must provide a mask path in ``source_mask_file``. If you do +# not have distinct subject masks, use the group mask path for each row. # -# Cohort CSV columns (names are fixed, not user-defined): +# The cohort CSV may use either of these layouts: # -# * ``scalar_name`` — which metric is being analysed (e.g., ``FA``) -# * ``source_file`` — path to the subject's NIfTI file -# * ``source_mask_file`` — path to the subject-specific mask (or the group mask if none exists) +# * **Long format:** one row per subject and scalar, with the fixed columns ``scalar_name`` and +# ``source_file``. +# * **Wide format:** one row per subject, with a separate file-path column for each scalar +# (for example, ``FA`` and ``MD``). Pass those column names to ``--scalar-columns``. +# +# In either layout, the ``source_mask_file`` column and a valid mask path for every row are +# required. Other columns, such as subject IDs and demographics, may appear alongside the path +# columns. # %% # Example folder structure @@ -35,7 +40,8 @@ # # /home/username/myProject/data # | -# ├── cohort_FA.csv +# ├── cohort_long.csv +# ├── cohort_wide.csv # ├── group_mask.nii.gz # │ # ├── FA @@ -44,6 +50,12 @@ # │ ├── sub-03_FA.nii.gz # │ └── ... # │ +# ├── MD +# │ ├── sub-01_MD.nii.gz +# │ ├── sub-02_MD.nii.gz +# │ ├── sub-03_MD.nii.gz +# │ └── ... +# │ # ├── individual_masks # │ ├── sub-01_mask.nii.gz # │ ├── sub-02_mask.nii.gz @@ -51,7 +63,10 @@ # │ └── ... # └── ... # -# Corresponding ``cohort_FA.csv`` for scalar FA: +# Long-format cohort CSV +# ---------------------- +# +# Corresponding ``cohort_long.csv`` for scalars FA and MD: # # .. list-table:: # :header-rows: 1 @@ -81,6 +96,24 @@ # - sub-03 # - 15 # - F +# * - MD +# - /home/username/myProject/data/MD/sub-01_MD.nii.gz +# - /home/username/myProject/data/individual_masks/sub-01_mask.nii.gz +# - sub-01 +# - 10 +# - F +# * - MD +# - /home/username/myProject/data/MD/sub-02_MD.nii.gz +# - /home/username/myProject/data/individual_masks/sub-02_mask.nii.gz +# - sub-02 +# - 20 +# - M +# * - MD +# - /home/username/myProject/data/MD/sub-03_MD.nii.gz +# - /home/username/myProject/data/individual_masks/sub-03_mask.nii.gz +# - sub-03 +# - 15 +# - F # * - ... # - ... # - ... @@ -91,14 +124,56 @@ # Notes: # # * Column order does not matter. +# * ``source_mask_file`` must contain a valid subject-mask path for every row. +# * ``--mask`` separately supplies the required group mask. # * Values are case-sensitive — folder names, file names, and scalar names must match exactly # between the CSV and disk. # %% -# Convert NIfTI files to HDF5 -# --------------------------- +# Wide-format cohort CSV +# ---------------------- # -# Using the FA dataset from the example above: +# A wide CSV stores all scalar paths for a subject on one row. For example, +# ``cohort_wide.csv`` can contain both FA and MD: +# +# .. list-table:: +# :header-rows: 1 +# :widths: auto +# +# * - subject_id +# - **FA** +# - **MD** +# - **source_mask_file** *(required)* +# - age +# - sex +# * - sub-01 +# - /home/username/myProject/data/FA/sub-01_FA.nii.gz +# - /home/username/myProject/data/MD/sub-01_MD.nii.gz +# - /home/username/myProject/data/individual_masks/sub-01_mask.nii.gz +# - 10 +# - F +# * - sub-02 +# - /home/username/myProject/data/FA/sub-02_FA.nii.gz +# - /home/username/myProject/data/MD/sub-02_MD.nii.gz +# - /home/username/myProject/data/individual_masks/sub-02_mask.nii.gz +# - 20 +# - M +# * - ... +# - ... +# - ... +# - ... +# - ... +# - ... +# +# ``FA`` and ``MD`` are user-defined scalar column names. The required ``source_mask_file`` +# value is applied to every scalar for that subject. + +# %% +# Convert a long-format cohort +# ---------------------------- +# +# Long-format cohorts combine all scalars into one output by default. The +# ``--no-split-files`` flag below makes that choice explicit: # # .. code-block:: console # @@ -106,26 +181,69 @@ # conda activate # # modelarrayio to-modelarray \ -# --mask /home/username/myProject/data/group_mask.nii.gz \ -# --cohort-file /home/username/myProject/data/cohort_FA.csv \ -# --output /home/username/myProject/data/FA.h5 +# --mask /home/username/myProject/data/group_mask.nii.gz \ +# --cohort-file /home/username/myProject/data/cohort_long.csv \ +# --no-split-files \ +# --output /home/username/myProject/data/modelarray.h5 +# +# This creates one ``modelarray.h5`` containing both ``scalars/FA`` and ``scalars/MD``. +# To write one file per scalar instead, use ``--split-files`` with the same output basename: +# +# .. code-block:: console +# +# modelarrayio to-modelarray \ +# --mask /home/username/myProject/data/group_mask.nii.gz \ +# --cohort-file /home/username/myProject/data/cohort_long.csv \ +# --split-files \ +# --output /home/username/myProject/data/modelarray.h5 +# +# The split command creates ``FA_modelarray.h5`` and ``MD_modelarray.h5``. You can then use +# `ModelArray `_ to run statistical analyses on either +# the combined output or the scalar-specific outputs. + +# %% +# Convert a wide-format cohort +# ---------------------------- +# +# Name each scalar path column with ``--scalar-columns``: +# +# .. code-block:: console +# +# modelarrayio to-modelarray \ +# --mask /home/username/myProject/data/group_mask.nii.gz \ +# --cohort-file /home/username/myProject/data/cohort_wide.csv \ +# --scalar-columns FA MD \ +# --output /home/username/myProject/data/modelarray.h5 +# +# Wide cohorts write one output per scalar by default. This command creates +# ``FA_modelarray.h5`` and ``MD_modelarray.h5``. To override that default, add +# ``--no-split-files``: +# +# .. code-block:: console +# +# modelarrayio to-modelarray \ +# --mask /home/username/myProject/data/group_mask.nii.gz \ +# --cohort-file /home/username/myProject/data/cohort_wide.csv \ +# --scalar-columns FA MD \ +# --no-split-files \ +# --output /home/username/myProject/data/modelarray.h5 # -# This produces ``FA.h5`` in ``/home/username/myProject/data``. You can then use -# `ModelArray `_ to run statistical analyses on it. +# This creates one ``modelarray.h5`` containing both scalar groups. # %% # Convert result .h5 back to NIfTI # -------------------------------- # -# After running **ModelArray** and obtaining statistical results inside ``FA.h5`` (suppose the -# analysis name is ``"mylm"``), use ``modelarrayio export-results`` to export them as NIfTI files. +# After running **ModelArray** and obtaining statistical results inside ``FA_modelarray.h5`` +# (suppose the analysis name is ``"mylm"``), use ``modelarrayio export-results`` to export them +# as NIfTI files. # # .. code-block:: console # # modelarrayio export-results \ # --mask /home/username/myProject/data/group_mask.nii.gz \ # --analysis-name mylm \ -# --input-hdf5 /home/username/myProject/data/FA.h5 \ +# --input-hdf5 /home/username/myProject/data/FA_modelarray.h5 \ # --output-dir /home/username/myProject/data/FA_stats # # All converted volume data are saved as ``float32`` and compressed (``.nii.gz``) by default. diff --git a/docs/outputs.rst b/docs/outputs.rst index c124c5d..8358c05 100644 --- a/docs/outputs.rst +++ b/docs/outputs.rst @@ -26,16 +26,86 @@ The commands fall into two groups: Output splitting ================ -By default, wide cohorts supplied with ``--scalar-columns`` write one output per -scalar, while long-format cohorts write all scalars to one combined output. Override -either default explicitly: +CSV layout and output splitting are independent choices: -- ``--split-files`` writes one output file or TileDB directory per scalar. -- ``--no-split-files`` writes all scalars to one combined output. +- A **long-format** CSV contains the fixed ``scalar_name`` and ``source_file`` + columns. Do not pass ``--scalar-columns``. Multiple scalar names may appear in + the same CSV. +- A **wide-format** CSV contains one path column per scalar. Pass all of those + column names to ``--scalar-columns``. -For example, ``--split-files --output modelarray.h5`` with scalars ``alpha`` and -``beta`` writes ``alpha_modelarray.h5`` and ``beta_modelarray.h5``. The same prefix -rule applies to TileDB output paths. +The CSV layout selects the default output behavior, while ``--split-files`` and +``--no-split-files`` explicitly override that default: + +.. list-table:: Output behavior by cohort layout + :header-rows: 1 + :widths: auto + + * - Cohort input + - No splitting flag + - ``--split-files`` + - ``--no-split-files`` + * - Long format + - One combined output + - One output per scalar + - One combined output + * - Wide format with ``--scalar-columns`` + - One output per scalar + - One output per scalar + - One combined output + +The flags control only how the converted scalars are packaged; they do not change +how rows or scalar path columns are interpreted. + +The commands below focus on splitting behavior and omit modality-specific arguments +such as ``--mask``, ``--index-file``, and ``--directions-file``. + +Combined outputs +---------------- + +A combined output uses exactly the path given to ``--output``. For example, a +long-format cohort containing ``FA`` and ``MD`` writes both ``scalars/FA`` and +``scalars/MD`` inside ``modelarray.h5`` with this command: + +.. code-block:: console + + modelarrayio to-modelarray \ + --cohort-file cohort_long.csv \ + --no-split-files \ + --output modelarray.h5 + +Because combined output is the long-format default, omitting ``--no-split-files`` +from this command produces the same result. + +Split outputs +------------- + +Split output prefixes the requested output name with each scalar name: + +.. code-block:: console + + modelarrayio to-modelarray \ + --cohort-file cohort_long.csv \ + --split-files \ + --output modelarray.h5 + +For scalars ``FA`` and ``MD``, this writes ``FA_modelarray.h5`` and +``MD_modelarray.h5``. Each output contains only its named scalar. With +``--backend tiledb --output modelarray.tdb``, the corresponding directories are +``FA_modelarray.tdb`` and ``MD_modelarray.tdb``. + +For a wide cohort, the equivalent default command is: + +.. code-block:: console + + modelarrayio to-modelarray \ + --cohort-file cohort_wide.csv \ + --scalar-columns FA MD \ + --output modelarray.h5 + +It also writes ``FA_modelarray.h5`` and ``MD_modelarray.h5``. Adding +``--no-split-files`` instead writes both scalars into the single +``modelarray.h5`` output. *********************** diff --git a/src/modelarrayio/cli/to_modelarray.py b/src/modelarrayio/cli/to_modelarray.py index 6b39e83..bba8371 100644 --- a/src/modelarrayio/cli/to_modelarray.py +++ b/src/modelarrayio/cli/to_modelarray.py @@ -135,7 +135,8 @@ def _parse_to_modelarray(): nargs='+', help=( 'Column names containing scalar file paths when the cohort table is in wide format. ' - 'If omitted, the cohort file must include "scalar_name" and "source_file" columns.' + 'If omitted, the cohort file must include "scalar_name" and "source_file" columns. ' + 'Wide-format cohorts split outputs by default; use --no-split-files to combine them.' ), ) split_group = parser.add_mutually_exclusive_group() @@ -144,14 +145,20 @@ def _parse_to_modelarray(): '--split_files', dest='split_outputs', action='store_true', - help='Write one output file or TileDB directory per scalar.', + help=( + 'Write one output file or TileDB directory per scalar. This overrides the combined ' + 'output default for long-format cohorts.' + ), ) split_group.add_argument( '--no-split-files', '--no_split_files', dest='split_outputs', action='store_false', - help='Write all scalars to one combined output.', + help=( + 'Write all scalars to one combined output. This overrides the split output default ' + 'for wide-format cohorts.' + ), ) parser.set_defaults(split_outputs=None) parser.add_argument(