diff --git a/mkdocs.yml b/mkdocs.yml index c81a99bc26..de7ed1704e 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -47,6 +47,7 @@ nav: - Coordinate systems: appendices/coordinate-systems.md - Quantitative MRI: appendices/qmri.md - Arterial Spin Labeling: appendices/arterial-spin-labeling.md + - Media files: appendices/media-files.md - Cross modality correspondence: appendices/cross-modality-correspondence.md - Changelog: CHANGES.md - The BIDS Website: diff --git a/src/appendices/media-files.md b/src/appendices/media-files.md new file mode 100644 index 0000000000..24ff988d9d --- /dev/null +++ b/src/appendices/media-files.md @@ -0,0 +1,165 @@ +# Media Files + +## Introduction + +Several BIDS datatypes make use of media files — audio recordings, video recordings, +combined audio-video recordings, and still images. +This appendix defines the common file formats, metadata conventions, +and codec identification schemes shared across all datatypes that use media files. + +The following media suffixes are defined: + +{{ MACROS___make_suffix_table(["audio", "video", "audiovideo", "image"]) }} + +Datatypes that incorporate media files (for example, behavioral recordings or stimuli) +define their own file-naming rules, directory placement, and datatype-specific metadata. +The conventions described here apply uniformly to all such datatypes. + +### Relationship to the `photo` suffix + +The media file definitions introduced here generalize the concept of all media in BIDS. +The existing `photo` suffix (used for photographs of anatomical landmarks, +head localization coils, and tissue samples) predates this framework and covers +a narrower use case — still images in specific electrophysiology and microscopy datatypes. + +The media suffixes (`audio`, `video`, `audiovideo`, `image`) are intended as the +general-purpose mechanism for all media content in BIDS. +In practice, a "photo" could equally be a video of an experimental setup with verbal +narration, an audio recording describing electrode placement, or a drawing rather than +a photograph. +The media file framework should be generally adopted for new datatypes, +and a future proposal may deprecate the `photo` suffix in favor of the broader `image` +suffix with appropriate migration tooling +(see [bids-utils](https://github.com/bids-standard/bids-utils)). + +## Supported Formats + +### Audio formats + +{{ MACROS___make_extension_table(["wav", "mp3", "aac", "ogg"]) }} + +### Video container formats + +{{ MACROS___make_extension_table(["mp4", "avi", "mkv", "webm"]) }} + +### Image formats + +{{ MACROS___make_extension_table(["jpg", "png", "svg", "webp", "tif", "tiff"]) }} + +When choosing a format, consider the trade-off between file size and data fidelity. +Uncompressed or lossless formats (WAV, PNG, TIFF) preserve full quality +but produce larger files. +Lossy formats (MP3, AAC, JPEG) significantly reduce file size +at the cost of some data loss. + +## Media Stream Metadata + +Media files SHOULD be accompanied by a JSON sidecar file +containing technical metadata about the media streams. +The following metadata fields are defined for media files. + +### Duration + +Applies to suffixes: `audio`, `video`, `audiovideo`. + +{{ MACROS___make_sidecar_table("media.MediaDuration") }} + +`RecordingDuration` reuses the existing BIDS metadata field already defined for +electrophysiology recordings (EEG, iEEG, MEG, and others). + +### Audio stream properties + +Applies to suffixes: `audio`, `audiovideo`. + +{{ MACROS___make_sidecar_table("media.MediaAudioProperties") }} + +Note: `AudioSampleRate` is used instead of the existing `SamplingFrequency` field +because audio-video files require distinguishing the audio sampling rate from the +video frame rate. The `Audio` prefix makes this unambiguous in multi-stream containers. + +### Visual properties + +Applies to suffixes: `video`, `audiovideo`, `image`. + +{{ MACROS___make_sidecar_table("media.MediaVisualProperties") }} + +### Video stream properties + +Applies to suffixes: `video`, `audiovideo`. + +{{ MACROS___make_sidecar_table("media.MediaVideoProperties") }} + +## Codec Identification + +Codec identification uses two complementary naming systems: + +### FFmpeg codec names (RECOMMENDED) + +The `AudioCodec` and `VideoCodec` fields use +[FFmpeg codec names](https://www.ffmpeg.org/ffmpeg-codecs.html) as the RECOMMENDED +convention. These names are the de facto standard in scientific computing and can be +auto-extracted from media files using: + +```bash +ffprobe -v quiet -print_format json -show_streams +``` + +### RFC 6381 codec strings (OPTIONAL) + +The `AudioCodecRFC6381` and `VideoCodecRFC6381` fields use +[RFC 6381](https://datatracker.ietf.org/doc/html/rfc6381) codec strings. +These provide precise codec profile and level information useful for +web and broadcast interoperability. + +### Common codec reference + +| Codec | FFmpeg Name | RFC 6381 String | Notes | +| -------------- | ----------- | ------------------ | ----------------------- | +| H.264 / AVC | `h264` | `avc1.640028` | Most widely supported | +| H.265 / HEVC | `hevc` | `hev1.1.6.L93.B0` | High efficiency | +| VP9 | `vp9` | `vp09.00.10.08` | Open, royalty-free | +| AV1 | `av1` | `av01.0.01M.08` | Next-gen open codec | +| AAC-LC | `aac` | `mp4a.40.2` | Default audio for MP4 | +| MP3 | `mp3` | `mp4a.6B` | Legacy lossy audio | +| Opus | `opus` | `Opus` | Open, low-latency audio | +| FLAC | `flac` | `fLaC` | Open lossless audio | +| PCM 16-bit LE | `pcm_s16le` | — | Uncompressed (WAV) | + +The FFmpeg name column shows the value to use for `VideoCodec` or `AudioCodec`. +The RFC 6381 column shows the value for `VideoCodecRFC6381` or `AudioCodecRFC6381`. +RFC 6381 strings vary by profile and level; +the values shown are representative examples. + +## Privacy Considerations + +Media files — particularly audio and video recordings — may contain +personally identifiable information (PII), including but not limited to: + +- Voices and speech content +- Facial features and other physical characteristics +- Background environments that could identify locations +- Metadata embedded in file headers (for example, GPS coordinates, device identifiers) + +Researchers MUST ensure that sharing of media files complies with the +informed consent obtained from participants and with applicable privacy regulations. +De-identification techniques (for example, voice distortion, face blurring, +metadata stripping) SHOULD be applied where appropriate before data sharing. + +## Example + +A complete sidecar JSON file for an audio-video recording: + +```json +{ + "RecordingDuration": 312.5, + "VideoCodec": "h264", + "VideoCodecRFC6381": "avc1.640028", + "FrameRate": 30, + "Width": 1920, + "Height": 1080, + "AudioCodec": "aac", + "AudioCodecRFC6381": "mp4a.40.2", + "AudioSampleRate": 48000, + "AudioChannelCount": 2 +} +``` diff --git a/src/schema/objects/extensions.yaml b/src/schema/objects/extensions.yaml index 3c7ef248fa..26ce7a6eb0 100644 --- a/src/schema/objects/extensions.yaml +++ b/src/schema/objects/extensions.yaml @@ -1,5 +1,11 @@ --- # This file describes valid file extensions in the specification. +aac: + value: .aac + display_name: Advanced Audio Coding + description: | + An [Advanced Audio Coding](https://en.wikipedia.org/wiki/Advanced_Audio_Coding) + audio file. ave: value: .ave display_name: AVE # not sure what ave stands for @@ -7,6 +13,12 @@ ave: File containing data averaged by segments of interest. Used by KIT, Yokogawa, and Ricoh MEG systems. +avi: + value: .avi + display_name: Audio Video Interleave + description: | + An [Audio Video Interleave](https://en.wikipedia.org/wiki/Audio_Video_Interleave) + media container file. bdf: value: .bdf display_name: Biosemi Data Format @@ -153,6 +165,22 @@ md: display_name: Markdown description: | A Markdown file. +mkv: + value: .mkv + display_name: Matroska Video + description: | + A [Matroska](https://www.matroska.org/) media container file. +mp3: + value: .mp3 + display_name: MP3 Audio + description: | + An [MP3](https://en.wikipedia.org/wiki/MP3) audio file. +mp4: + value: .mp4 + display_name: MPEG-4 Part 14 + description: | + An [MPEG-4 Part 14](https://en.wikipedia.org/wiki/MP4_file_format) + media container file. mefd: value: .mefd/ display_name: Multiscale Electrophysiology File Format Version 3.0 @@ -201,6 +229,12 @@ nwb: A [Neurodata Without Borders](https://nwb-schema.readthedocs.io/en/latest/) file. Each recording consists of a single `.nwb` file. +ogg: + value: .ogg + display_name: Ogg Vorbis + description: | + An [Ogg](https://en.wikipedia.org/wiki/Ogg) audio file, + typically containing Vorbis-encoded audio. OMEBigTiff: value: .ome.btf display_name: Open Microscopy Environment BigTIFF @@ -249,6 +283,11 @@ snirf: display_name: Shared Near Infrared Spectroscopy Format description: | HDF5 file organized according to the [SNIRF specification](https://github.com/fNIRS/snirf) +svg: + value: .svg + display_name: Scalable Vector Graphics + description: | + A [Scalable Vector Graphics](https://en.wikipedia.org/wiki/SVG) image file. sqd: value: .sqd display_name: SQD @@ -263,6 +302,12 @@ tif: display_name: Tag Image File Format description: | A [Tag Image File Format](https://en.wikipedia.org/wiki/TIFF) file. +tiff: + value: .tiff + display_name: Tag Image File Format + description: | + A [Tag Image File Format](https://en.wikipedia.org/wiki/TIFF) image file. + The `.tiff` extension is the long form of `.tif`. trg: value: .trg display_name: KRISS TRG @@ -307,6 +352,23 @@ vmrk: A text marker file in the [BrainVision Core Data Format](https://www.brainproducts.com/support-resources/brainvision-core-data-format-1-0/). These files come in three-file sets, including a `.vhdr`, a `.vmrk`, and a `.eeg` file. +wav: + value: .wav + display_name: Waveform Audio + description: | + A [Waveform Audio File Format](https://en.wikipedia.org/wiki/WAV) + audio file, typically containing uncompressed PCM audio. +webm: + value: .webm + display_name: WebM + description: | + A [WebM](https://www.webmproject.org/) media container file, + typically containing VP8/VP9 video and Vorbis/Opus audio. +webp: + value: .webp + display_name: WebP Image + description: | + A [WebP](https://en.wikipedia.org/wiki/WebP) image file. Any: value: .* display_name: Any Extension diff --git a/src/schema/objects/metadata.yaml b/src/schema/objects/metadata.yaml index f7bdc4defa..14dd465823 100644 --- a/src/schema/objects/metadata.yaml +++ b/src/schema/objects/metadata.yaml @@ -237,6 +237,42 @@ AttenuationCorrectionMethodReference: description: | Reference paper for the attenuation correction method used. type: string +AudioChannelCount: + name: AudioChannelCount + display_name: Audio Channel Count + description: | + Number of audio channels in the audio or audio-video file + (for example, `1` for mono, `2` for stereo). + type: integer + minimum: 1 +AudioCodec: + name: AudioCodec + display_name: Audio Codec + description: | + The audio codec used to encode the audio stream, expressed as an + [FFmpeg codec name](https://www.ffmpeg.org/ffmpeg-codecs.html) + (for example, `"aac"`, `"mp3"`, `"opus"`, `"flac"`, `"pcm_s16le"`). + This value can be auto-extracted using + `ffprobe -v quiet -print_format json -show_streams`. + type: string +AudioCodecRFC6381: + name: AudioCodecRFC6381 + display_name: Audio Codec (RFC 6381) + description: | + The audio codec expressed as an + [RFC 6381](https://datatracker.ietf.org/doc/html/rfc6381) codec string + (for example, `"mp4a.40.2"` for AAC-LC). + This representation is useful for web and broadcast interoperability. + type: string +AudioSampleRate: + name: AudioSampleRate + display_name: Audio Sample Rate + description: | + Sampling frequency of the audio stream, in Hz + (for example, `44100`, `48000`, `96000`). + type: number + exclusiveMinimum: 0 + unit: Hz Authors: name: Authors display_name: Authors @@ -1544,6 +1580,15 @@ FlipAngle: unit: degree exclusiveMinimum: 0 maximum: 360 +FrameRate: + name: FrameRate + display_name: Frame Rate + description: | + The video frame rate of the video stream, in Hz + (for example, `24`, `25`, `29.97`, `30`, `60`). + type: number + exclusiveMinimum: 0 + unit: Hz OnsetSource: name: OnsetSource display_name: Column Name of the Onset Source @@ -1767,6 +1812,14 @@ HardwareFilters: - type: string enum: - n/a +Height: + name: Height + display_name: Height + description: | + Height of the video frame or image, in pixels. + type: integer + minimum: 1 + unit: px HeadCircumference: name: HeadCircumference display_name: Head Circumference @@ -4496,6 +4549,25 @@ VisionCorrection: Equipment used to correct participant vision during an experiment. Example: "spectacles", "lenses", "none". type: string +VideoCodec: + name: VideoCodec + display_name: Video Codec + description: | + The video codec used to encode the video stream, expressed as an + [FFmpeg codec name](https://www.ffmpeg.org/ffmpeg-codecs.html) + (for example, `"h264"`, `"hevc"`, `"vp9"`, `"av1"`). + This value can be auto-extracted using + `ffprobe -v quiet -print_format json -show_streams`. + type: string +VideoCodecRFC6381: + name: VideoCodecRFC6381 + display_name: Video Codec (RFC 6381) + description: | + The video codec expressed as an + [RFC 6381](https://datatracker.ietf.org/doc/html/rfc6381) codec string + (for example, `"avc1.640028"` for H.264 High Profile Level 4.0). + This representation is useful for web and broadcast interoperability. + type: string VolumeTiming: name: VolumeTiming display_name: Volume Timing @@ -4531,6 +4603,14 @@ WholeBloodAvail: If `true`, the `whole_blood_radioactivity` column MUST be present in the corresponding `*_blood.tsv` file. type: boolean +Width: + name: Width + display_name: Width + description: | + Width of the video frame or image, in pixels. + type: integer + minimum: 1 + unit: px WithdrawalRate: name: WithdrawalRate display_name: Withdrawal Rate diff --git a/src/schema/objects/suffixes.yaml b/src/schema/objects/suffixes.yaml index 1fa2fb58f6..ea1029bae7 100644 --- a/src/schema/objects/suffixes.yaml +++ b/src/schema/objects/suffixes.yaml @@ -516,6 +516,18 @@ asl: The complete ASL time series stored as a 4D NIfTI file in the original acquisition order, with possible volume types including: control, label, m0scan, deltam, cbf. +audio: + value: audio + display_name: Audio file + description: | + An audio data file containing one or more audio streams. + Common formats include WAV (uncompressed), MP3, AAC, and Ogg Vorbis. +audiovideo: + value: audiovideo + display_name: Audio-video file + description: | + A media file containing both audio and video streams. + Common containers include MP4, MKV, AVI, and WebM. aslcontext: value: aslcontext display_name: Arterial Spin Labeling Context @@ -666,6 +678,12 @@ ieeg: display_name: Intracranial Electroencephalography description: | Intracranial electroencephalography recording data. +image: + value: image + display_name: Image file + description: | + A still image data file. + Common formats include JPEG, PNG, SVG, WebP, and TIFF. inplaneT1: value: inplaneT1 display_name: Inplane T1 @@ -897,3 +915,9 @@ unloc: description: | MRS acquisitions run without localization. This includes signals detected using coil sensitivity only. +video: + value: video + display_name: Video file + description: | + A video data file containing one or more video streams but no audio. + Common containers include MP4, MKV, AVI, and WebM. diff --git a/src/schema/rules/sidecars/media.yaml b/src/schema/rules/sidecars/media.yaml new file mode 100644 index 0000000000..e188221ee6 --- /dev/null +++ b/src/schema/rules/sidecars/media.yaml @@ -0,0 +1,34 @@ +# +# Groups of related metadata fields for media files +# + +--- +MediaDuration: + selectors: + - intersects([suffix], ["audio", "video", "audiovideo"]) + fields: + RecordingDuration: recommended + +MediaAudioProperties: + selectors: + - intersects([suffix], ["audio", "audiovideo"]) + fields: + AudioCodec: recommended + AudioSampleRate: recommended + AudioChannelCount: recommended + AudioCodecRFC6381: optional + +MediaVisualProperties: + selectors: + - intersects([suffix], ["video", "audiovideo", "image"]) + fields: + Width: recommended + Height: recommended + +MediaVideoProperties: + selectors: + - intersects([suffix], ["video", "audiovideo"]) + fields: + VideoCodec: recommended + FrameRate: recommended + VideoCodecRFC6381: optional diff --git a/tools/mkdocs_macros_bids/macros.py b/tools/mkdocs_macros_bids/macros.py index 2e7c2f893e..7738dcf295 100644 --- a/tools/mkdocs_macros_bids/macros.py +++ b/tools/mkdocs_macros_bids/macros.py @@ -203,6 +203,52 @@ def make_suffix_table(suffixes, src_path=None): return table +def make_extension_table(extensions, src_path=None): + """Generate a markdown table of file extension information. + + Parameters + ---------- + extensions : list of str + A list of the extension keys to include in the table. + Keys correspond to entries in the schema's objects.extensions + (for example, ``["wav", "mp3", "aac", "ogg"]``). + src_path : str or None + The file where this macro is called, which may be explicitly provided + by the "page.file.src_path" variable. + + Returns + ------- + table : str + A Markdown-format table containing the extension information. + """ + if src_path is None: + src_path = _get_source_path() + + schema_obj = schema.load_schema() + ext_objects = schema_obj["objects"]["extensions"] + + # Compute the relative path to the glossary from the calling file + src_dir = os.path.dirname(src_path) + glossary_path = os.path.relpath("glossary.md", src_dir) + + rows = [] + for ext_key in extensions: + ext = ext_objects[ext_key] + value = ext["value"] + display_name = ext["display_name"] + # Collapse multi-line description to single line + description = " ".join(ext["description"].strip().split()) + + # Link to glossary anchor + link = f"[{value}]({glossary_path}#objects.extensions.{ext_key})" + + rows.append(f"| {display_name} | {link} | {description} |") + + header = "| **Format** | **Extension** | **Description** |" + separator = "| --- | --- | --- |" + return "\n".join([header, separator] + rows) + + def make_metadata_table(field_info, src_path=None): """Generate a markdown table of metadata field information. diff --git a/tools/mkdocs_macros_bids/main.py b/tools/mkdocs_macros_bids/main.py index 7fa873247a..e4cbd2ba70 100644 --- a/tools/mkdocs_macros_bids/main.py +++ b/tools/mkdocs_macros_bids/main.py @@ -38,6 +38,7 @@ def define_env(env): ) env.macro(macros.make_glossary, "MACROS___make_glossary") env.macro(macros.make_suffix_table, "MACROS___make_suffix_table") + env.macro(macros.make_extension_table, "MACROS___make_extension_table") env.macro(macros.make_metadata_table, "MACROS___make_metadata_table") env.macro(macros.make_json_table, "MACROS___make_json_table") env.macro(macros.make_sidecar_table, "MACROS___make_sidecar_table") diff --git a/tools/schemacode/src/bidsschematools/tests/test_render_tables.py b/tools/schemacode/src/bidsschematools/tests/test_render_tables.py index 22676689d0..7cd77ad951 100644 --- a/tools/schemacode/src/bidsschematools/tests/test_render_tables.py +++ b/tools/schemacode/src/bidsschematools/tests/test_render_tables.py @@ -1,8 +1,16 @@ """Tests for the bidsschematools package.""" +import sys +from pathlib import Path + from bidsschematools.render import tables from bidsschematools.render.utils import normalize_requirements +# Make mkdocs_macros_bids importable +_macros_dir = Path(__file__).parents[5] / "tools" / "mkdocs_macros_bids" +if str(_macros_dir) not in sys.path: + sys.path.insert(0, str(_macros_dir)) + def test_make_entity_table(schema_obj): """ @@ -145,3 +153,39 @@ def test_make_columns_table(schema_obj): assert level.upper() in render_row assert level_addendum.split("\n")[0] in render_row assert description_addendum.split("\n")[0] in render_row + + +def test_make_extension_table(schema_obj): + """Test whether expected extensions are present and listed correctly. + + This tests the make_extension_table macro from mkdocs_macros_bids. + """ + import macros as mkdocs_macros # type: ignore[import-not-found] + + target_extensions = ["wav", "mp4", "jpg"] + table = mkdocs_macros.make_extension_table( + target_extensions, + src_path="appendices/media-files.md", + ) + + rendered_lines = table.split("\n") + + # Header and separator + assert rendered_lines[0].startswith("| **Format**") + assert rendered_lines[1].startswith("| ---") + + # One data row per extension + assert len(rendered_lines) == len(target_extensions) + 2 + + # Check each extension is rendered with correct display name and value + expected = { + "wav": (".wav", "Waveform Audio"), + "mp4": (".mp4", "MPEG-4 Part 14"), + "jpg": (".jpg", "Joint Photographic Experts Group"), + } + for ext_key, render_row in zip(target_extensions, rendered_lines[2:]): + value, display_name = expected[ext_key] + assert display_name in render_row + assert value in render_row + # Glossary link + assert f"glossary.md#objects.extensions.{ext_key}" in render_row