diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 00000000..cd527f6b --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,141 @@ +name: Deploy to PyPI + +on: + push: + branches: [main] + workflow_dispatch: # Allow manual triggering + workflow_run: + workflows: [ + "Python CI", + "Tests", + "Generate Auto Examples", + "Update Changelog Documentation", + "Pre-commit Checks", + ] # Trigger after CI and Tests complete + types: + - completed + branches: [main] + +permissions: + contents: read + id-token: write # For trusted publishing to PyPI + +jobs: + check-version: + runs-on: ubuntu-latest + # Only run if workflow_run was successful or if triggered by push/manual + if: github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' + outputs: + version-changed: ${{ steps.version-check.outputs.changed }} + current-version: ${{ steps.version-check.outputs.version }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # Fetch full history for version comparison + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + + - name: Check if version changed + id: version-check + run: | + # Get current version from kaira/version.py + CURRENT_VERSION=$(python -c "from kaira.version import __version__; print(__version__)") + echo "current-version=$CURRENT_VERSION" >> $GITHUB_OUTPUT + echo "Current version: $CURRENT_VERSION" + + # Check if this version exists on PyPI using simple curl + if curl -f -s "https://pypi.org/pypi/pykaira/$CURRENT_VERSION/json" > /dev/null 2>&1; then + echo "Version $CURRENT_VERSION already exists on PyPI" + echo "changed=false" >> $GITHUB_OUTPUT + else + echo "Version $CURRENT_VERSION is new" + echo "changed=true" >> $GITHUB_OUTPUT + fi + + deploy-pypi: + needs: check-version + if: needs.check-version.outputs.version-changed == 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Install build dependencies + run: | + python -m pip install --upgrade pip + pip install build twine setuptools wheel + + - name: Clean build artifacts + run: | + # Clean Python build artifacts and cache files + find . -type d -name "__pycache__" -exec rm -rf {} + || true + find . -type d -name "*.egg-info" -exec rm -rf {} + || true + find . -type d -name ".eggs" -exec rm -rf {} + || true + find . -type f -name "*.pyc" -delete || true + find . -type f -name "*.pyo" -delete || true + find . -type f -name "*.pyd" -delete || true + find . -type f -name ".coverage" -delete || true + find . -type f -name "coverage.xml" -delete || true + find . -type d -name ".pytest_cache" -exec rm -rf {} + || true + find . -type d -name ".coverage*" -exec rm -rf {} + || true + find . -type d -name "htmlcov" -exec rm -rf {} + || true + + # Clean documentation build artifacts + rm -rf docs/_build/ || true + rm -rf docs/gen_modules/ || true + rm -rf docs/generated/ || true + rm -rf docs/auto_examples/ || true + + # Remove build and dist directories + rm -rf build/ dist/ ./*.egg-info/ || true + + - name: Build distribution packages + run: | + echo "Building distribution for version ${{ needs.check-version.outputs.current-version }}" + python setup.py sdist bdist_wheel + + - name: Check package with twine + run: twine check dist/* + + - name: Upload to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + # Use API token stored in repository secrets + password: ${{ secrets.PYPI_API_TOKEN }} + verbose: true + + - name: Verify deployment + run: | + # Wait a bit for the package to be available + sleep 30 + + # Try to install from PyPI + pip install pykaira==${{ needs.check-version.outputs.current-version }} + + # Basic import test + python -c "import kaira; print(f'Successfully deployed kaira version: {kaira.__version__}')" + + - name: Create GitHub Release + uses: softprops/action-gh-release@v1 + with: + tag_name: v${{ needs.check-version.outputs.current-version }} + name: Release v${{ needs.check-version.outputs.current-version }} + body: | + ## Changes in v${{ needs.check-version.outputs.current-version }} + + This release has been automatically deployed to PyPI. + + Install with: `pip install pykaira==${{ needs.check-version.outputs.current-version }}` + draft: false + prerelease: false diff --git a/.gitignore b/.gitignore index 15836af9..16889c6a 100644 --- a/.gitignore +++ b/.gitignore @@ -8,10 +8,9 @@ paper* # C extensions *.so results -examples/benchmarks/example_results -examples/benchmarks/benchmark_results -examples/benchmarks/visualization_results .gradio +wandb +presentation # Distribution / packaging .Python diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 00000000..fddc4faf --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,3 @@ +# Code of Conduct + +We follow the [Python Software Foundation Code of Conduct](https://www.python.org/psf/codeofconduct/). All contributors are expected to adhere to its principles of openness, respect, and collaboration. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4526ee7c..d39c0c3b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -39,11 +39,14 @@ There are many ways to contribute to Kaira: python -m venv venv source venv/bin/activate # On Windows: venv\Scripts\activate - # Install development dependencies - pip install -e ".[dev]" - # Or alternatively: + # Install the package in development mode pip install -e . + + # Install development dependencies pip install -r requirements-dev.txt + + # Set up pre-commit hooks (recommended) + pre-commit install ``` ### Making Changes @@ -66,10 +69,14 @@ There are many ways to contribute to Kaira: pytest ``` -4. **Check code style**: +4. **Check code style and formatting**: ```bash - bash scripts/lint.sh + # Run all pre-commit hooks (formatting, linting, etc.) + pre-commit run -a + + # Or use the Makefile shortcut + make format ``` ### Submitting a Pull Request @@ -183,10 +190,9 @@ Examples are organized into categories: - `metrics` - Performance metrics and evaluation tools - `models` - Neural network models and architectures - `models_fec` - Forward Error Correction models -- `benchmarks` - Benchmarking tools and comparisons - And more... -For detailed information, see `docs/automated_example_gallery.md`. +For detailed information, see the existing examples in each category directory. ## Testing diff --git a/configs/training_example.yaml b/configs/training_example.yaml new file mode 100644 index 00000000..8204a765 --- /dev/null +++ b/configs/training_example.yaml @@ -0,0 +1,101 @@ +# @package _global_ + +# Hydra configuration for Kaira training +# This configuration demonstrates how to set up training parameters +# for communication system models using Hydra + +defaults: + - _self_ + +# Model configuration +model: + _target_: kaira.models.DeepJSCCModel + type: deepjscc + input_dim: 512 + channel_uses: 64 + hidden_dim: 256 + encoder_layers: 3 + decoder_layers: 3 + activation: relu + +# Training configuration +training: + output_dir: ./training_results + num_train_epochs: 10 + per_device_train_batch_size: 32 + per_device_eval_batch_size: 32 + learning_rate: 1e-4 + weight_decay: 0.01 + warmup_steps: 1000 + logging_steps: 100 + eval_steps: 500 + save_steps: 1000 + eval_strategy: steps + save_strategy: steps + save_total_limit: 3 + + # Communication-specific parameters + snr_min: 0.0 + snr_max: 20.0 + noise_variance_min: 0.1 + noise_variance_max: 2.0 + channel_uses: 64 + channel_type: awgn + + # Training flags + do_eval: true + do_predict: false + fp16: false + dataloader_num_workers: 0 + + # Optimization + gradient_accumulation_steps: 1 + max_grad_norm: 1.0 + lr_scheduler_type: linear + + # Logging and monitoring + logging_dir: ${training.output_dir}/logs + run_name: deepjscc_training + report_to: [] # Can be set to ["wandb", "tensorboard"] for monitoring + +# Data configuration +data: + dataset_name: null # Most communication models generate synthetic data + train_batch_size: ${training.per_device_train_batch_size} + eval_batch_size: ${training.per_device_eval_batch_size} + max_train_samples: null + max_eval_samples: null + preprocessing_num_workers: 4 + +# Channel simulation configuration +channel: + type: ${training.channel_type} + snr_range: + - ${training.snr_min} + - ${training.snr_max} + noise_type: gaussian + fading: false + +# Optimizer configuration +optimizer: + type: adamw + lr: ${training.learning_rate} + weight_decay: ${training.weight_decay} + betas: + - 0.9 + - 0.999 + eps: 1e-8 + +# Scheduler configuration +scheduler: + type: ${training.lr_scheduler_type} + warmup_steps: ${training.warmup_steps} + num_training_steps: null # Will be calculated automatically + +# Hydra configuration +hydra: + run: + dir: ${training.output_dir}/hydra_outputs/${now:%Y-%m-%d_%H-%M-%S} + job: + name: kaira_training + chdir: true diff --git a/configs/training_simple.yaml b/configs/training_simple.yaml new file mode 100644 index 00000000..be9e12c9 --- /dev/null +++ b/configs/training_simple.yaml @@ -0,0 +1,26 @@ +# @package _global_ + +# Simple Hydra configuration for Kaira training +defaults: + - _self_ + +# Model configuration +model: + _target_: kaira.models.DeepJSCCModel + type: deepjscc + input_dim: 256 + channel_uses: 32 + hidden_dim: 128 + +# Training configuration +training: + output_dir: ./simple_training + num_train_epochs: 5 + per_device_train_batch_size: 16 + learning_rate: 1e-3 + snr_min: 0.0 + snr_max: 10.0 + channel_type: awgn + eval_strategy: no + save_strategy: epoch + logging_steps: 50 diff --git a/docs/api_reference.rst b/docs/api_reference.rst index 964854aa..563d58e1 100644 --- a/docs/api_reference.rst +++ b/docs/api_reference.rst @@ -248,68 +248,23 @@ Models module for Kaira. ConfigurableModel DeepJSCCModel FeedbackChannelModel + ModelConfig ModelRegistry MultipleAccessChannelModel WynerZivModel -Soft Bit Thresholding -^^^^^^^^^^^^^^^^^^^^^ - -Soft bit thresholding module for binary data processing. - -This module provides various thresholding techniques for converting soft bit representations -(probabilities, LLRs, etc.) to hard decisions. These thresholders can be used with soft decoders or -as standalone components in signal processing pipelines. - -Soft bit processing is crucial in modern communication systems to extract maximum information from -the received signals. The techniques implemented here are based on established methods in -communication theory. - -.. currentmodule:: kaira.models.binary.soft_bit_thresholding - -.. autosummary:: - :toctree: generated - :template: class.rst - :nosignatures: - - AdaptiveThresholder - DynamicThresholder - FixedThresholder - HysteresisThresholder - InputType - LLRThresholder - MinDistanceThresholder - OutputType - RepetitionSoftBitDecoder - SoftBitEnsembleThresholder - SoftBitThresholder - WeightedThresholder - - -Components -^^^^^^^^^^ - -Components module for Kaira models. - -.. currentmodule:: kaira.models.components - -.. autosummary:: - :toctree: generated - :template: class.rst - :nosignatures: +Forward Error Correction (FEC) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - AFModule - ConvDecoder - ConvEncoder - MLPDecoder - MLPEncoder - Projection - ProjectionType +Forward Error Correction module for Kaira models. +This module provides comprehensive implementations for forward error correction, including both +encoders and decoders for various coding schemes. The encoders and decoders are designed to work +seamlessly together to provide robust error correction capabilities for communication systems. Decoders -^^^^^^^^ +~~~~~~~~ Forward Error Correction (FEC) decoders for Kaira. @@ -317,23 +272,8 @@ This module provides various decoder implementations for forward error correctio The decoders in this module are designed to work seamlessly with the corresponding encoders from the `kaira.models.fec.encoders` module. -Decoders --------- -- BlockDecoder: Base class for all block code decoders -- SyndromeLookupDecoder: Decoder using syndrome lookup tables for efficient error correction -- BerlekampMasseyDecoder: Implementation of Berlekamp-Massey algorithm for decoding BCH and Reed-Solomon codes -- ReedMullerDecoder: Implementation of Reed-Muller decoding algorithm for Reed-Muller codes -- WagnerSoftDecisionDecoder: Implementation of Wagner's soft-decision decoder for single-parity check codes -- BruteForceMLDecoder: Maximum likelihood decoder that searches through all possible codewords -- BeliefPropagationDecoder: Implementation of belief propagation algorithm :cite:`kschischang2001factor` for decoding LDPC codes -- MinSumLDPCDecoder: Min-Sum decoder :cite:`chen2005reduced` for LDPC codes with reduced computational complexity - -These decoders can be used to recover original messages from possibly corrupted codewords -that have been transmitted over noisy channels. Each decoder has specific strengths and -is optimized for particular types of codes or error patterns. - -Examples --------- +Example Usage +""""""""""""" >>> from kaira.models.fec.encoders import BCHCodeEncoder >>> from kaira.models.fec.decoders import BerlekampMasseyDecoder >>> encoder = BCHCodeEncoder(15, 7) @@ -362,25 +302,13 @@ Examples Encoders -^^^^^^^^ +~~~~~~~~ Forward Error Correction encoders for Kaira. -This module provides various encoder implementations for forward error correction, including: -- Block codes: Fundamental error correction codes that operate on fixed-size blocks -- Linear block codes: Codes with linear algebraic structure allowing matrix operations -- LDPC codes: Low-Density Parity-Check codes with sparse parity-check matrices -- Cyclic codes: Special class of linear codes with cyclic shift properties -- BCH codes: Powerful algebraic codes with precise error-correction capabilities -- Reed-Solomon codes: Widely-used subset of BCH codes for burst error correction -- Hamming codes: Simple single-error-correcting codes with efficient implementation -- Repetition codes: Basic codes that repeat each bit multiple times -- Golay codes: Perfect codes with specific error correction properties -- Single parity-check codes: Simple error detection through parity bit addition - -These encoders can be used to add redundancy to data for enabling error detection and correction +This module provides various encoder implementations for forward error correction.These encoders can be used to add redundancy to data for enabling error detection and correction in communication systems, storage devices, and other applications requiring reliable data -transmission over noisy channels :cite:`lin2004error,moon2005error`. +transmission over noisy channels. .. currentmodule:: kaira.models.fec.encoders @@ -404,6 +332,61 @@ transmission over noisy channels :cite:`lin2004error,moon2005error`. SystematicLinearBlockCodeEncoder +Soft Bit Thresholding +^^^^^^^^^^^^^^^^^^^^^ + +Soft bit thresholding module for binary data processing. + +This module provides various thresholding techniques for converting soft bit representations +(probabilities, LLRs, etc.) to hard decisions. These thresholders can be used with soft decoders or +as standalone components in signal processing pipelines. + +Soft bit processing is crucial in modern communication systems to extract maximum information from +the received signals. The techniques implemented here are based on established methods in +communication theory. + +.. currentmodule:: kaira.models.binary.soft_bit_thresholding + +.. autosummary:: + :toctree: generated + :template: class.rst + :nosignatures: + + AdaptiveThresholder + DynamicThresholder + FixedThresholder + HysteresisThresholder + InputType + LLRThresholder + MinDistanceThresholder + OutputType + RepetitionSoftBitDecoder + SoftBitEnsembleThresholder + SoftBitThresholder + WeightedThresholder + + +Components +^^^^^^^^^^ + +Components module for Kaira models. + +.. currentmodule:: kaira.models.components + +.. autosummary:: + :toctree: generated + :template: class.rst + :nosignatures: + + AFModule + ConvDecoder + ConvEncoder + MLPDecoder + MLPEncoder + Projection + ProjectionType + + Generic ^^^^^^^ @@ -476,7 +459,13 @@ Image compressor models, including standard and neural network-based methods. :nosignatures: BPGCompressor + BaseImageCompressor + JPEG2000Compressor + JPEGCompressor + JPEGXLCompressor NeuralCompressor + PNGCompressor + WebPCompressor Modulations @@ -561,52 +550,6 @@ This package provides various loss functions for different modalities. LossRegistry -Adversarial -^^^^^^^^^^^ - -Adversarial Losses module for Kaira. - -This module contains various adversarial loss functions for GAN-based training. - -.. currentmodule:: kaira.losses.adversarial - -.. autosummary:: - :toctree: generated - :template: class.rst - :nosignatures: - - FeatureMatchingLoss - HingeLoss - LSGANLoss - R1GradientPenalty - VanillaGANLoss - WassersteinGANLoss - - -Audio -^^^^^ - -Audio Losses module for Kaira. - -This module contains various loss functions for training audio-based communication systems. - -.. currentmodule:: kaira.losses.audio - -.. autosummary:: - :toctree: generated - :template: class.rst - :nosignatures: - - AudioContrastiveLoss - FeatureMatchingLoss - L1AudioLoss - LogSTFTMagnitudeLoss - MelSpectrogramLoss - MultiResolutionSTFTLoss - STFTLoss - SpectralConvergenceLoss - - Image ^^^^^ @@ -638,74 +581,66 @@ computer vision tasks :cite:`wang2009mean` :cite:`zhang2018unreasonable`. VGGLoss -Multimodal -^^^^^^^^^^ +Data +---- -Multimodal Losses module for Kaira. +Data utilities for Kaira. -This module contains various loss functions for training multimodal systems. +This module provides simple and efficient dataset classes for communication systems and information +theory experiments. All datasets are memory-efficient and generate data on-demand. -.. currentmodule:: kaira.losses.multimodal +.. currentmodule:: kaira.data .. autosummary:: :toctree: generated :template: class.rst :nosignatures: - AlignmentLoss - CMCLoss - ContrastiveLoss - InfoNCELoss - TripletLoss + BinaryDataset + CorrelatedDataset + FunctionDataset + GaussianDataset + ImageDataset + UniformDataset -Text -^^^^ +Datasets +^^^^^^^^ -Text Losses module for Kaira. +Simple and efficient dataset implementations for Kaira. -This module contains various loss functions for training text-based systems. +This module provides dataset classes for communication systems and information theory experiments. +All datasets generate data on-demand for memory efficiency and support PyTorch DataLoader. -.. currentmodule:: kaira.losses.text +.. currentmodule:: kaira.data.datasets .. autosummary:: :toctree: generated :template: class.rst :nosignatures: - CosineSimilarityLoss - CrossEntropyLoss - LabelSmoothingLoss - Word2VecLoss - - -Data ----- + BinaryDataset + CorrelatedDataset + FunctionDataset + GaussianDataset + UniformDataset -Data utilities for Kaira, including data generation and correlation models. -.. currentmodule:: kaira.data - -.. autosummary:: - :toctree: generated - :template: class.rst - :nosignatures: +Sample Data +^^^^^^^^^^^ - BinaryTensorDataset - UniformTensorDataset - WynerZivCorrelationDataset +Simple image dataset utilities for Kaira. +This module provides basic image dataset functionality for testing and examples. -.. currentmodule:: kaira.data +.. currentmodule:: kaira.data.sample_data .. autosummary:: :toctree: generated - :template: function.rst + :template: class.rst :nosignatures: - create_binary_tensor - create_uniform_tensor - load_sample_images + ImageDataset Utils @@ -735,6 +670,7 @@ General utility functions for the Kaira library. calculate_snr estimate_signal_power noise_power_to_snr + seed_everything snr_db_to_linear snr_linear_to_db snr_to_noise_power @@ -762,45 +698,35 @@ Utility functions for Signal-to-Noise Ratio (SNR) calculations and conversions. snr_to_noise_power -Benchmarks ----------- - -Kaira Benchmarking System. +Training +-------- -This module provides standardized benchmarks for evaluating communication system components and -deep learning models in Kaira. +Kaira training module. -.. currentmodule:: kaira.benchmarks +This module provides training infrastructure for communication models, including: +- TrainingArguments: Flexible training arguments supporting multiple config systems +- Trainer: Unified trainer for all communication models -.. autosummary:: - :toctree: generated - :template: class.rst - :nosignatures: +Examples: + Basic usage with TrainingArguments: + >>> from kaira.training import TrainingArguments, Trainer + >>> args = TrainingArguments(output_dir="./results", num_train_epochs=10) + >>> trainer = Trainer(model, args) - BaseBenchmark - BenchmarkConfig - BenchmarkRegistry - BenchmarkResult - BenchmarkResultsManager - BenchmarkSuite - BenchmarkVisualizer - ComparisonRunner - ParallelRunner - ParametricRunner - StandardMetrics - StandardRunner + Using Hydra configurations: + >>> args = TrainingArguments.from_hydra(hydra_config) + >>> trainer = Trainer.from_hydra_config(hydra_config, model) + Direct dict configurations: + >>> args = TrainingArguments.from_dict({"output_dir": "./results"}) + >>> trainer = Trainer(model, args) -.. currentmodule:: kaira.benchmarks +.. currentmodule:: kaira.training .. autosummary:: :toctree: generated - :template: function.rst + :template: class.rst :nosignatures: - create_benchmark - get_benchmark - get_config - list_benchmarks - list_configs - register_benchmark + Trainer + TrainingArguments diff --git a/docs/benchmarks.rst b/docs/benchmarks.rst deleted file mode 100644 index 889d7afe..00000000 --- a/docs/benchmarks.rst +++ /dev/null @@ -1,337 +0,0 @@ -Kaira Benchmarking System -========================= - -The Kaira benchmarking system provides standardized benchmarks for evaluating communication system components and deep learning models. This system enables fair comparison of different approaches and reproducible performance evaluation. - -Overview --------- - -The benchmarking system consists of: - -- **Base classes** for creating custom benchmarks -- **Standard benchmarks** for common communication tasks -- **Metrics** for evaluating performance -- **Runners** for executing benchmarks in different modes -- **Configuration management** for reproducible experiments -- **CLI tool** for command-line usage - -Quick Start ------------ - -Basic usage with the new organized results system:: - - from kaira.benchmarks import get_benchmark, StandardRunner, BenchmarkConfig - - # Create a benchmark - ber_benchmark = get_benchmark("ber_simulation")(modulation="bpsk") - - # Configure the benchmark - config = BenchmarkConfig( - snr_range=list(range(-5, 11)), - num_bits=100000 - ) - - # Run the benchmark with automatic result organization - runner = StandardRunner() - result = runner.run_benchmark(ber_benchmark, **config.to_dict()) - - # Results are automatically saved to organized directory structure - print(f"BER results: {result.metrics['ber_simulated']}") - - # Access saved results using the results manager - saved_files = runner.save_all_results(experiment_name="ber_evaluation") - print(f"Results saved to: {saved_files}") - -Traditional usage (still supported):: - - # Manual result saving - result.save("benchmark_result.json") - -Available Benchmarks ------------------------ - -Standard Communication Benchmarks -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -- **ber_simulation**: Bit Error Rate simulation for various modulation schemes -- **channel_capacity**: Shannon channel capacity calculations -- **throughput_test**: System throughput evaluation -- **latency_test**: System latency measurement -- **model_complexity**: Model computational complexity analysis - -Custom Benchmarks -~~~~~~~~~~~~~~~~~ - -You can create custom benchmarks by inheriting from ``BaseBenchmark``:: - - from kaira.benchmarks import BaseBenchmark, register_benchmark - - @register_benchmark("my_benchmark") - class MyBenchmark(BaseBenchmark): - def setup(self, **kwargs): - super().setup(**kwargs) - # Initialize benchmark - - def run(self, **kwargs): - # Run benchmark and return metrics - return {"success": True, "metric_value": 42} - -Configuration -------------- - -Predefined Configurations -~~~~~~~~~~~~~~~~~~~~~~~~~ - -- **fast**: Quick testing configuration -- **accurate**: High-accuracy configuration for publication results -- **comprehensive**: Full evaluation with all metrics -- **gpu**: GPU-optimized configuration -- **minimal**: Minimal configuration for CI/CD - -Custom Configuration:: - - config = BenchmarkConfig( - name="my_config", - num_trials=10, - snr_range=list(range(-10, 16)), - device="cuda", - verbose=True - ) - -Benchmark Execution ------------------------ - -Sequential Execution:: - - runner = StandardRunner(verbose=True) - result = runner.run_benchmark(benchmark, **config.to_dict()) - -Parallel Execution:: - - runner = ParallelRunner(max_workers=4) - results = runner.run_benchmarks(benchmarks, **config.to_dict()) - -Benchmark Suites:: - - suite = BenchmarkSuite("My Suite") - suite.add_benchmark(benchmark1) - suite.add_benchmark(benchmark2) - - results = runner.run_suite(suite, **config.to_dict()) - -Comparison and Analysis:: - - runner = ComparisonRunner() - results = runner.run_comparison( - [benchmark1, benchmark2], - "Algorithm Comparison", - **config.to_dict() - ) - -Metrics and Analysis ------------------------ - -Standard Metrics -~~~~~~~~~~~~~~~~~~~~~~ - -The ``StandardMetrics`` class provides common communication system metrics: - -- Bit Error Rate (BER) -- Block Error Rate (BLER) -- Signal-to-Noise Ratio (SNR) -- Mutual Information -- Throughput -- Latency statistics -- Channel capacity -- Confidence intervals - -Example:: - - from kaira.benchmarks import StandardMetrics - - ber = StandardMetrics.bit_error_rate(transmitted, received) - snr = StandardMetrics.signal_to_noise_ratio(signal, noise) - capacity = StandardMetrics.channel_capacity(snr_db=10.0) - -Results Management -------------------------- - -Kaira provides an organized results management system that automatically structures benchmark results in a clean directory hierarchy. - -Results Directory Structure -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The benchmark system creates the following directory structure:: - - results/ - ├── benchmarks/ # Individual benchmark results - │ ├── experiment_name/ - │ └── benchmark_files.json - ├── suites/ # Benchmark suite results - │ ├── suite_name/ - │ └── summary.json - ├── experiments/ # Experimental runs - ├── comparisons/ # Comparative studies - ├── archives/ # Archived old results - ├── configs/ # Configuration files - ├── logs/ # Execution logs - └── summaries/ # Summary reports - -Using the Results Manager -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The new results management system provides automated organization:: - - from kaira.benchmarks import StandardRunner, BenchmarkResultsManager - - # Create a results manager (uses 'results/' directory by default) - results_manager = BenchmarkResultsManager("my_results") - - # Create a runner with the results manager - runner = StandardRunner(results_manager=results_manager) - - # Run benchmarks - results are automatically saved and organized - result = runner.run_benchmark(benchmark, experiment_name="my_experiment") - - # Results are automatically saved to: - # my_results/benchmarks/my_experiment/benchmark_name_timestamp_id.json - -Manual Results Management -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -You can also manage results manually:: - - # Save individual result with automatic organization - results_manager = BenchmarkResultsManager() - saved_path = results_manager.save_benchmark_result( - result, - category="benchmarks", - experiment_name="my_experiment" - ) - - # Save suite results - saved_files = results_manager.save_suite_results( - results_list, - suite_name="performance_suite", - experiment_name="my_experiment" - ) - - # List available results - all_results = results_manager.list_results() - experiment_results = results_manager.list_results( - category="benchmarks", - experiment_name="my_experiment" - ) - - # Load results - result = results_manager.load_benchmark_result(result_path) - -Loading and Analysis -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - # Load results using the results manager - results_manager = BenchmarkResultsManager() - result_paths = results_manager.list_results(category="benchmarks") - - for path in result_paths: - result = results_manager.load_benchmark_result(path) - print(f"Result: {result.name}, Time: {result.execution_time:.2f}s") - - # Create comparison reports - comparison_path = results_manager.create_comparison_report( - result_paths[:3], - "algorithm_comparison" - ) - - -Results Maintenance -~~~~~~~~~~~~~~~~~~~~~~~ - -The system includes maintenance features for long-term management:: - - # Archive old results (older than 30 days) - results_manager.archive_old_results(days_old=30) - - # Clean up empty directories - results_manager.cleanup_empty_directories() - -Command Line Interface ------------------------------- - -The ``kaira-benchmark`` CLI tool provides easy access to benchmarks:: - - # List available benchmarks - kaira-benchmark --list - - # Run a single benchmark - kaira-benchmark --benchmark ber_simulation --config fast - - # Run multiple benchmarks in parallel - kaira-benchmark --benchmark ber_simulation throughput_test --parallel - - # Run benchmark suite - kaira-benchmark --suite --config comprehensive --output ./results - - # Custom parameters - kaira-benchmark --benchmark ber_simulation --snr-range -5 10 --num-bits 50000 - -Best Practices --------------- - -1. **Use appropriate configurations** for your use case (fast for development, accurate for publications) - -2. **Set random seeds** for reproducible results:: - - config = BenchmarkConfig(seed=42) - -3. **Save raw data** for important experiments:: - - config = BenchmarkConfig(save_raw_data=True) - -4. **Use confidence intervals** for statistical analysis:: - - config = BenchmarkConfig( - calculate_confidence_intervals=True, - confidence_level=0.95 - ) - -5. **Monitor memory usage** for large experiments:: - - config = BenchmarkConfig(memory_limit_mb=8192) - -Examples --------- - -See the ``examples/benchmarks/`` directory for comprehensive examples: - -- ``basic_usage.py``: Basic benchmark usage -- ``comparison_example.py``: Comparing different approaches -- ``custom_benchmark.py``: Creating custom benchmarks -- ``demo_new_results_system.py``: New results management system demonstration - -Results Management Example -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The ``demo_new_results_system.py`` example demonstrates the complete workflow:: - - # Create and configure results manager - results_manager = BenchmarkResultsManager("example_results") - - # Run benchmarks with automatic result organization - runner = StandardRunner(results_manager=results_manager) - - # Create and run benchmark suites - suite = BenchmarkSuite("Performance Suite") - # ... add benchmarks to suite - results = runner.run_suite(suite, experiment_name="demo_experiment") - - # Results are automatically organized in structured directories - -API Reference -------------- - -.. automodule:: kaira.benchmarks - :members: - :undoc-members: - :show-inheritance: - :noindex: diff --git a/docs/conf.py b/docs/conf.py index f56b2cf1..b5611695 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -108,7 +108,6 @@ "../examples/losses", "../examples/models", "../examples/models_fec", - "../examples/benchmarks", ], "gallery_dirs": [ "auto_examples/channels", @@ -119,7 +118,6 @@ "auto_examples/losses", "auto_examples/models", "auto_examples/models_fec", - "auto_examples/benchmarks", ], # File patterns and organization "filename_pattern": r"\.py$", @@ -340,6 +338,9 @@ suppress_warnings = [ "autodoc.duplicate_object", "config.cache", # Suppress warnings about unpicklable configuration values like sphinx_gallery_conf + # Suppress docutils warnings from inherited Transformers library docstrings + "docutils.parsers.rst", + "docutils", ] # Commented out # Add setting to prevent duplicate documentation of enum members # "app.add_directive", "app.add_role", "app.add_generic_role", "app.add_transform", @@ -383,6 +384,17 @@ def skip_member(app, what, name, obj, skip, options): if what == "method" and name == "plot" and hasattr(obj, "__module__") and "torchmetrics.image.ssim" in getattr(obj, "__module__", ""): return True + # Skip problematic inherited methods from Transformers library that have malformed docstrings + transformers_methods_to_skip = ["from_pretrained", "push_to_hub", "set_dataloader", "set_evaluate", "set_logging", "set_lr_scheduler", "set_optimizer", "set_push_to_hub", "set_save", "set_testing", "set_training"] + + if name in transformers_methods_to_skip: + # Check if this is coming from a Transformers class + if hasattr(obj, "__module__") and "transformers" in getattr(obj, "__module__", ""): + return True + # Also skip if the object's class is from transformers + if hasattr(obj, "__qualname__") and any(cls in str(type(obj)) for cls in ["TrainingArguments", "PretrainedConfig"]): + return True + return False diff --git a/docs/examples/benchmarks/index.rst b/docs/examples/benchmarks/index.rst deleted file mode 100644 index 00c98b09..00000000 --- a/docs/examples/benchmarks/index.rst +++ /dev/null @@ -1,121 +0,0 @@ -:orphan: - -Benchmarks -========== - -Benchmarking tools and performance comparisons for different algorithms, models, and system configurations. - -.. raw:: html - -