From c1aff2248244c1412406cf1489c778871962c003 Mon Sep 17 00:00:00 2001 From: BitcrushedHeart <215311041+BitcrushedHeart@users.noreply.github.com> Date: Sun, 17 May 2026 21:13:11 +0100 Subject: [PATCH 1/2] Resolve LoadingPipeline forward reference in PipelineModule --- src/mgds/PipelineModule.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/mgds/PipelineModule.py b/src/mgds/PipelineModule.py index 5af0b7d..ad96eb4 100644 --- a/src/mgds/PipelineModule.py +++ b/src/mgds/PipelineModule.py @@ -4,6 +4,7 @@ from concurrent import futures from contextlib import ExitStack from random import Random +from typing import TYPE_CHECKING import torch @@ -11,6 +12,9 @@ from mgds.pipelineModuleTypes.SerialPipelineModule import SerialPipelineModule from mgds.pipelineModuleTypes.SingleVariationRandomAccessPipelineModule import SingleVariationRandomAccessPipelineModule +if TYPE_CHECKING: + from mgds.LoadingPipeline import LoadingPipeline + class PipelineState: """Container for state shared amongst all pipeline modules in a pipeline. From 94ab5830567259c091783f0b341124c7b246e680 Mon Sep 17 00:00:00 2001 From: BitcrushedHeart <215311041+BitcrushedHeart@users.noreply.github.com> Date: Sun, 17 May 2026 21:20:21 +0100 Subject: [PATCH 2/2] Fix ruff check errors and invalid annotation - Remove unused imports (MGDS, ImageToVideo) - Use `s not in xs` instead of `not(s in xs)` (DropTags) - Drop unused `as e` from re-raised exception (LoadVideo) - Drop unused `path_names` local (PipelineModule) - Fix `definition: [PipelineModule]` annotation to `list[PipelineModule]` (MGDS) --- src/mgds/MGDS.py | 4 ++-- src/mgds/PipelineModule.py | 1 - src/mgds/pipelineModules/DropTags.py | 4 ++-- src/mgds/pipelineModules/ImageToVideo.py | 2 -- src/mgds/pipelineModules/LoadVideo.py | 2 +- 5 files changed, 5 insertions(+), 8 deletions(-) diff --git a/src/mgds/MGDS.py b/src/mgds/MGDS.py index 4c0c90d..faa2c76 100644 --- a/src/mgds/MGDS.py +++ b/src/mgds/MGDS.py @@ -1,7 +1,7 @@ import random import torch -from torch.utils.data import DataLoader, Dataset, IterableDataset +from torch.utils.data import DataLoader, IterableDataset from mgds.ConceptPipelineModule import ConceptPipelineModule from mgds.LoadingPipeline import LoadingPipeline @@ -19,7 +19,7 @@ def __init__( device: torch.device, concepts: list[dict], settings: dict, - definition: [PipelineModule], + definition: list[PipelineModule], batch_size: int, #local batch size state: PipelineState, seed: int = 42, diff --git a/src/mgds/PipelineModule.py b/src/mgds/PipelineModule.py index ad96eb4..d3681b9 100644 --- a/src/mgds/PipelineModule.py +++ b/src/mgds/PipelineModule.py @@ -146,7 +146,6 @@ def _get_previous_item(self, variation: int, name: str, index: int): def _get_previous_length(self, name: str): split_name = name.split('.') item_name = split_name[0] - path_names = split_name[1::] for previous_module_index in range(self.__module_index - 1, -1, -1): module = self.pipeline.modules[previous_module_index] diff --git a/src/mgds/pipelineModules/DropTags.py b/src/mgds/pipelineModules/DropTags.py index 725e318..81fe342 100644 --- a/src/mgds/pipelineModules/DropTags.py +++ b/src/mgds/pipelineModules/DropTags.py @@ -144,7 +144,7 @@ def get_item(self, variation: int, index: int, requested_name: str = None) -> di for s in dropout_tags: if special_tag_mode == "WHITELIST" and s in special_tags_list: pruned_tags.append(s) - elif special_tag_mode == "BLACKLIST" and not(s in special_tags_list): + elif special_tag_mode == "BLACKLIST" and s not in special_tags_list: pruned_tags.append(s) elif dropout_mode.startswith("RANDOM"): #iterate through dropout_tags and add to pruned_tags if random > probability @@ -156,7 +156,7 @@ def get_item(self, variation: int, index: int, requested_name: str = None) -> di elif special_tag_mode == "BLACKLIST": for i, s in enumerate(dropout_tags): if rand.random() > self.probability_weighted(probability, dropout_mode, i, len(dropout_tags)) \ - or not(s in special_tags_list): + or s not in special_tags_list: pruned_tags.append(s) else: #NONE or any other unexpected values for i, s in enumerate(dropout_tags): diff --git a/src/mgds/pipelineModules/ImageToVideo.py b/src/mgds/pipelineModules/ImageToVideo.py index e30e280..fe75f0a 100644 --- a/src/mgds/pipelineModules/ImageToVideo.py +++ b/src/mgds/pipelineModules/ImageToVideo.py @@ -1,5 +1,3 @@ -from transformers import CLIPTokenizer, T5Tokenizer, T5TokenizerFast, GemmaTokenizer, LlamaTokenizer - from mgds.PipelineModule import PipelineModule from mgds.pipelineModuleTypes.RandomAccessPipelineModule import RandomAccessPipelineModule diff --git a/src/mgds/pipelineModules/LoadVideo.py b/src/mgds/pipelineModules/LoadVideo.py index 3acd6ba..8fa855d 100644 --- a/src/mgds/pipelineModules/LoadVideo.py +++ b/src/mgds/pipelineModules/LoadVideo.py @@ -123,7 +123,7 @@ def get_item(self, variation: int, index: int, requested_name: str = None) -> di except FileNotFoundError: video_tensor = None - except Exception as e: + except Exception: print("could not load video, it might be corrupted: " + path) raise else: