From f2a43bdd6d1517ed7820ad2cb16888cb7d36d67a Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Wed, 13 Mar 2024 18:43:33 +0200 Subject: [PATCH 1/3] Apply black v2024 --- datumaro/components/dataset_generator.py | 3 +-- datumaro/components/format_detection.py | 3 +-- datumaro/plugins/cityscapes_format.py | 8 +++++--- datumaro/plugins/coco_format/converter.py | 8 ++++---- datumaro/plugins/coco_format/extractor.py | 6 ++---- datumaro/plugins/tf_detection_api_format/converter.py | 4 ++-- 6 files changed, 15 insertions(+), 17 deletions(-) diff --git a/datumaro/components/dataset_generator.py b/datumaro/components/dataset_generator.py index 9677441296..59b35c6fb8 100644 --- a/datumaro/components/dataset_generator.py +++ b/datumaro/components/dataset_generator.py @@ -18,5 +18,4 @@ def generate(cls, output_dir, count, shape, **options): return generator.generate_dataset() @abstractmethod - def generate_dataset(self): - ... + def generate_dataset(self): ... diff --git a/datumaro/components/format_detection.py b/datumaro/components/format_detection.py index 4acb405cbb..632ccc89a4 100644 --- a/datumaro/components/format_detection.py +++ b/datumaro/components/format_detection.py @@ -445,8 +445,7 @@ def __call__( format_name: str, reason: RejectionReason, human_message: str, - ) -> Any: - ... + ) -> Any: ... def detect_dataset_format( diff --git a/datumaro/plugins/cityscapes_format.py b/datumaro/plugins/cityscapes_format.py index 41b19d8dc7..b6986c5083 100644 --- a/datumaro/plugins/cityscapes_format.py +++ b/datumaro/plugins/cityscapes_format.py @@ -442,9 +442,11 @@ def apply(self): compiled_mask = CompiledMask.from_instance_masks( masks, instance_ids=[ - self._label_id_mapping(m.label) - if m.attributes.get("is_crowd", False) - else self._label_id_mapping(m.label) * 1000 + (m.id or (i + 1)) + ( + self._label_id_mapping(m.label) + if m.attributes.get("is_crowd", False) + else self._label_id_mapping(m.label) * 1000 + (m.id or (i + 1)) + ) for i, m in enumerate(masks) ], instance_labels=[self._label_id_mapping(m.label) for m in masks], diff --git a/datumaro/plugins/coco_format/converter.py b/datumaro/plugins/coco_format/converter.py index 9a9eeca28e..4af7587f25 100644 --- a/datumaro/plugins/coco_format/converter.py +++ b/datumaro/plugins/coco_format/converter.py @@ -566,9 +566,8 @@ def _split_tasks_string(s): @classmethod def build_cmdline_parser(cls, **kwargs): - kwargs[ - "description" - ] = """ + kwargs["description"] = ( + """ Segmentation mask modes ('--segmentation-mode'):|n - '{sm.guess.name}': guess the mode for each instance,|n |s|suse 'is_crowd' attribute as hint|n @@ -592,7 +591,8 @@ def build_cmdline_parser(cls, **kwargs): directory, otherwise they are saved in separate directories by subsets. """.format( - sm=SegmentationMode + sm=SegmentationMode + ) ) parser = super().build_cmdline_parser(**kwargs) parser.add_argument( diff --git a/datumaro/plugins/coco_format/extractor.py b/datumaro/plugins/coco_format/extractor.py index c607ebf963..a0b8e4c2e1 100644 --- a/datumaro/plugins/coco_format/extractor.py +++ b/datumaro/plugins/coco_format/extractor.py @@ -292,12 +292,10 @@ def _get_label_id(self, ann): return label_id @overload - def _parse_field(self, ann: Dict[str, Any], key: str, cls: Type[T]) -> T: - ... + def _parse_field(self, ann: Dict[str, Any], key: str, cls: Type[T]) -> T: ... @overload - def _parse_field(self, ann: Dict[str, Any], key: str, cls: Tuple[Type, ...]) -> Any: - ... + def _parse_field(self, ann: Dict[str, Any], key: str, cls: Tuple[Type, ...]) -> Any: ... def _parse_field( self, ann: Dict[str, Any], key: str, cls: Union[Type[T], Tuple[Type, ...]] diff --git a/datumaro/plugins/tf_detection_api_format/converter.py b/datumaro/plugins/tf_detection_api_format/converter.py index d615d4e01f..c1b3d3f438 100644 --- a/datumaro/plugins/tf_detection_api_format/converter.py +++ b/datumaro/plugins/tf_detection_api_format/converter.py @@ -73,8 +73,8 @@ def apply(self): os.makedirs(self._save_dir, exist_ok=True) label_categories = self._extractor.categories().get(AnnotationType.label, LabelCategories()) - get_label = ( - lambda label_id: label_categories.items[label_id].name if label_id is not None else "" + get_label = lambda label_id: ( + label_categories.items[label_id].name if label_id is not None else "" ) label_ids = OrderedDict( (label.name, 1 + idx) for idx, label in enumerate(label_categories.items) From 4c767addd0077d75dd189a854f3f3efa4aca6ea7 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Wed, 13 Mar 2024 19:06:38 +0200 Subject: [PATCH 2/3] Ignore some bandit warnings --- datumaro/plugins/sampler/random_sampler.py | 4 ++-- datumaro/plugins/synthetic_data/image_generator.py | 8 +++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/datumaro/plugins/sampler/random_sampler.py b/datumaro/plugins/sampler/random_sampler.py index b7935bae1a..90bb83be77 100644 --- a/datumaro/plugins/sampler/random_sampler.py +++ b/datumaro/plugins/sampler/random_sampler.py @@ -67,7 +67,7 @@ def __init__( def __iter__(self): if self._indices is None: - rng = Random(self._seed) + rng = Random(self._seed) # nosec - disable B311 if self._subset: n = len(self._extractor.get_subset(self._subset)) @@ -213,7 +213,7 @@ def _make_bucket(): buckets = defaultdict(_make_bucket) # subset -> subset_buckets - rng = Random(self._seed) + rng = Random(self._seed) # nosec - disable B311 for i, item in enumerate(self._extractor): labels = set(getattr(ann, "label", None) for ann in item.annotations) diff --git a/datumaro/plugins/synthetic_data/image_generator.py b/datumaro/plugins/synthetic_data/image_generator.py index 818a147adc..5634561333 100644 --- a/datumaro/plugins/synthetic_data/image_generator.py +++ b/datumaro/plugins/synthetic_data/image_generator.py @@ -66,7 +66,9 @@ def generate_dataset(self) -> None: mp_ctx = get_context("spawn") # On Mac 10.15 and Python 3.7 fork leads to hangs with mp_ctx.Pool(processes=self._cpu_count) as pool: - params = pool.map(self._generate_category, [Random(i) for i in range(self._categories)]) + params = pool.map( + self._generate_category, [Random(i) for i in range(self._categories)] + ) # nosec - disable B311 instances_weights = np.repeat(self._weights, self._instances, axis=0) weight_per_img = np.tile(instances_weights, (self._categories, 1)) @@ -110,7 +112,7 @@ def _generate_image_batch( for i, param, w in zip(indices, params, weights): image = self._generate_image( - Random(i), + Random(i), # nosec - disable B311 param, self._iterations, self._height, @@ -119,7 +121,7 @@ def _generate_image_batch( weight=w, ) color_image = colorize(image, net) - aug_image = augment(Random(i), color_image, background_colors) + aug_image = augment(Random(i), color_image, background_colors) # nosec - disable B311 save_image( osp.join(self._output_dir, "{:06d}.png".format(i)), aug_image, create_dir=True ) From 88710818aadeeb7bcd019d8abd3e7fdd69786709 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Wed, 13 Mar 2024 19:09:22 +0200 Subject: [PATCH 3/3] Update code --- datumaro/plugins/synthetic_data/image_generator.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/datumaro/plugins/synthetic_data/image_generator.py b/datumaro/plugins/synthetic_data/image_generator.py index 5634561333..3f6533ff59 100644 --- a/datumaro/plugins/synthetic_data/image_generator.py +++ b/datumaro/plugins/synthetic_data/image_generator.py @@ -67,8 +67,9 @@ def generate_dataset(self) -> None: mp_ctx = get_context("spawn") # On Mac 10.15 and Python 3.7 fork leads to hangs with mp_ctx.Pool(processes=self._cpu_count) as pool: params = pool.map( - self._generate_category, [Random(i) for i in range(self._categories)] - ) # nosec - disable B311 + self._generate_category, + [Random(i) for i in range(self._categories)], # nosec - disable B311 + ) instances_weights = np.repeat(self._weights, self._instances, axis=0) weight_per_img = np.tile(instances_weights, (self._categories, 1))