From b970c1f6007936267c14ff27a33172ddf762d1bf Mon Sep 17 00:00:00 2001 From: Scott Date: Tue, 14 Apr 2020 23:04:48 -0500 Subject: [PATCH 01/42] Strip down dataset function for testing --- delta/imagery/imagery_dataset.py | 43 +++++++++++++++++++----------- delta/imagery/sources/worldview.py | 3 ++- delta/ml/train.py | 20 +++++++++----- delta/subcommands/classify.py | 5 ++++ delta/subcommands/train.py | 4 +++ scripts/fetch/fetch_hdds_images.py | 11 ++++++-- setup.py | 2 +- tests/conftest.py | 1 + 8 files changed, 63 insertions(+), 26 deletions(-) diff --git a/delta/imagery/imagery_dataset.py b/delta/imagery/imagery_dataset.py index 9722c73a..d8842d98 100644 --- a/delta/imagery/imagery_dataset.py +++ b/delta/imagery/imagery_dataset.py @@ -12,6 +12,8 @@ from delta.imagery import rectangle from delta.imagery.sources import loader +import numpy as np + class ImageryDataset: """Create dataset with all files as described in the provided config file. """ @@ -91,19 +93,26 @@ def _load_images(self, is_labels, data_type): Loads a list of images as tensors. If label_list is specified, load labels instead. The corresponding image files are still required however. """ - ds_input = self._tile_images() - def load_tile(image_index, x1, y1, x2, y2): - img = tf.py_function(functools.partial(self._load_tensor_imagery, - is_labels), - [image_index, [x1, y1, x2, y2]], data_type) - return img - ret = ds_input.map(load_tile, num_parallel_calls=config.threads()) - - return ret.prefetch(tf.data.experimental.AUTOTUNE) + #ds_input = self._tile_images() + #def load_tile(image_index, x1, y1, x2, y2): + # tf.print("load_tile", output_stream=sys.stdout) + # #img = tf.py_function(functools.partial(self._load_tensor_imagery, + # # is_labels), + # # [image_index, [x1, y1, x2, y2]], data_type) + # img = tf.zeros(shape=(128, 128, 8), dtype=tf.float32) # DEBUG + # return img + #ret = ds_input.map(load_tile, num_parallel_calls=1)#config.threads()) + #return ret#.prefetch(4)#tf.data.experimental.AUTOTUNE) + + def fake_tile(dummy): + tf.print("load_tile", output_stream=sys.stdout) + return tf.zeros(shape=(128, 128, 8), dtype=tf.float32) + ret = tf.data.Dataset.range(1000).map(fake_tile, num_parallel_calls=1)#config.threads()) + return ret#.prefetch(4)#tf.data.experimental.AUTOTUNE) def _chunk_image(self, image): """Split up a tensor image into tensor chunks""" - + tf.print("chunk_image", output_stream=sys.stdout) ksizes = [1, self._chunk_size, self._chunk_size, 1] # Size of the chunks strides = [1, self._chunk_stride, self._chunk_stride, 1] # SPacing between chunk starts rates = [1, 1, 1, 1] @@ -117,7 +126,8 @@ def _chunk_image(self, image): def _reshape_labels(self, labels): """Reshape the labels to account for the chunking process.""" w = (self._chunk_size - self._output_size) // 2 - labels = tf.image.crop_to_bounding_box(labels, w, w, tf.shape(labels)[0] - 2 * w, tf.shape(labels)[1] - 2 * w) + labels = tf.image.crop_to_bounding_box(labels, w, w, tf.shape(labels)[0] - 2 * w, + tf.shape(labels)[1] - 2 * w) ksizes = [1, self._output_size, self._output_size, 1] strides = [1, self._chunk_stride, self._chunk_stride, 1] @@ -132,6 +142,7 @@ def data(self): """ ret = self._load_images(False, self._data_type) ret = ret.map(self._chunk_image, num_parallel_calls=config.threads()) + #ret = ret.prefetch(4)#tf.data.experimental.AUTOTUNE) return ret.unbatch() def labels(self): @@ -139,8 +150,8 @@ def labels(self): Unbatched dataset of labels. """ label_set = self._load_images(True, self._label_type) - label_set = label_set.map(self._reshape_labels) - + label_set = label_set.map(self._reshape_labels, num_parallel_calls=config.threads()) + #label_set = label_set.prefetch(4)#tf.data.experimental.AUTOTUNE) return label_set.unbatch() def dataset(self): @@ -151,9 +162,9 @@ def dataset(self): # Pair the data and labels in our dataset ds = tf.data.Dataset.zip((self.data(), self.labels())) # ignore labels with no data - if self._labels.nodata_value(): - ds = ds.filter(lambda x, y: tf.math.not_equal(y, self._labels.nodata_value())) - +# if self._labels.nodata_value(): +# ds = ds.filter(lambda x, y: tf.math.not_equal(y, self._labels.nodata_value())) + #ds = ds.prefetch(tf.data.experimental.AUTOTUNE) return ds def num_bands(self): diff --git a/delta/imagery/sources/worldview.py b/delta/imagery/sources/worldview.py index 74fdd043..519faab5 100644 --- a/delta/imagery/sources/worldview.py +++ b/delta/imagery/sources/worldview.py @@ -57,7 +57,8 @@ def _unpack(self, paths): #print('Already have unpacked files in ' + unpack_folder) pass else: - print('Unpacking file ' + paths + ' to folder ' + unpack_folder) + tf.print('Unpacking file ' + paths + ' to folder ' + unpack_folder, + output_stream=sys.stdout) utilities.unpack_to_folder(paths, unpack_folder) (tif_path, imd_path) = _get_files_from_unpack_folder(unpack_folder) return (tif_path, imd_path) diff --git a/delta/ml/train.py b/delta/ml/train.py index dd21be60..5603e303 100644 --- a/delta/ml/train.py +++ b/delta/ml/train.py @@ -23,9 +23,9 @@ def _devices(num_gpus): ''' devs = None if num_gpus == 0: - devs = [x.name for x in tf.config.list_logical_devices('CPU')] + devs = [x.name for x in tf.config.experimental.list_logical_devices('CPU')] else: - devs = [x.name for x in tf.config.list_logical_devices('GPU')] + devs = [x.name for x in tf.config.experimental.list_logical_devices('GPU')] assert len(devs) >= num_gpus,\ "Requested %d GPUs with only %d available." % (num_gpus, len(devs)) if num_gpus > 0: @@ -44,7 +44,9 @@ def _strategy(devices): def _prep_datasets(ids, tc, chunk_size, output_size): ds = ids.dataset() ds = ds.batch(tc.batch_size) - if tc.validation: + ds = ds.prefetch(12) + if False:#tc.validation: + print('Using validation!') if tc.validation.from_training: validation = ds.take(tc.validation.steps) ds = ds.skip(tc.validation.steps) @@ -56,11 +58,14 @@ def _prep_datasets(ids, tc, chunk_size, output_size): else: vimagery = ImageryDataset(vimg, vlabel, chunk_size, output_size, tc.chunk_stride) validation = vimagery.dataset().batch(tc.batch_size).take(tc.validation.steps) + #validation = validation.prefetch(4)#tf.data.experimental.AUTOTUNE) else: + print('validation = None') validation = None if tc.steps: ds = ds.take(tc.steps) - ds = ds.repeat(tc.epochs) + #ds = ds.prefetch(4)#tf.data.experimental.AUTOTUNE) +# ds = ds.repeat(tc.epochs) return (ds, validation) def _log_mlflow_params(model, dataset, training_spec): @@ -78,7 +83,7 @@ def _log_mlflow_params(model, dataset, training_spec): mlflow.log_param('Batch Size', training_spec.batch_size) mlflow.log_param('Optimizer', training_spec.optimizer) mlflow.log_param('Model Layers', len(model.layers)) - mlflow.log_param('Status', 'Running') + #mlflow.log_param('Status', 'Running') Illegal to change the value! class _MLFlowCallback(tf.keras.callbacks.Callback): """ @@ -183,12 +188,15 @@ def train(model_fn, dataset : ImageryDataset, training_spec): mcb = _mlflow_train_setup(model, dataset, training_spec) callbacks.append(mcb) + print('steps = ', training_spec.steps) + print('val steps = ', training_spec.validation.steps) + #raise Exception('DEBUG') try: history = model.fit(ds, epochs=training_spec.epochs, callbacks=callbacks, validation_data=validation, - validation_steps=training_spec.validation.steps if training_spec.validation else None, + #validation_steps=training_spec.validation.steps if training_spec.validation else None, steps_per_epoch=training_spec.steps) if config.mlflow_enabled(): model_path = os.path.join(mcb.temp_dir, 'final_model.h5') diff --git a/delta/subcommands/classify.py b/delta/subcommands/classify.py index 4beaa080..39912842 100644 --- a/delta/subcommands/classify.py +++ b/delta/subcommands/classify.py @@ -3,6 +3,7 @@ """ import os.path +import time import numpy as np import matplotlib.pyplot as plt import tensorflow as tf @@ -58,6 +59,8 @@ def main(options): error_colors = np.array([[0x0, 0x0, 0x0], [0xFF, 0x00, 0x00]], dtype=np.uint8) + start_time = time.time() + images = config.images() labels = config.labels() @@ -98,4 +101,6 @@ def main(options): if options.autoencoder: tiff.write_tiff('orig_' + base_name + '.tiff', ae_convert(image.read()), metadata=image.metadata()) + stop_time = time.time() + print('Elapsed time = ', stop_time - start_time) return 0 diff --git a/delta/subcommands/train.py b/delta/subcommands/train.py index c79842dc..aa8e7b4f 100644 --- a/delta/subcommands/train.py +++ b/delta/subcommands/train.py @@ -3,6 +3,7 @@ """ import sys +import time import tensorflow as tf @@ -22,6 +23,7 @@ def setup_parser(subparsers): config.setup_arg_parser(sub, train=True) def main(options): + start_time = time.time() images = config.images() if not images: print('No images specified.', file=sys.stderr) @@ -50,4 +52,6 @@ def main(options): print() print('Training cancelled.') + stop_time = time.time() + print('Elapsed time = ', stop_time-start_time) return 0 diff --git a/scripts/fetch/fetch_hdds_images.py b/scripts/fetch/fetch_hdds_images.py index 27c76e18..15330270 100755 --- a/scripts/fetch/fetch_hdds_images.py +++ b/scripts/fetch/fetch_hdds_images.py @@ -53,7 +53,7 @@ def get_dataset_list(options): # Each event is a dataset, start by fetching the list of all HDDS datasets. print('Submitting HDDS dataset query...') results = api.datasets("", CATALOG) - + print(results) if not results['data']: raise Exception('Did not find any HDDS data!') print('Found ' + str(len(results['data'])) + ' matching datasets.') @@ -140,7 +140,7 @@ def main(argsIn): #pylint: disable=R0914,R0912 try: - usage = "usage: get_landsat_dswe_labels [options]" + usage = "usage: fetch_hdds_images.py [options]" parser = argparse.ArgumentParser(usage=usage) parser.add_argument("--output-folder", dest="output_folder", required=True, @@ -163,6 +163,9 @@ def main(argsIn): #pylint: disable=R0914,R0912 dest="refetch_scenes", default=False, help="Force refetches of scene lists for each dataset.") + parser.add_argument("--event-name", dest="event_name", default=None, + help="Only download images from this event.") + options = parser.parse_args(argsIn) except argparse.ArgumentError: @@ -197,6 +200,10 @@ def main(argsIn): #pylint: disable=R0914,R0912 #if counter == 1: # continue + if options.event_name: # Only download images from the specified event + if options.event_name not in full_name: + continue + dataset_folder = os.path.join(options.output_folder, full_name) scene_list_path = os.path.join(dataset_folder, 'scene_list.dat') done_flag_path = os.path.join(dataset_folder, 'done.flag') diff --git a/setup.py b/setup.py index d6e82c5f..52a42107 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ 'numpy', 'scipy', 'matplotlib', - 'tensorflow>=2.1', + 'tensorflow-gpu==2.1', 'mlflow', 'portalocker', 'appdirs', diff --git a/tests/conftest.py b/tests/conftest.py index 377bacab..dc37221d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python #pylint:disable=redefined-outer-name import os import random From 211b6f7408e9fa5b63c53030cdedb2a843c13736 Mon Sep 17 00:00:00 2001 From: Scott Date: Sat, 18 Apr 2020 00:48:08 -0500 Subject: [PATCH 02/42] Performance testing --- delta/imagery/imagery_dataset.py | 54 +++++++++++++++++++++----------- delta/ml/train.py | 3 +- 2 files changed, 37 insertions(+), 20 deletions(-) diff --git a/delta/imagery/imagery_dataset.py b/delta/imagery/imagery_dataset.py index d8842d98..3d4d7f22 100644 --- a/delta/imagery/imagery_dataset.py +++ b/delta/imagery/imagery_dataset.py @@ -93,26 +93,25 @@ def _load_images(self, is_labels, data_type): Loads a list of images as tensors. If label_list is specified, load labels instead. The corresponding image files are still required however. """ - #ds_input = self._tile_images() - #def load_tile(image_index, x1, y1, x2, y2): + ds_input = self._tile_images() + def load_tile(image_index, x1, y1, x2, y2): + #tf.print("load_tile", output_stream=sys.stdout) + img = tf.py_function(functools.partial(self._load_tensor_imagery, + is_labels), + [image_index, [x1, y1, x2, y2]], data_type) + return img + ret = ds_input.map(load_tile, num_parallel_calls=tf.data.experimental.AUTOTUNE)#config.threads()) + return ret#.prefetch(4)#tf.data.experimental.AUTOTUNE) + + #def fake_tile(dummy): # tf.print("load_tile", output_stream=sys.stdout) - # #img = tf.py_function(functools.partial(self._load_tensor_imagery, - # # is_labels), - # # [image_index, [x1, y1, x2, y2]], data_type) - # img = tf.zeros(shape=(128, 128, 8), dtype=tf.float32) # DEBUG - # return img - #ret = ds_input.map(load_tile, num_parallel_calls=1)#config.threads()) + # return tf.zeros(shape=(128, 128, 8), dtype=tf.float32) + #ret = tf.data.Dataset.range(400).map(fake_tile, num_parallel_calls=tf.data.experimental.AUTOTUNE)#config.threads()) #return ret#.prefetch(4)#tf.data.experimental.AUTOTUNE) - - def fake_tile(dummy): - tf.print("load_tile", output_stream=sys.stdout) - return tf.zeros(shape=(128, 128, 8), dtype=tf.float32) - ret = tf.data.Dataset.range(1000).map(fake_tile, num_parallel_calls=1)#config.threads()) - return ret#.prefetch(4)#tf.data.experimental.AUTOTUNE) def _chunk_image(self, image): """Split up a tensor image into tensor chunks""" - tf.print("chunk_image", output_stream=sys.stdout) + #tf.print("chunk_image", output_stream=sys.stdout) ksizes = [1, self._chunk_size, self._chunk_size, 1] # Size of the chunks strides = [1, self._chunk_stride, self._chunk_stride, 1] # SPacing between chunk starts rates = [1, 1, 1, 1] @@ -141,7 +140,7 @@ def data(self): Unbatched dataset of image chunks. """ ret = self._load_images(False, self._data_type) - ret = ret.map(self._chunk_image, num_parallel_calls=config.threads()) + ret = ret.map(self._chunk_image, num_parallel_calls=tf.data.experimental.AUTOTUNE)#config.threads()) #ret = ret.prefetch(4)#tf.data.experimental.AUTOTUNE) return ret.unbatch() @@ -150,7 +149,7 @@ def labels(self): Unbatched dataset of labels. """ label_set = self._load_images(True, self._label_type) - label_set = label_set.map(self._reshape_labels, num_parallel_calls=config.threads()) + label_set = label_set.map(self._reshape_labels, num_parallel_calls=tf.data.experimental.AUTOTUNE)#config.threads()) #label_set = label_set.prefetch(4)#tf.data.experimental.AUTOTUNE) return label_set.unbatch() @@ -159,12 +158,29 @@ def dataset(self): Return the underlying TensorFlow dataset object that this class creates. """ + def double_chunk(image, label): + #di = tf.data.Dataset.from_tensors(image) + #dl = tf.data.Dataset.from_tensors(label) + #di = di.map(self._chunk_image, num_parallel_calls=tf.data.experimental.AUTOTUNE) + #dl = dl.map(self._reshape_labels, num_parallel_calls=tf.data.experimental.AUTOTUNE) + #return tf.data.Dataset.zip((di.unbatch(), dl.unbatch())) + ret = tf.data.Dataset.from_tensors(self._chunk_image(image)) + label_set = tf.data.Dataset.from_tensors(tf.cast(self._reshape_labels(label), tf.float32)) + return tf.data.Dataset.zip((ret.unbatch(), label_set.unbatch())) + + #ret_data = self._load_images(False, self._data_type) + #label_set = self._load_images(True, self._label_type) + #ds = tf.data.Dataset.zip((ret_data, label_set)) + #ds = ds.interleave(double_chunk, num_parallel_calls=tf.data.experimental.AUTOTUNE) + #ds = ds.unbatch() + # Pair the data and labels in our dataset ds = tf.data.Dataset.zip((self.data(), self.labels())) # ignore labels with no data -# if self._labels.nodata_value(): -# ds = ds.filter(lambda x, y: tf.math.not_equal(y, self._labels.nodata_value())) + if self._labels.nodata_value(): + ds = ds.filter(lambda x, y: tf.math.not_equal(y, self._labels.nodata_value())) #ds = ds.prefetch(tf.data.experimental.AUTOTUNE) + print(ds) return ds def num_bands(self): diff --git a/delta/ml/train.py b/delta/ml/train.py index 5603e303..c1796fbe 100644 --- a/delta/ml/train.py +++ b/delta/ml/train.py @@ -44,7 +44,8 @@ def _strategy(devices): def _prep_datasets(ids, tc, chunk_size, output_size): ds = ids.dataset() ds = ds.batch(tc.batch_size) - ds = ds.prefetch(12) + #ds = ds.cache() + ds = ds.prefetch(tf.data.experimental.AUTOTUNE) if False:#tc.validation: print('Using validation!') if tc.validation.from_training: From 7a042e070b8e69127a5f3e320d6823320b5ab66a Mon Sep 17 00:00:00 2001 From: Scott Date: Sun, 19 Apr 2020 21:16:54 -0700 Subject: [PATCH 03/42] Let random split tool do recursive search --- scripts/fetch/random_folder_split.py | 54 ++++++++++++++++------------ 1 file changed, 32 insertions(+), 22 deletions(-) diff --git a/scripts/fetch/random_folder_split.py b/scripts/fetch/random_folder_split.py index 074fc8b7..1b785682 100644 --- a/scripts/fetch/random_folder_split.py +++ b/scripts/fetch/random_folder_split.py @@ -30,7 +30,7 @@ def main(argsIn): parser.add_argument("--image-folder", dest="image_folder", required=True, help="Folder containing the input image files.") - parser.add_argument("--label-folder", dest="label_folder", required=True, + parser.add_argument("--label-folder", dest="label_folder", default=None, help="Folder containing the input label files.") parser.add_argument("--output-folder", dest="output_folder", required=True, @@ -68,37 +68,45 @@ def main(argsIn): os.mkdir(out_train_folder) os.mkdir(out_valid_folder) os.mkdir(train_image_folder) - os.mkdir(train_label_folder) os.mkdir(valid_image_folder) - os.mkdir(valid_label_folder) + if options.label_folder: + os.mkdir(train_label_folder) + os.mkdir(valid_label_folder) - input_image_list = os.listdir(options.image_folder) + # Recursively find image files, obtaining the full path for each file. + input_image_list = [os.path.join(root, name) + for root, dirs, files in os.walk(options.image_folder) + for name in files + if name.endswith((options.image_extension))] train_count = 0 valid_count = 0 - for f in input_image_list: - # Skip other files - ext = os.path.splitext(f)[1] - if ext != options.image_extension: - continue + for image_path in input_image_list: - # Get file names - image_path = os.path.join(options.image_folder, f) - label_path = get_label_path(f, options) - label_name = os.path.basename(label_path) + image_name = os.path.basename(image_path) - # Decide where to make the symlinks, train or label + # Use for validation or for training? use_for_valid = (random.random() < options.validate_fraction) + + # Handle the image file if use_for_valid: - image_dest = os.path.join(valid_image_folder, f) - label_dest = os.path.join(valid_label_folder, label_name) + image_dest = os.path.join(valid_image_folder, image_name) valid_count += 1 else: - image_dest = os.path.join(train_image_folder, f) - label_dest = os.path.join(train_label_folder, label_name) + image_dest = os.path.join(train_image_folder, image_name) train_count += 1 - os.symlink(image_path, image_dest) + + if not options.label_folder: + continue + + # Handle the label file + label_path = get_label_path(image_name, options) + label_name = os.path.basename(label_path) + if use_for_valid: + label_dest = os.path.join(valid_label_folder, label_name) + else: + label_dest = os.path.join(train_label_folder, label_name) os.symlink(label_path, label_dest) # Copy config file if provided @@ -111,14 +119,16 @@ def main(argsIn): config_data = yaml.load(f, Loader=yaml.FullLoader) config_data['images']['directory'] = train_image_folder - config_data['labels']['directory'] = train_label_folder config_data['train']['validation']['images']['directory'] = valid_image_folder - config_data['train']['validation']['labels']['directory'] = valid_label_folder + + if options.label_folder: + config_data['labels']['directory'] = train_label_folder + config_data['train']['validation']['labels']['directory'] = valid_label_folder with open(config_out_path, 'w') as f: yaml.dump(config_data, f) print('Wrote config file: ' + config_out_path) - except Exception as e: + except Exception as e: #pylint: disable=W0703 print('Failed to copy config file!') print(str(e)) From 4222f13226064b7f0db21323467af5f653b040b3 Mon Sep 17 00:00:00 2001 From: Michael Furlong Date: Wed, 29 Apr 2020 11:23:35 -0700 Subject: [PATCH 04/42] Added segnet short model with fewer filters --- .../networks/segnet-short-fewer-filters.yaml | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 delta/config/networks/segnet-short-fewer-filters.yaml diff --git a/delta/config/networks/segnet-short-fewer-filters.yaml b/delta/config/networks/segnet-short-fewer-filters.yaml new file mode 100644 index 00000000..c044bcac --- /dev/null +++ b/delta/config/networks/segnet-short-fewer-filters.yaml @@ -0,0 +1,51 @@ +layers: + - Input: + shape: in_shape + - Conv2D: + filters: 64 + kernel_size: [7, 7] + padding: same + use_bias: false + name: conv_1_1 + - BatchNormalization: + - Activation: + activation: relu + - Conv2D: + filters: 64 + kernel_size: [7, 7] + padding: same + use_bias: false + name: conv_1_2 + - BatchNormalization: + - Activation: + activation: relu + - MaxPooling2D: + pool_size: [2, 2] + strides: 2 + name: pooling_1 + - UpSampling2D: + size: [2, 2] + name: upsampling_5 + - Conv2DTranspose: + filters: 64 + kernel_size: [7, 7] + strides: [1, 1] + padding: same + name: conv_T_5_1 + - BatchNormalization: + - Activation: + activation: relu + - Conv2DTranspose: + filters: 64 + kernel_size: [7, 7] + strides: [1, 1] + padding: same + name: conv_T_5_2 + - BatchNormalization: + - Activation: + activation: relu + - Conv2D: + filters: num_bands + kernel_size: [7, 7] + activation: relu + padding: same From 85117b31b3baf811571698baf0cef586336e0b51 Mon Sep 17 00:00:00 2001 From: Michael Furlong Date: Wed, 29 Apr 2020 11:25:18 -0700 Subject: [PATCH 05/42] Added modified segnet-short network for performance testing --- .../networks/segnet-short-small-filters.yaml | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 delta/config/networks/segnet-short-small-filters.yaml diff --git a/delta/config/networks/segnet-short-small-filters.yaml b/delta/config/networks/segnet-short-small-filters.yaml new file mode 100644 index 00000000..4b0b9498 --- /dev/null +++ b/delta/config/networks/segnet-short-small-filters.yaml @@ -0,0 +1,51 @@ +layers: + - Input: + shape: in_shape + - Conv2D: + filters: 100 + kernel_size: [5, 5] + padding: same + use_bias: false + name: conv_1_1 + - BatchNormalization: + - Activation: + activation: relu + - Conv2D: + filters: 100 + kernel_size: [5, 5] + padding: same + use_bias: false + name: conv_1_2 + - BatchNormalization: + - Activation: + activation: relu + - MaxPooling2D: + pool_size: [2, 2] + strides: 2 + name: pooling_1 + - UpSampling2D: + size: [2, 2] + name: upsampling_5 + - Conv2DTranspose: + filters: 100 + kernel_size: [5, 5] + strides: [1, 1] + padding: same + name: conv_T_5_1 + - BatchNormalization: + - Activation: + activation: relu + - Conv2DTranspose: + filters: 100 + kernel_size: [5, 5] + strides: [1, 1] + padding: same + name: conv_T_5_2 + - BatchNormalization: + - Activation: + activation: relu + - Conv2D: + filters: num_bands + kernel_size: [5, 5] + activation: relu + padding: same From 34f64f5e5ad5bd424d7fd294180866748d5a4e10 Mon Sep 17 00:00:00 2001 From: Michael Furlong Date: Mon, 4 May 2020 16:41:31 -0700 Subject: [PATCH 06/42] Reduced number of filters in segnet networks to decrease training time --- delta/config/networks/segnet-medium.yaml | 16 +++---- delta/config/networks/segnet-short.yaml | 8 ++-- delta/config/networks/segnet.yaml | 54 ++++++++++++------------ 3 files changed, 39 insertions(+), 39 deletions(-) diff --git a/delta/config/networks/segnet-medium.yaml b/delta/config/networks/segnet-medium.yaml index 21c0f6f5..749b66e3 100644 --- a/delta/config/networks/segnet-medium.yaml +++ b/delta/config/networks/segnet-medium.yaml @@ -2,7 +2,7 @@ layers: - Input: shape: in_shape - Conv2D: - filters: 100 + filters: 64 kernel_size: [7, 7] padding: same use_bias: false @@ -11,7 +11,7 @@ layers: - Activation: activation: relu - Conv2D: - filters: 100 + filters: 64 kernel_size: [7, 7] padding: same use_bias: false @@ -24,7 +24,7 @@ layers: strides: 2 name: pooling_1 - Conv2D: - filters: 100 + filters: 64 kernel_size: [7, 7] padding: same use_bias: false @@ -33,7 +33,7 @@ layers: - Activation: activation: relu - Conv2D: - filters: 100 + filters: 64 kernel_size: [7, 7] padding: same use_bias: false @@ -49,7 +49,7 @@ layers: size: [2, 2] name: upsampling_1 - Conv2DTranspose: - filters: 100 + filters: 64 kernel_size: [7, 7] strides: [1, 1] padding: same @@ -58,7 +58,7 @@ layers: - Activation: activation: relu - Conv2DTranspose: - filters: 100 + filters: 64 kernel_size: [7, 7] strides: [1, 1] padding: same @@ -70,7 +70,7 @@ layers: size: [2, 2] name: upsampling_2 - Conv2DTranspose: - filters: 100 + filters: 64 kernel_size: [7, 7] strides: [1, 1] padding: same @@ -79,7 +79,7 @@ layers: - Activation: activation: relu - Conv2DTranspose: - filters: 100 + filters: 64 kernel_size: [7, 7] strides: [1, 1] padding: same diff --git a/delta/config/networks/segnet-short.yaml b/delta/config/networks/segnet-short.yaml index 11fb3113..c044bcac 100644 --- a/delta/config/networks/segnet-short.yaml +++ b/delta/config/networks/segnet-short.yaml @@ -2,7 +2,7 @@ layers: - Input: shape: in_shape - Conv2D: - filters: 100 + filters: 64 kernel_size: [7, 7] padding: same use_bias: false @@ -11,7 +11,7 @@ layers: - Activation: activation: relu - Conv2D: - filters: 100 + filters: 64 kernel_size: [7, 7] padding: same use_bias: false @@ -27,7 +27,7 @@ layers: size: [2, 2] name: upsampling_5 - Conv2DTranspose: - filters: 100 + filters: 64 kernel_size: [7, 7] strides: [1, 1] padding: same @@ -36,7 +36,7 @@ layers: - Activation: activation: relu - Conv2DTranspose: - filters: 100 + filters: 64 kernel_size: [7, 7] strides: [1, 1] padding: same diff --git a/delta/config/networks/segnet.yaml b/delta/config/networks/segnet.yaml index 605afee8..a693b238 100644 --- a/delta/config/networks/segnet.yaml +++ b/delta/config/networks/segnet.yaml @@ -2,7 +2,7 @@ layers: - Input: shape: in_shape - Conv2D: - filters: 100 + filters: 64 kernel_size: [7, 7] padding: same use_bias: false @@ -11,7 +11,7 @@ layers: - Activation: activation: relu - Conv2D: - filters: 100 + filters: 64 kernel_size: [7, 7] padding: same use_bias: false @@ -24,7 +24,7 @@ layers: strides: 2 name: pooling_1 - Conv2D: - filters: 100 + filters: 64 kernel_size: [7, 7] padding: same use_bias: false @@ -33,7 +33,7 @@ layers: - Activation: activation: relu - Conv2D: - filters: 100 + filters: 64 kernel_size: [7, 7] padding: same use_bias: false @@ -46,7 +46,7 @@ layers: strides: 2 name: pooling_2 - Conv2D: - filters: 100 + filters: 64 kernel_size: [7, 7] padding: same use_bias: false @@ -55,7 +55,7 @@ layers: - Activation: activation: relu - Conv2D: - filters: 100 + filters: 64 kernel_size: [7, 7] padding: same use_bias: false @@ -64,7 +64,7 @@ layers: - Activation: activation: relu - Conv2D: - filters: 100 + filters: 64 kernel_size: [7, 7] padding: same use_bias: false @@ -77,7 +77,7 @@ layers: strides: 2 name: pooling_3 - Conv2D: - filters: 100 + filters: 64 kernel_size: [7, 7] padding: same use_bias: false @@ -86,7 +86,7 @@ layers: - Activation: activation: relu - Conv2D: - filters: 100 + filters: 64 kernel_size: [7, 7] padding: same use_bias: false @@ -95,7 +95,7 @@ layers: - Activation: activation: relu - Conv2D: - filters: 100 + filters: 64 kernel_size: [7, 7] padding: same use_bias: false @@ -108,7 +108,7 @@ layers: strides: 2 name: pooling_4 - Conv2D: - filters: 100 + filters: 64 kernel_size: [7, 7] padding: same use_bias: false @@ -117,7 +117,7 @@ layers: - Activation: activation: relu - Conv2D: - filters: 100 + filters: 64 kernel_size: [7, 7] padding: same use_bias: false @@ -126,7 +126,7 @@ layers: - Activation: activation: relu - Conv2D: - filters: 100 + filters: 64 kernel_size: [7, 7] padding: same use_bias: false @@ -142,7 +142,7 @@ layers: size: [2, 2] name: upsampling_1 - Conv2DTranspose: - filters: 100 + filters: 64 kernel_size: [7, 7] strides: [1, 1] padding: same @@ -154,7 +154,7 @@ layers: size: [2, 2] name: upsampling_1 - Conv2DTranspose: - filters: 100 + filters: 64 kernel_size: [7, 7] strides: [1, 1] padding: same @@ -163,7 +163,7 @@ layers: - Activation: activation: relu - Conv2DTranspose: - filters: 100 + filters: 64 kernel_size: [7, 7] strides: [1, 1] padding: same @@ -172,7 +172,7 @@ layers: - Activation: activation: relu - Conv2DTranspose: - filters: 100 + filters: 64 kernel_size: [7, 7] strides: [1, 1] padding: same @@ -184,7 +184,7 @@ layers: size: [2, 2] name: upsampling_2 - Conv2DTranspose: - filters: 100 + filters: 64 kernel_size: [7, 7] strides: [1, 1] padding: same @@ -193,7 +193,7 @@ layers: - Activation: activation: relu - Conv2DTranspose: - filters: 100 + filters: 64 kernel_size: [7, 7] strides: [1, 1] padding: same @@ -202,7 +202,7 @@ layers: - Activation: activation: relu - Conv2DTranspose: - filters: 100 + filters: 64 kernel_size: [7, 7] strides: [1, 1] padding: same @@ -214,7 +214,7 @@ layers: size: [2, 2] name: upsampling_3 - Conv2DTranspose: - filters: 100 + filters: 64 kernel_size: [7, 7] strides: [1, 1] padding: same @@ -223,7 +223,7 @@ layers: - Activation: activation: relu - Conv2DTranspose: - filters: 100 + filters: 64 kernel_size: [7, 7] strides: [1, 1] padding: same @@ -232,7 +232,7 @@ layers: - Activation: activation: relu - Conv2DTranspose: - filters: 100 + filters: 64 kernel_size: [7, 7] strides: [1, 1] padding: same @@ -244,7 +244,7 @@ layers: size: [2, 2] name: upsampling_4 - Conv2DTranspose: - filters: 100 + filters: 64 kernel_size: [7, 7] strides: [1, 1] padding: same @@ -253,7 +253,7 @@ layers: - Activation: activation: relu - Conv2DTranspose: - filters: 100 + filters: 64 kernel_size: [7, 7] strides: [1, 1] padding: same @@ -265,7 +265,7 @@ layers: size: [2, 2] name: upsampling_5 - Conv2DTranspose: - filters: 100 + filters: 64 kernel_size: [7, 7] strides: [1, 1] padding: same @@ -274,7 +274,7 @@ layers: - Activation: activation: relu - Conv2DTranspose: - filters: 100 + filters: 64 kernel_size: [7, 7] strides: [1, 1] padding: same From 89468842085b3441bb2b0d6637dc8f91436035b5 Mon Sep 17 00:00:00 2001 From: Michael Furlong Date: Tue, 5 May 2020 09:41:20 -0700 Subject: [PATCH 07/42] Added other autoencoder_cov models with different filter sizes --- delta/config/networks/autoencoder_conv.yaml | 8 ++--- .../autoencoder_conv_med_filters.yaml | 36 +++++++++++++++++++ .../autoencoder_conv_wide_filters.yaml | 36 +++++++++++++++++++ 3 files changed, 76 insertions(+), 4 deletions(-) create mode 100644 delta/config/networks/autoencoder_conv_med_filters.yaml create mode 100644 delta/config/networks/autoencoder_conv_wide_filters.yaml diff --git a/delta/config/networks/autoencoder_conv.yaml b/delta/config/networks/autoencoder_conv.yaml index c1dd1938..751c0c1c 100644 --- a/delta/config/networks/autoencoder_conv.yaml +++ b/delta/config/networks/autoencoder_conv.yaml @@ -2,28 +2,28 @@ layers: - Input: shape: in_shape - Conv2D: - filters: 300 + filters: 50 kernel_size: [3, 3] activation: relu padding: same - MaxPooling2D: pool_size: [2, 2] - Conv2D: - filters: 150 + filters: 50 kernel_size: [3, 3] activation: relu padding: same - MaxPooling2D: pool_size: [2, 2] - Conv2D: - filters: 75 + filters: 50 kernel_size: [3, 3] activation: relu padding: same - UpSampling2D: size: [2, 2] - Conv2D: - filters: 150 + filters: 50 kernel_size: [3, 3] activation: relu padding: same diff --git a/delta/config/networks/autoencoder_conv_med_filters.yaml b/delta/config/networks/autoencoder_conv_med_filters.yaml new file mode 100644 index 00000000..505deaf6 --- /dev/null +++ b/delta/config/networks/autoencoder_conv_med_filters.yaml @@ -0,0 +1,36 @@ +layers: + - Input: + shape: in_shape + - Conv2D: + filters: 50 + kernel_size: [5, 5] + activation: relu + padding: same + - MaxPooling2D: + pool_size: [2, 2] + - Conv2D: + filters: 50 + kernel_size: [5, 5] + activation: relu + padding: same + - MaxPooling2D: + pool_size: [2, 2] + - Conv2D: + filters: 50 + kernel_size: [5, 5] + activation: relu + padding: same + - UpSampling2D: + size: [2, 2] + - Conv2D: + filters: 50 + kernel_size: [5, 5] + activation: relu + padding: same + - UpSampling2D: + size: [2, 2] + - Conv2D: + filters: num_bands + kernel_size: [5, 5] + activation: relu + padding: same diff --git a/delta/config/networks/autoencoder_conv_wide_filters.yaml b/delta/config/networks/autoencoder_conv_wide_filters.yaml new file mode 100644 index 00000000..7548cd72 --- /dev/null +++ b/delta/config/networks/autoencoder_conv_wide_filters.yaml @@ -0,0 +1,36 @@ +layers: + - Input: + shape: in_shape + - Conv2D: + filters: 50 + kernel_size: [7, 7] + activation: relu + padding: same + - MaxPooling2D: + pool_size: [2, 2] + - Conv2D: + filters: 50 + kernel_size: [7, 7] + activation: relu + padding: same + - MaxPooling2D: + pool_size: [2, 2] + - Conv2D: + filters: 50 + kernel_size: [7, 7] + activation: relu + padding: same + - UpSampling2D: + size: [2, 2] + - Conv2D: + filters: 50 + kernel_size: [7, 7] + activation: relu + padding: same + - UpSampling2D: + size: [2, 2] + - Conv2D: + filters: num_bands + kernel_size: [7, 7] + activation: relu + padding: same From b9c9628990193dd31eb662d60552d6f915acda2f Mon Sep 17 00:00:00 2001 From: Scott Date: Tue, 5 May 2020 20:59:17 -0500 Subject: [PATCH 08/42] Add limit option to random folder split tool --- scripts/fetch/random_folder_split.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/scripts/fetch/random_folder_split.py b/scripts/fetch/random_folder_split.py index 1b785682..6ee35071 100644 --- a/scripts/fetch/random_folder_split.py +++ b/scripts/fetch/random_folder_split.py @@ -44,6 +44,9 @@ def main(argsIn): parser.add_argument("--label-ext", dest="label_extension", default='.tif', help="Extension for label files.") + parser.add_argument("--image-limit", dest="image_limit", default=None, type=int, + help="Only use this many image files total.") + parser.add_argument("--config-file", dest="config_path", default=None, help="Make a copy of this config file with paths changed. The config " + "file must be fully set up, as only the directory entries will be updated.") @@ -97,17 +100,19 @@ def main(argsIn): train_count += 1 os.symlink(image_path, image_dest) - if not options.label_folder: - continue - - # Handle the label file - label_path = get_label_path(image_name, options) - label_name = os.path.basename(label_path) - if use_for_valid: - label_dest = os.path.join(valid_label_folder, label_name) - else: - label_dest = os.path.join(train_label_folder, label_name) - os.symlink(label_path, label_dest) + if options.label_folder: # Handle the label file + label_path = get_label_path(image_name, options) + label_name = os.path.basename(label_path) + if use_for_valid: + label_dest = os.path.join(valid_label_folder, label_name) + else: + label_dest = os.path.join(train_label_folder, label_name) + os.symlink(label_path, label_dest) + + # Check the image limit if it was specified + total_count = valid_count + train_count + if options.image_limit and (total_count >= options.image_limit): + break # Copy config file if provided if options.config_path: From 1bd01a427e1928c3d6f082919e286427078fc6bb Mon Sep 17 00:00:00 2001 From: Scott Date: Wed, 6 May 2020 14:54:13 -0500 Subject: [PATCH 09/42] Fix issue with image fetcher where not all of large sets were downloaded --- scripts/fetch/fetch_hdds_images.py | 61 ++++++++++++++++++++++++++---- 1 file changed, 53 insertions(+), 8 deletions(-) diff --git a/scripts/fetch/fetch_hdds_images.py b/scripts/fetch/fetch_hdds_images.py index 15330270..119bb9e7 100755 --- a/scripts/fetch/fetch_hdds_images.py +++ b/scripts/fetch/fetch_hdds_images.py @@ -60,7 +60,7 @@ def get_dataset_list(options): # Go through all the datasets and identify the events we are interested in. TARGET_TYPES = ['flood', 'hurricane', 'cyclone', 'tsunami', 'dam_collapse', 'storm'] - SKIP = ['test', 'snowstorm', 'adhoc', 'ad hoc', 'ad_hoc'] # TODO: What is ad hoc here? + SKIP = ['test', 'icestorm', 'snowstorm', 'adhoc', 'ad hoc', 'ad_hoc'] # TODO: What is ad hoc here? handle = open(dataset_cache_path, 'w') @@ -201,7 +201,7 @@ def main(argsIn): #pylint: disable=R0914,R0912 # continue if options.event_name: # Only download images from the specified event - if options.event_name not in full_name: + if options.event_name.lower() not in full_name.lower(): continue dataset_folder = os.path.join(options.output_folder, full_name) @@ -216,12 +216,35 @@ def main(argsIn): #pylint: disable=R0914,R0912 print('--> Search scenes for: ' + full_name) + BATCH_SIZE = 10000 if not os.path.exists(scene_list_path) or options.refetch_scenes: # Request the scene list from USGS #details = {'Agency - Platform - Vendor':'WORLDVIEW', 'Sensor Type':'MS'} #details = {'sensor_type':'MS'} details = {} # TODO: How do these work?? - results = api.search(dataset, CATALOG, where=details, max_results=5000, extended=False) + + # Large sets of results require multiple queries + done = False + error = False + all_scenes = [] + while not done: + print('starting_number = ' + str(len(all_scenes))) + results = api.search(dataset, CATALOG, where=details, + max_results=BATCH_SIZE, + starting_number=len(all_scenes), extended=False) + + if 'results' not in results['data']: + print('ERROR: Failed to get any results for dataset: ' + full_name) + error = True + break + if len(results['data']['results']) < BATCH_SIZE: + done = True + all_scenes += results['data']['results'] + + if error: + continue + + results['data']['results'] = all_scenes # Cache the results to disk with open(scene_list_path, 'wb') as f: @@ -231,9 +254,6 @@ def main(argsIn): #pylint: disable=R0914,R0912 with open(scene_list_path, 'rb') as f: results = pickle.load(f) - if 'results' not in results['data']: - print('ERROR: Failed to get any results for dataset: ' + full_name) - continue print('Got ' + str(len(results['data']['results'])) + ' scene results.') for scene in results['data']['results']: @@ -241,7 +261,7 @@ def main(argsIn): #pylint: disable=R0914,R0912 #print(scene) fail = False - REQUIRED_PARTS = ['displayId', 'summary'] + REQUIRED_PARTS = ['displayId', 'summary', 'entityId', 'displayId'] for p in REQUIRED_PARTS: if (p not in scene) or (not scene[p]): print('scene object is missing element: ' + p) @@ -250,6 +270,29 @@ def main(argsIn): #pylint: disable=R0914,R0912 if fail: continue + # DEBUG: Only download these files! + desired_ids = [ +'WV02S11_162083E030_4279162019122200000000MS00', +'WV02N16_024291W016_4697352013033100000000MS00', +'WV02N34_364027E132_7086112018071500000000MS00', +'WV02N39_970882W091_5066782018102300000000MS00', +'WV02N50_838472E043_9616662016071800000000MS00', +'WV02N26_526388E086_9256942017082300000000MS00', +'WV02N08_267222W062_7001382017081000000000MS00', +'WV02S28_527916W071_1270832017052600000000MS00', +'WV02N39_620277W118_4440272016110200000000MS00', +'WV02N59_979444W149_6266662017091900000000MS00', +'WV02N28_708333W096_0125002017083000000000MS00', +'WV02N29_982083W095_0877772017083100000000MS00', +'WV02N36_728780E007_9239362011080800000000MS00', +'WV02N46_676908W092_1901272012062400000000MS00', +'WV02N48_971380W097_2213192013051200000000MS00', +'WV02N29_782222W085_1613882018101200000000MS00', +'WV02N34_105694E134_1177772016032600000000MS00', +'WV02N15_373714E032_3806602013081100000000MS01'] + + #if scene['displayId'] not in desired_ids: + # continue # Figure out the downloaded file path for this image file_name = scene['displayId'] + '.zip' @@ -276,6 +319,7 @@ def main(argsIn): #pylint: disable=R0914,R0912 print('Undesired sensor: ' + scene['summary']) continue + # Investigate the number of bands PLATFORM_BAND_COUNTS = {'worldview':8, 'TODO':1} min_num_bands = PLATFORM_BAND_COUNTS[platform] @@ -318,6 +362,7 @@ def main(argsIn): #pylint: disable=R0914,R0912 if not ready: raise Exception('Missing download option for scene: ' + str(types)) + # Get the download URL of the file we want. r = api.download(dataset, CATALOG, [scene['entityId']], product=download_type) @@ -336,7 +381,7 @@ def main(argsIn): #pylint: disable=R0914,R0912 #raise Exception('DEBUG') print('Finished processing dataset: ' + full_name) - os.system('touch ' + done_flag_path) # Mark this dataset as finished +# os.system('touch ' + done_flag_path) # Mark this dataset as finished #raise Exception('DEBUG') #if not os.path.exists(output_path): From d9dd3d31b411bda65b52476b0c87eae284d4e062 Mon Sep 17 00:00:00 2001 From: Scott Date: Wed, 6 May 2020 23:15:03 -0700 Subject: [PATCH 10/42] - Make dataset continue past bad datasets - Added convert_image_list helper tool - Added more options to other tools --- delta/imagery/imagery_dataset.py | 35 ++++++++--------- scripts/fetch/convert_image_list.py | 58 ++++++++++++++++++++++++++++ scripts/fetch/fetch_hdds_images.py | 45 ++++++++------------- scripts/fetch/random_folder_split.py | 13 +++++++ 4 files changed, 105 insertions(+), 46 deletions(-) create mode 100755 scripts/fetch/convert_image_list.py diff --git a/delta/imagery/imagery_dataset.py b/delta/imagery/imagery_dataset.py index 3d4d7f22..f435712e 100644 --- a/delta/imagery/imagery_dataset.py +++ b/delta/imagery/imagery_dataset.py @@ -12,7 +12,7 @@ from delta.imagery import rectangle from delta.imagery.sources import loader -import numpy as np +#import numpy as np class ImageryDataset: """Create dataset with all files as described in the provided config file. @@ -101,13 +101,12 @@ def load_tile(image_index, x1, y1, x2, y2): [image_index, [x1, y1, x2, y2]], data_type) return img ret = ds_input.map(load_tile, num_parallel_calls=tf.data.experimental.AUTOTUNE)#config.threads()) + + # Don't let the entire session be taken down by one bad dataset input. + # - Would be better to handle this somehow but it is not clear if TF supports that. + ret = ret.apply(tf.data.experimental.ignore_errors()) + return ret#.prefetch(4)#tf.data.experimental.AUTOTUNE) - - #def fake_tile(dummy): - # tf.print("load_tile", output_stream=sys.stdout) - # return tf.zeros(shape=(128, 128, 8), dtype=tf.float32) - #ret = tf.data.Dataset.range(400).map(fake_tile, num_parallel_calls=tf.data.experimental.AUTOTUNE)#config.threads()) - #return ret#.prefetch(4)#tf.data.experimental.AUTOTUNE) def _chunk_image(self, image): """Split up a tensor image into tensor chunks""" @@ -126,7 +125,7 @@ def _reshape_labels(self, labels): """Reshape the labels to account for the chunking process.""" w = (self._chunk_size - self._output_size) // 2 labels = tf.image.crop_to_bounding_box(labels, w, w, tf.shape(labels)[0] - 2 * w, - tf.shape(labels)[1] - 2 * w) + tf.shape(labels)[1] - 2 * w) #pylint: disable=C0330 ksizes = [1, self._output_size, self._output_size, 1] strides = [1, self._chunk_stride, self._chunk_stride, 1] @@ -149,7 +148,7 @@ def labels(self): Unbatched dataset of labels. """ label_set = self._load_images(True, self._label_type) - label_set = label_set.map(self._reshape_labels, num_parallel_calls=tf.data.experimental.AUTOTUNE)#config.threads()) + label_set = label_set.map(self._reshape_labels, num_parallel_calls=tf.data.experimental.AUTOTUNE)#config.threads()) #pylint: disable=C0301 #label_set = label_set.prefetch(4)#tf.data.experimental.AUTOTUNE) return label_set.unbatch() @@ -158,15 +157,15 @@ def dataset(self): Return the underlying TensorFlow dataset object that this class creates. """ - def double_chunk(image, label): - #di = tf.data.Dataset.from_tensors(image) - #dl = tf.data.Dataset.from_tensors(label) - #di = di.map(self._chunk_image, num_parallel_calls=tf.data.experimental.AUTOTUNE) - #dl = dl.map(self._reshape_labels, num_parallel_calls=tf.data.experimental.AUTOTUNE) - #return tf.data.Dataset.zip((di.unbatch(), dl.unbatch())) - ret = tf.data.Dataset.from_tensors(self._chunk_image(image)) - label_set = tf.data.Dataset.from_tensors(tf.cast(self._reshape_labels(label), tf.float32)) - return tf.data.Dataset.zip((ret.unbatch(), label_set.unbatch())) + #def double_chunk(image, label): + # #di = tf.data.Dataset.from_tensors(image) + # #dl = tf.data.Dataset.from_tensors(label) + # #di = di.map(self._chunk_image, num_parallel_calls=tf.data.experimental.AUTOTUNE) + # #dl = dl.map(self._reshape_labels, num_parallel_calls=tf.data.experimental.AUTOTUNE) + # #return tf.data.Dataset.zip((di.unbatch(), dl.unbatch())) + # ret = tf.data.Dataset.from_tensors(self._chunk_image(image)) + # label_set = tf.data.Dataset.from_tensors(tf.cast(self._reshape_labels(label), tf.float32)) + # return tf.data.Dataset.zip((ret.unbatch(), label_set.unbatch())) #ret_data = self._load_images(False, self._data_type) #label_set = self._load_images(True, self._label_type) diff --git a/scripts/fetch/convert_image_list.py b/scripts/fetch/convert_image_list.py new file mode 100755 index 00000000..530c9015 --- /dev/null +++ b/scripts/fetch/convert_image_list.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# __BEGIN_LICENSE__ +# Copyright (c) 2009-2013, United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. All +# rights reserved. +# +# The NGT platform is licensed under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the +# License. You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# __END_LICENSE__ + +#pylint: disable=R0915,R0914,R0912 +""" +Script to extract a list of associated image files from the label file csv list. +""" +import sys + + +def main(argsIn): #pylint: disable=R0914,R0912 + + if len(argsIn) != 2: + print("usage: convert_image_list.py ") + + input_path = argsIn[0] + output_path = argsIn[1] + + # Just find the image name for every line with a label ID (integer) + output_list = [] + with open(input_path, 'r') as f: + for line in f: + parts = line.split(',') + try: + label_num = int(parts[0]) #pylint: disable=W0612 + image_name = parts[1] + output_list.append(image_name) + #print('%s -> %s' % (label_num, image_name)) + # Header lines etc will throw exceptions trying to cast the integer + except: #pylint: disable=W0702 + pass + + # Write out a text file with all of the image names. + with open(output_path, 'w') as f: + for line in output_list: + f.write(line+'\n') + print('Wrote out ' + str(len(output_list)) + ' items.') + + return 0 + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/scripts/fetch/fetch_hdds_images.py b/scripts/fetch/fetch_hdds_images.py index 119bb9e7..be2ebd0b 100755 --- a/scripts/fetch/fetch_hdds_images.py +++ b/scripts/fetch/fetch_hdds_images.py @@ -163,6 +163,9 @@ def main(argsIn): #pylint: disable=R0914,R0912 dest="refetch_scenes", default=False, help="Force refetches of scene lists for each dataset.") + parser.add_argument("--image-list-path", dest="image_list_path", default=None, + help="Path to text file containing list of image IDs to download, one per line.") + parser.add_argument("--event-name", dest="event_name", default=None, help="Only download images from this event.") @@ -175,6 +178,12 @@ def main(argsIn): #pylint: disable=R0914,R0912 if options.output_folder and not os.path.exists(options.output_folder): os.mkdir(options.output_folder) + images_to_use = [] + if options.image_list_path: + with open(options.image_list_path, 'r') as f: + for line in f: + images_to_use.append(line.strip()) + # Only log in if our session expired (ugly function use to check!) if options.force_login or (not api._get_api_key(None)): #pylint: disable=W0212 print('Logging in to USGS EarthExplorer...') @@ -223,14 +232,14 @@ def main(argsIn): #pylint: disable=R0914,R0912 #details = {'sensor_type':'MS'} details = {} # TODO: How do these work?? - # Large sets of results require multiple queries + # Large sets of results require multiple queries in order to get all of the data done = False error = False - all_scenes = [] + all_scenes = [] # Acculumate all scene data here while not done: - print('starting_number = ' + str(len(all_scenes))) + print('Searching with start offset = ' + str(len(all_scenes))) results = api.search(dataset, CATALOG, where=details, - max_results=BATCH_SIZE, + max_results=BATCH_SIZE, starting_number=len(all_scenes), extended=False) if 'results' not in results['data']: @@ -270,29 +279,9 @@ def main(argsIn): #pylint: disable=R0914,R0912 if fail: continue - # DEBUG: Only download these files! - desired_ids = [ -'WV02S11_162083E030_4279162019122200000000MS00', -'WV02N16_024291W016_4697352013033100000000MS00', -'WV02N34_364027E132_7086112018071500000000MS00', -'WV02N39_970882W091_5066782018102300000000MS00', -'WV02N50_838472E043_9616662016071800000000MS00', -'WV02N26_526388E086_9256942017082300000000MS00', -'WV02N08_267222W062_7001382017081000000000MS00', -'WV02S28_527916W071_1270832017052600000000MS00', -'WV02N39_620277W118_4440272016110200000000MS00', -'WV02N59_979444W149_6266662017091900000000MS00', -'WV02N28_708333W096_0125002017083000000000MS00', -'WV02N29_982083W095_0877772017083100000000MS00', -'WV02N36_728780E007_9239362011080800000000MS00', -'WV02N46_676908W092_1901272012062400000000MS00', -'WV02N48_971380W097_2213192013051200000000MS00', -'WV02N29_782222W085_1613882018101200000000MS00', -'WV02N34_105694E134_1177772016032600000000MS00', -'WV02N15_373714E032_3806602013081100000000MS01'] - - #if scene['displayId'] not in desired_ids: - # continue + # If image list was provided skip other image names + if images_to_use and (scene['displayId'] not in images_to_use): + continue # Figure out the downloaded file path for this image file_name = scene['displayId'] + '.zip' @@ -362,7 +351,7 @@ def main(argsIn): #pylint: disable=R0914,R0912 if not ready: raise Exception('Missing download option for scene: ' + str(types)) - + # Get the download URL of the file we want. r = api.download(dataset, CATALOG, [scene['entityId']], product=download_type) diff --git a/scripts/fetch/random_folder_split.py b/scripts/fetch/random_folder_split.py index 6ee35071..e10de350 100644 --- a/scripts/fetch/random_folder_split.py +++ b/scripts/fetch/random_folder_split.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +#pylint: disable=R0914 """ Given folders of input image/label files, create a new pair of train/validate folders which contain symlinks to random non-overlapping subsets of the input files. @@ -47,6 +48,9 @@ def main(argsIn): parser.add_argument("--image-limit", dest="image_limit", default=None, type=int, help="Only use this many image files total.") + parser.add_argument("--file-list-path", dest="file_list_path", default=None, + help="Path to text file containing list of image file names to use, one per line.") + parser.add_argument("--config-file", dest="config_path", default=None, help="Make a copy of this config file with paths changed. The config " + "file must be fully set up, as only the directory entries will be updated.") @@ -82,11 +86,20 @@ def main(argsIn): for name in files if name.endswith((options.image_extension))] + images_to_use = [] + if options.file_list_path: + with open(options.file_list_path, 'r') as f: + for line in f: + images_to_use.append(line.strip()) + train_count = 0 valid_count = 0 for image_path in input_image_list: + # If an image list was provided skip images which are not in the list. image_name = os.path.basename(image_path) + if images_to_use and (image_name not in images_to_use): + continue # Use for validation or for training? use_for_valid = (random.random() < options.validate_fraction) From d41eafe4dcf826feb441b67368fb5179796e387c Mon Sep 17 00:00:00 2001 From: Scott Date: Sun, 10 May 2020 23:48:54 -0500 Subject: [PATCH 11/42] Minor updates before merging --- delta/imagery/sources/worldview.py | 3 +++ delta/ml/train.py | 2 +- delta/subcommands/train.py | 7 +++++++ scripts/fetch/random_folder_split.py | 2 +- 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/delta/imagery/sources/worldview.py b/delta/imagery/sources/worldview.py index 519faab5..1d957f64 100644 --- a/delta/imagery/sources/worldview.py +++ b/delta/imagery/sources/worldview.py @@ -5,8 +5,11 @@ import math import functools import os +import sys import numpy as np +import tensorflow as tf + from delta.config import config from delta.imagery import utilities from . import tiff diff --git a/delta/ml/train.py b/delta/ml/train.py index c1796fbe..42a919ee 100644 --- a/delta/ml/train.py +++ b/delta/ml/train.py @@ -198,7 +198,7 @@ def train(model_fn, dataset : ImageryDataset, training_spec): callbacks=callbacks, validation_data=validation, #validation_steps=training_spec.validation.steps if training_spec.validation else None, - steps_per_epoch=training_spec.steps) + steps_per_epoch=training_spec.steps)#, verbose=2) if config.mlflow_enabled(): model_path = os.path.join(mcb.temp_dir, 'final_model.h5') print('\nFinished, saving model to %s.' % (mlflow.get_artifact_uri() + '/final_model.h5')) diff --git a/delta/subcommands/train.py b/delta/subcommands/train.py index aa8e7b4f..99dd1ea8 100644 --- a/delta/subcommands/train.py +++ b/delta/subcommands/train.py @@ -5,6 +5,9 @@ import sys import time +#import logging +#logging.getLogger("tensorflow").setLevel(logging.DEBUG) + import tensorflow as tf from delta.config import config @@ -13,6 +16,10 @@ from delta.ml.model_parser import config_model from delta.ml.layers import ALL_LAYERS + +#tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.DEBUG) + + def setup_parser(subparsers): sub = subparsers.add_parser('train', help='Train a task-specific classifier.') sub.add_argument('--autoencoder', action='store_true', diff --git a/scripts/fetch/random_folder_split.py b/scripts/fetch/random_folder_split.py index e10de350..981ec786 100644 --- a/scripts/fetch/random_folder_split.py +++ b/scripts/fetch/random_folder_split.py @@ -98,7 +98,7 @@ def main(argsIn): # If an image list was provided skip images which are not in the list. image_name = os.path.basename(image_path) - if images_to_use and (image_name not in images_to_use): + if images_to_use and (os.path.splitext(image_name)[0] not in images_to_use): continue # Use for validation or for training? From e5e9230bb3dc2a2e4aba6a89057ee0be051caa5a Mon Sep 17 00:00:00 2001 From: Scott Date: Sun, 10 May 2020 21:58:31 -0700 Subject: [PATCH 12/42] Update fetch message --- scripts/fetch/fetch_hdds_images.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/fetch/fetch_hdds_images.py b/scripts/fetch/fetch_hdds_images.py index be2ebd0b..937e9d4c 100755 --- a/scripts/fetch/fetch_hdds_images.py +++ b/scripts/fetch/fetch_hdds_images.py @@ -267,8 +267,6 @@ def main(argsIn): #pylint: disable=R0914,R0912 for scene in results['data']['results']: - #print(scene) - fail = False REQUIRED_PARTS = ['displayId', 'summary', 'entityId', 'displayId'] for p in REQUIRED_PARTS: @@ -327,7 +325,7 @@ def main(argsIn): #pylint: disable=R0914,R0912 if not num_bands: raise KeyError() # Treat like the except case if num_bands < min_num_bands: - print('Skipping, too few bands: ' + str(num_bands)) + print('Skipping %s, too few bands: %d' % (scene['displayId'], num_bands)) continue except KeyError: print('Unable to perform metadata check!') From ec9839fa59095c3704c08b8cbc8e554ec14d8b77 Mon Sep 17 00:00:00 2001 From: Scott Date: Mon, 11 May 2020 16:09:15 -0700 Subject: [PATCH 13/42] Fix merge bugs --- delta/ml/train.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/delta/ml/train.py b/delta/ml/train.py index 7dc45173..b6da7da8 100644 --- a/delta/ml/train.py +++ b/delta/ml/train.py @@ -82,7 +82,7 @@ def _prep_datasets(ids, tc, chunk_size, output_size): if tc.steps: ds = ds.take(tc.steps) #ds = ds.prefetch(4)#tf.data.experimental.AUTOTUNE) -# ds = ds.repeat(tc.epochs) + #ds = ds.repeat(tc.epochs) return (ds, validation) def _log_mlflow_params(model, dataset, training_spec): @@ -205,9 +205,6 @@ def train(model_fn, dataset : ImageryDataset, training_spec): mcb = _mlflow_train_setup(model, dataset, training_spec) callbacks.append(mcb) - print('steps = ', training_spec.steps) - print('val steps = ', training_spec.validation.steps) - #raise Exception('DEBUG') try: history = model.fit(ds, epochs=training_spec.epochs, @@ -215,7 +212,7 @@ def train(model_fn, dataset : ImageryDataset, training_spec): validation_data=validation, validation_steps=training_spec.validation.steps if training_spec.validation else None, steps_per_epoch=training_spec.steps)#, verbose=2) - if config.mlflow_enabled(): + if config.mlflow.enabled(): model_path = os.path.join(mcb.temp_dir, 'final_model.h5') print('\nFinished, saving model to %s.' % (mlflow.get_artifact_uri() + '/final_model.h5')) model.save(model_path, save_format='h5') From a0ede636c88dbef44ae580d006f9ac84cb0e032e Mon Sep 17 00:00:00 2001 From: Brian Coltin Date: Thu, 14 May 2020 16:55:27 -0700 Subject: [PATCH 14/42] Fix bug with validation config. --- delta/ml/ml_config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/delta/ml/ml_config.py b/delta/ml/ml_config.py index 537ea4b0..99004431 100644 --- a/delta/ml/ml_config.py +++ b/delta/ml/ml_config.py @@ -116,8 +116,8 @@ def __init__(self): 'If from training, validate for this many steps.') self.register_field('from_training', bool, 'from_training', None, None, 'Take validation data from training data.') - self.register_component(ImageSetConfig(), 'images') - self.register_component(ImageSetConfig(), 'labels') + self.register_component(ImageSetConfig(), 'images', '__image_comp') + self.register_component(ImageSetConfig(), 'labels', '__label_comp') self.__images = None self.__labels = None From bcf310a7bb5878e531c28b1506f1e988525f295b Mon Sep 17 00:00:00 2001 From: Scott Date: Sat, 16 May 2020 20:25:35 -0500 Subject: [PATCH 15/42] fix bugs loading autoencoder training data --- delta/imagery/sources/worldview.py | 23 +++++++++++++---------- delta/ml/train.py | 13 +++++++++---- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/delta/imagery/sources/worldview.py b/delta/imagery/sources/worldview.py index 9cd81edc..2df40f39 100644 --- a/delta/imagery/sources/worldview.py +++ b/delta/imagery/sources/worldview.py @@ -24,6 +24,7 @@ import os import sys import numpy as np +import portalocker import tensorflow as tf @@ -70,17 +71,19 @@ def _unpack(self, paths): name = '_'.join([self._sensor, self._date]) unpack_folder = config.io.cache.manager().register_item(name) - # Check if we already unpacked this data - (tif_path, imd_path) = _get_files_from_unpack_folder(unpack_folder) - - if imd_path and tif_path: - #print('Already have unpacked files in ' + unpack_folder) - pass - else: - tf.print('Unpacking file ' + paths + ' to folder ' + unpack_folder, - output_stream=sys.stdout) - utilities.unpack_to_folder(paths, unpack_folder) + with portalocker.Lock(paths, 'r', timeout=300) as unused: + # Check if we already unpacked this data (tif_path, imd_path) = _get_files_from_unpack_folder(unpack_folder) + + if imd_path and tif_path: + tf.print('Already have unpacked files in ' + unpack_folder, + output_stream=sys.stdout) + pass + else: + tf.print('Unpacking file ' + paths + ' to folder ' + unpack_folder, + output_stream=sys.stdout) + utilities.unpack_to_folder(paths, unpack_folder) + (tif_path, imd_path) = _get_files_from_unpack_folder(unpack_folder) return (tif_path, imd_path) # This function is currently set up for the HDDS archived WV data, files from other diff --git a/delta/ml/train.py b/delta/ml/train.py index b6da7da8..43df1859 100644 --- a/delta/ml/train.py +++ b/delta/ml/train.py @@ -28,6 +28,7 @@ from delta.config import config from delta.imagery.imagery_dataset import ImageryDataset +from delta.imagery.imagery_dataset import AutoencoderDataset from .layers import DeltaLayer def _devices(num_gpus): @@ -70,19 +71,22 @@ def _prep_datasets(ids, tc, chunk_size, output_size): else: vimg = tc.validation.images vlabel = tc.validation.labels - if not vimg or not vlabel: + if not vimg: validation = None else: - vimagery = ImageryDataset(vimg, vlabel, chunk_size, output_size, tc.chunk_stride) + if vlabel: + vimagery = ImageryDataset(vimg, vlabel, chunk_size, output_size, tc.chunk_stride) + else: + vimagery = AutoencoderDataset(vimg, chunk_size, tc.chunk_stride) validation = vimagery.dataset().batch(tc.batch_size).take(tc.validation.steps) #validation = validation.prefetch(4)#tf.data.experimental.AUTOTUNE) else: - print('validation = None') + validation = None if tc.steps: ds = ds.take(tc.steps) #ds = ds.prefetch(4)#tf.data.experimental.AUTOTUNE) - #ds = ds.repeat(tc.epochs) + ds = ds.repeat(tc.epochs) return (ds, validation) def _log_mlflow_params(model, dataset, training_spec): @@ -206,6 +210,7 @@ def train(model_fn, dataset : ImageryDataset, training_spec): callbacks.append(mcb) try: + print(training_spec.validation) history = model.fit(ds, epochs=training_spec.epochs, callbacks=callbacks, From 64047e938b95c76500441b749ce79e668f9aba59 Mon Sep 17 00:00:00 2001 From: Michael Furlong Date: Wed, 20 May 2020 13:05:44 -0700 Subject: [PATCH 16/42] Changed the default number of filters in the convpool network --- delta/config/networks/convpool.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/delta/config/networks/convpool.yaml b/delta/config/networks/convpool.yaml index bf4fb806..c1f8d0c9 100644 --- a/delta/config/networks/convpool.yaml +++ b/delta/config/networks/convpool.yaml @@ -1,6 +1,6 @@ params: dropout_rate: 0.3 - num_filters: 100 + num_filters: 64 layers: - Input: shape: in_shape From e3e92f1b4b779076b802d3fcf0b44504235a567f Mon Sep 17 00:00:00 2001 From: Scott Date: Wed, 20 May 2020 14:52:01 -0700 Subject: [PATCH 17/42] Added label image size check --- delta/imagery/imagery_dataset.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/delta/imagery/imagery_dataset.py b/delta/imagery/imagery_dataset.py index 58e7a790..d04f79a1 100644 --- a/delta/imagery/imagery_dataset.py +++ b/delta/imagery/imagery_dataset.py @@ -72,6 +72,11 @@ def tile_generator(): tgs = [] for i in range(len(self._images)): img = loader.load_image(self._images, i) + if self._labels: # If we have labels make sure they are the same size as the input images + label = loader.load_image(self._labels, i) + if label.size() != img.size(): + raise Exception('Label file ' + self._labels[i] + ' with size ' + str(label.size()) + + ' does not match input image size of ' + str(img.size())) # w * h * bands * 4 * chunk * chunk = max_block_bytes tile_width = int(math.sqrt(max_block_bytes / img.num_bands() / self._data_type.size / config.io.tile_ratio())) From c88349449013128d65cadb25a884fb69542a3ef6 Mon Sep 17 00:00:00 2001 From: Scott Date: Wed, 27 May 2020 15:19:21 -0700 Subject: [PATCH 18/42] Fix WV cache bug --- delta/imagery/sources/worldview.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/delta/imagery/sources/worldview.py b/delta/imagery/sources/worldview.py index 2df40f39..2d9c9242 100644 --- a/delta/imagery/sources/worldview.py +++ b/delta/imagery/sources/worldview.py @@ -68,16 +68,15 @@ def __init__(self, paths): def _unpack(self, paths): # Get the folder where this will be stored from the cache manager - name = '_'.join([self._sensor, self._date]) - unpack_folder = config.io.cache.manager().register_item(name) + unpack_folder = config.io.cache.manager().register_item(self._name) with portalocker.Lock(paths, 'r', timeout=300) as unused: # Check if we already unpacked this data (tif_path, imd_path) = _get_files_from_unpack_folder(unpack_folder) if imd_path and tif_path: - tf.print('Already have unpacked files in ' + unpack_folder, - output_stream=sys.stdout) + #tf.print('Already have unpacked files in ' + unpack_folder, + # output_stream=sys.stdout) pass else: tf.print('Unpacking file ' + paths + ' to folder ' + unpack_folder, @@ -96,7 +95,8 @@ def _prep(self, paths): assert isinstance(paths, str) parts = os.path.basename(paths).split('_') self._sensor = parts[0][0:4] - self._date = parts[2][6:14] + self._date = parts[2][6:14] + self._name = os.path.splitext(os.path.basename(paths))[0] (tif_path, imd_path) = self._unpack(paths) From f7aa46cceba71dc357239d4cb34546a617bccf5d Mon Sep 17 00:00:00 2001 From: Michael Furlong Date: Wed, 17 Jun 2020 16:10:54 -0700 Subject: [PATCH 19/42] Added TerminateOnNaN callback as default for all training --- delta/ml/train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/delta/ml/train.py b/delta/ml/train.py index 43df1859..71d9c40a 100644 --- a/delta/ml/train.py +++ b/delta/ml/train.py @@ -190,7 +190,7 @@ def train(model_fn, dataset : ImageryDataset, training_spec): (ds, validation) = _prep_datasets(dataset, training_spec, chunk_size, output_shape[1]) - callbacks = [] + callbacks = [tf.keras.callbacks.TerminateOnNaN()] # add callbacks from DeltaLayers for l in model.layers: if isinstance(l, DeltaLayer): From 48cfc4cfc37be25c0c650548a6db7320e0af21e7 Mon Sep 17 00:00:00 2001 From: Michael Furlong Date: Mon, 22 Jun 2020 17:05:21 -0700 Subject: [PATCH 20/42] Improved config file to be able create loss objects --- delta/ml/ml_config.py | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/delta/ml/ml_config.py b/delta/ml/ml_config.py index 99004431..5751d3d7 100644 --- a/delta/ml/ml_config.py +++ b/delta/ml/ml_config.py @@ -24,9 +24,40 @@ import pkg_resources import yaml +import tensorflow.keras.losses + from delta.imagery.imagery_config import ImageSet, ImageSetConfig, load_images_labels import delta.config as config + +def loss_function_factory(loss_spec): + ''' + loss_function_factory - Creates a loss function object, if an object is specified in the + config file, or a string if that is all that is specified. + + :param: loss_spec Specification of the loss function. Either a string that is compatible + with the keras interface (e.g. 'categorical_crossentropy') or an object defined by a dict + of the form {'LossFunctionName': {'arg1':arg1_val, ...,'argN',argN_val}} + ''' + + if isinstance(loss_spec, str): + return loss_spec + + if isinstance(loss_spec, list): + assert len(loss_spec) == 1, 'Too many loss functions specified' + assert isinstance(loss_spec[0], dict), '''Loss functions objects and parameters must + be specified as a yaml dictionary object + ''' + assert len(loss_spec[0].keys()) == 1, f'Too many loss functions specified: {dict.keys()}' + loss_type = list(loss_spec[0].keys())[0] + loss_fn_args = loss_spec[0][loss_type] + + loss_class = getattr(tensorflow.keras.losses, loss_type, None) + return loss_class(**loss_fn_args) + + raise RuntimeError(f'Did not recognize the loss function specification: {loss_spec}') + + class ValidationSet:#pylint:disable=too-few-public-methods """ Specifies the images and labels in a validation set. @@ -153,7 +184,7 @@ def __init__(self): 'Number of times to repeat training on the dataset.') self.register_field('batch_size', int, None, '--batch-size', config.validate_positive, 'Features to group into each training batch.') - self.register_field('loss_function', str, None, None, None, 'Keras loss function.') + self.register_field('loss_function', (str, list), None, None, None, 'Keras loss function.') self.register_field('metrics', list, None, None, None, 'List of metrics to apply.') self.register_field('steps', int, None, '--steps', config.validate_positive, 'Batches to train per epoch.') self.register_field('optimizer', str, None, None, None, 'Keras optimizer to use.') @@ -176,9 +207,10 @@ def spec(self) -> TrainingSpec: if not from_training: (vimg, vlabels) = (self._components['validation'].images(), self._components['validation'].labels()) validation = ValidationSet(vimg, vlabels, from_training, vsteps) + loss_fn = loss_function_factory(self._config_dict['loss_function']) self.__training = TrainingSpec(batch_size=self._config_dict['batch_size'], epochs=self._config_dict['epochs'], - loss_function=self._config_dict['loss_function'], + loss_function=loss_fn, metrics=self._config_dict['metrics'], validation=validation, steps=self._config_dict['steps'], From fb853d043967a385ded549b63e087b263f5e744d Mon Sep 17 00:00:00 2001 From: Scott Date: Mon, 29 Jun 2020 18:17:43 -0700 Subject: [PATCH 21/42] Fix some issues with saving output images --- delta/imagery/sources/tiff.py | 6 +++++- delta/ml/predict.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/delta/imagery/sources/tiff.py b/delta/imagery/sources/tiff.py index f8db711e..98f125a8 100644 --- a/delta/imagery/sources/tiff.py +++ b/delta/imagery/sources/tiff.py @@ -267,6 +267,10 @@ def numpy_dtype_to_gdal_type(dtype): return gdal.GDT_UInt16 if dtype == np.uint32: return gdal.GDT_UInt32 + if dtype == np.int16: + return gdal.GDT_Int16 + if dtype == np.int32: + return gdal.GDT_Int32 if dtype == np.float32: return gdal.GDT_Float32 if dtype == np.float64: @@ -413,7 +417,7 @@ def initialize(self, size, numpy_dtype, metadata=None): """ Prepare for writing with the given size and dtype. """ - assert len(size) == 3 + assert (len(size) == 3), ('Error: len(size) of '+str(size)+' != 3') TILE_SIZE = 256 self._tiff_w = TiffWriter(self._filename, size[0], size[1], num_bands=size[2], data_type=numpy_dtype_to_gdal_type(numpy_dtype), metadata=metadata, diff --git a/delta/ml/predict.py b/delta/ml/predict.py index 574adac0..989545bc 100644 --- a/delta/ml/predict.py +++ b/delta/ml/predict.py @@ -179,7 +179,7 @@ def _initialize(self, shape, label, image): self._output_image.initialize((shape[0], shape[1], self._colormap.shape[1]), self._colormap.dtype, image.metadata()) else: - self._output_image.initialize((shape[0], shape[1]), np.int32, image.metadata()) + self._output_image.initialize((shape[0], shape[1], 1), np.int32, image.metadata()) if self._prob_image: self._prob_image.initialize((shape[0], shape[1], self._num_classes), np.float32, image.metadata()) if self._error_image: From 2746db52ddf461875d16d8390ba5dcb7d32c68d7 Mon Sep 17 00:00:00 2001 From: Scott Date: Tue, 30 Jun 2020 10:24:22 -0700 Subject: [PATCH 22/42] Improve color/CPU support in classify command --- delta/subcommands/classify.py | 27 +++++++++++++++++++++++---- delta/subcommands/commands.py | 2 ++ 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/delta/subcommands/classify.py b/delta/subcommands/classify.py index 93a32f33..ec652a07 100644 --- a/delta/subcommands/classify.py +++ b/delta/subcommands/classify.py @@ -58,15 +58,29 @@ def ae_convert(data): return (data[:, :, [4, 2, 1]] * 256.0).astype(np.uint8) def main(options): - model = tf.keras.models.load_model(options.model, custom_objects=delta.ml.layers.ALL_LAYERS) + + # TODO: Share the way this is done with in ml/train.py + cpuOnly = (config.general.gpus()==0) + + if cpuOnly: + with tf.device('/cpu:0'): + model = tf.keras.models.load_model(options.model, custom_objects=delta.ml.layers.ALL_LAYERS) + else: + model = tf.keras.models.load_model(options.model, custom_objects=delta.ml.layers.ALL_LAYERS) colors = np.array([[0x0, 0x0, 0x0], [0x67, 0xa9, 0xcf], [0xf6, 0xef, 0xf7], [0xbd, 0xc9, 0xe1], - [0x02, 0x81, 0x8a]], dtype=np.uint8) + [0x02, 0x81, 0x8a], + [0x00, 0xff, 0xff], # TODO: Label and clean up colormap + [0xff, 0x00, 0xff], + [0xff, 0xff, 0x00]], + dtype=np.uint8) error_colors = np.array([[0x0, 0x0, 0x0], [0xFF, 0x00, 0x00]], dtype=np.uint8) + if options.noColormap: + colors=None # Forces raw one channel output start_time = time.time() images = config.dataset.images() @@ -88,15 +102,20 @@ def main(options): label = None if labels: label = loader.load_image(config.dataset.labels(), i) + if options.autoencoder: label = image predictor = predict.ImagePredictor(model, output_image, True, (ae_convert, np.uint8, 3)) else: predictor = predict.LabelPredictor(model, output_image, True, colormap=colors, prob_image=prob_image, - error_image=error_image, error_colors=error_colors) + error_image=error_image, error_colors=error_colors) try: - predictor.predict(image, label) + if cpuOnly: + with tf.device('/cpu:0'): + predictor.predict(image, label) + else: + predictor.predict(image, label) except KeyboardInterrupt: print('\nAborted.') return 0 diff --git a/delta/subcommands/commands.py b/delta/subcommands/commands.py index 6401e6a7..10042038 100644 --- a/delta/subcommands/commands.py +++ b/delta/subcommands/commands.py @@ -46,6 +46,8 @@ def setup_classify(subparsers): sub.add_argument('--prob', dest='prob', action='store_true', help='Save image of class probabilities.') sub.add_argument('--autoencoder', dest='autoencoder', action='store_true', help='Classify with the autoencoder.') + sub.add_argument('--no-colormap', dest='noColormap', action='store_true', + help='Save raw classification values instead of colormapped values.') sub.add_argument('model', help='File to save the network to.') sub.set_defaults(function=main_classify) From c517a8a790ff90d4de9f4dad6e9a1c79eca715c2 Mon Sep 17 00:00:00 2001 From: Scott Date: Mon, 1 Jun 2020 13:54:55 -0700 Subject: [PATCH 23/42] Prototype resume feature --- delta/imagery/imagery_dataset.py | 51 ++++++++++++++++++++++++++++++-- 1 file changed, 48 insertions(+), 3 deletions(-) diff --git a/delta/imagery/imagery_dataset.py b/delta/imagery/imagery_dataset.py index d04f79a1..8bf3b156 100644 --- a/delta/imagery/imagery_dataset.py +++ b/delta/imagery/imagery_dataset.py @@ -22,6 +22,8 @@ import math import random import sys +import os +import portalocker import tensorflow as tf @@ -35,11 +37,17 @@ class ImageryDataset: """Create dataset with all files as described in the provided config file. """ - def __init__(self, images, labels, chunk_size, output_size, chunk_stride=1): + def __init__(self, images, labels, chunk_size, output_size, chunk_stride=1, ): """ Initialize the dataset based on the specified image and label ImageSets """ + # TODO: Parameters! + self._resume_mode = False + self._log_folder = '/home/smcmich1/data/ds_log' + if not os.path.exists(self._log_folder): + os.mkdir(self._log_folder) + # Record some of the config values assert (chunk_size % 2) == (output_size % 2), 'Chunk size and output size must both be either even or odd.' self._chunk_size = chunk_size @@ -57,9 +65,39 @@ def __init__(self, images, labels, chunk_size, output_size, chunk_stride=1): # Load the first image to get the number of bands for the input files. self._num_bands = loader.load_image(images, 0).num_bands() + def _get_image_read_log_path(self, image_path): + """Return the path to the read log for an input image""" + if not self._log_folder: + return None + image_name = os.path.basename(image_path) + file_name = os.path.splitext(image_name)[0] + '_read.log' + log_path = os.path.join(self._log_folder, file_name) + return log_path + + def _get_image_read_count(self, image_path): + """Return the number of ROIs we have read from an image""" + log_path = self._get_image_read_log_path(image_path) + if not log_path: + return 0 + counter = 0 + with portalocker.Lock(log_path, 'r', timeout=300) as f: + for line in f: #pylint: disable=W0612 + counter += 1 + return counter + def _load_tensor_imagery(self, is_labels, image_index, bbox): """Loads a single image as a tensor.""" - image = loader.load_image(self._labels if is_labels else self._images, image_index.numpy()) + data = self._labels if is_labels else self._images + + if not is_labels: # Record each time we write a tile + file_path = data[image_index.numpy()] + log_path = self._get_image_read_log_path(file_path) + if log_path: + with portalocker.Lock(log_path, 'a', timeout=300) as f: + f.write(str(bbox) + '\n') + # TODO: What to write and when to clear it? + + image = loader.load_image(data, image_index.numpy()) w = int(bbox[2]) h = int(bbox[3]) rect = rectangle.Rectangle(int(bbox[0]), int(bbox[1]), w, h) @@ -71,6 +109,13 @@ def _tile_images(self): def tile_generator(): tgs = [] for i in range(len(self._images)): + + if self._resume_mode: + # Skip images which we have already read some number of tiles from + READ_CUTOFF = 200 # TODO How to set this + if self._get_image_read_count(self._images[i]) > READ_CUTOFF: + continue + img = loader.load_image(self._images, i) if self._labels: # If we have labels make sure they are the same size as the input images label = loader.load_image(self._labels, i) @@ -134,7 +179,7 @@ def _chunk_image(self, image): """Split up a tensor image into tensor chunks""" #tf.print("chunk_image", output_stream=sys.stdout) ksizes = [1, self._chunk_size, self._chunk_size, 1] # Size of the chunks - strides = [1, self._chunk_stride, self._chunk_stride, 1] # SPacing between chunk starts + strides = [1, self._chunk_stride, self._chunk_stride, 1] # Spacing between chunk starts rates = [1, 1, 1, 1] result = tf.image.extract_patches(tf.expand_dims(image, 0), ksizes, strides, rates, padding='VALID') From 3403b43c876269ea5cac7a8d6f37c7cd3dcf0d64 Mon Sep 17 00:00:00 2001 From: Scott Date: Tue, 9 Jun 2020 00:00:40 -0500 Subject: [PATCH 24/42] Complete resume feature --- delta/config/config.py | 2 ++ delta/config/delta.yaml | 4 ++++ delta/imagery/imagery_config.py | 5 +++++ delta/imagery/imagery_dataset.py | 31 +++++++++++++++--------------- delta/ml/ml_config.py | 1 + delta/ml/train.py | 18 ++++++++++++----- delta/subcommands/train.py | 18 +++++++++++++++-- scripts/fetch/fetch_hdds_images.py | 2 +- 8 files changed, 58 insertions(+), 23 deletions(-) diff --git a/delta/config/config.py b/delta/config/config.py index bf47c894..a9b877cf 100644 --- a/delta/config/config.py +++ b/delta/config/config.py @@ -181,6 +181,7 @@ def load(self, yaml_file: str = None, yaml_str: str = None): Loads a config file, then updates the default configuration with the loaded values. """ + print("Loading config file: " + yaml_file) base_path = None if yaml_file: if not os.path.exists(yaml_file): @@ -213,6 +214,7 @@ def initialize(self, options, config_files = None): """ self.reset() + print('initialize with config_files = ' + str(config_files)) if config_files is None: dirs = appdirs.AppDirs('delta', 'nasa') config_files = [os.path.join(dirs.site_config_dir, 'delta.yaml'), diff --git a/delta/config/delta.yaml b/delta/config/delta.yaml index 55d3b29c..86529fad 100644 --- a/delta/config/delta.yaml +++ b/delta/config/delta.yaml @@ -10,12 +10,16 @@ io: interleave_images: 5 # ratio of tile width and height when loading images tile_ratio: 5.0 + # When resuming training with a log_folder, skip input image where we have + # already loaded this many tiles. + resume_cutoff: 5000 cache: # default is OS-specific, in Linux, ~/.cache/delta dir: default limit: 8 dataset: + log_folder: ~ images: type: tiff # preprocess the images when loading (i.e., scaling) diff --git a/delta/imagery/imagery_config.py b/delta/imagery/imagery_config.py index a7f912c1..8b2f8ecf 100644 --- a/delta/imagery/imagery_config.py +++ b/delta/imagery/imagery_config.py @@ -222,6 +222,8 @@ def __init__(self): self.register_component(ImageSetConfig('label'), 'labels', '__label_comp') self.__images = None self.__labels = None + self.register_field('log_folder', str, 'log_folder', None, validate_path, + 'Directory where dataset progress is recorded.') def reset(self): super().reset() @@ -279,6 +281,9 @@ def __init__(self): 'Number of images to interleave at a time when training.') self.register_field('tile_ratio', float, 'tile_ratio', '--tile-ratio', validate_positive, 'Width to height ratio of blocks to load in images.') + self.register_field('resume_cutoff', int, 'resume_cutoff', None, None, + 'When resuming a dataset, skip images where we have read this many tiles.') + self.register_component(CacheConfig(), 'cache') def register(): diff --git a/delta/imagery/imagery_dataset.py b/delta/imagery/imagery_dataset.py index 8bf3b156..31c19b7a 100644 --- a/delta/imagery/imagery_dataset.py +++ b/delta/imagery/imagery_dataset.py @@ -37,25 +37,25 @@ class ImageryDataset: """Create dataset with all files as described in the provided config file. """ - def __init__(self, images, labels, chunk_size, output_size, chunk_stride=1, ): + def __init__(self, images, labels, chunk_size, output_size, chunk_stride=1, + resume_mode=False, log_folder=None): """ Initialize the dataset based on the specified image and label ImageSets """ - # TODO: Parameters! - self._resume_mode = False - self._log_folder = '/home/smcmich1/data/ds_log' - if not os.path.exists(self._log_folder): + self._resume_mode = resume_mode + self._log_folder = log_folder + if self._log_folder and not os.path.exists(self._log_folder): os.mkdir(self._log_folder) # Record some of the config values assert (chunk_size % 2) == (output_size % 2), 'Chunk size and output size must both be either even or odd.' - self._chunk_size = chunk_size - self._output_size = output_size - self._output_dims = 1 + self._chunk_size = chunk_size + self._output_size = output_size + self._output_dims = 1 self._chunk_stride = chunk_stride - self._data_type = tf.float32 - self._label_type = tf.uint8 + self._data_type = tf.float32 + self._label_type = tf.uint8 if labels: assert len(images) == len(labels) @@ -77,7 +77,7 @@ def _get_image_read_log_path(self, image_path): def _get_image_read_count(self, image_path): """Return the number of ROIs we have read from an image""" log_path = self._get_image_read_log_path(image_path) - if not log_path: + if (not log_path) or not os.path.exists(log_path): return 0 counter = 0 with portalocker.Lock(log_path, 'r', timeout=300) as f: @@ -112,8 +112,7 @@ def tile_generator(): if self._resume_mode: # Skip images which we have already read some number of tiles from - READ_CUTOFF = 200 # TODO How to set this - if self._get_image_read_count(self._images[i]) > READ_CUTOFF: + if self._get_image_read_count(self._images[i]) > config.io.resume_cutoff(): continue img = loader.load_image(self._images, i) @@ -137,6 +136,8 @@ def tile_generator(): overlap=self._chunk_size - 1) random.Random(0).shuffle(tiles) # gives consistent random ordering so labels will match tgs.append((i, tiles)) + if not tgs: + return while tgs: cur = tgs[:config.io.interleave_images()] tgs = tgs[config.io.interleave_images():] @@ -279,11 +280,11 @@ def label_set(self): class AutoencoderDataset(ImageryDataset): """Slightly modified dataset class for the Autoencoder which does not use separate label files""" - def __init__(self, images, chunk_size, chunk_stride=1): + def __init__(self, images, chunk_size, chunk_stride=1, resume_mode=False, log_folder=None): """ The images are used as labels as well. """ - super(AutoencoderDataset, self).__init__(images, None, chunk_size, chunk_size, chunk_stride=chunk_stride) + super(AutoencoderDataset, self).__init__(images, None, chunk_size, chunk_size, chunk_stride=chunk_stride, resume_mode=resume_mode, log_folder=log_folder) self._labels = self._images self._output_dims = self.num_bands() diff --git a/delta/ml/ml_config.py b/delta/ml/ml_config.py index 5751d3d7..f7bdf0b4 100644 --- a/delta/ml/ml_config.py +++ b/delta/ml/ml_config.py @@ -121,6 +121,7 @@ def as_dict(self) -> dict: yaml_file = pkg_resources.resource_filename('delta', resource) if not os.path.exists(yaml_file): raise ValueError('Model yaml_file does not exist: ' + yaml_file) + print('Opening model file: ' + yaml_file) with open(yaml_file, 'r') as f: return yaml.safe_load(f) return self._config_dict diff --git a/delta/ml/train.py b/delta/ml/train.py index 29a7c156..9a7ce473 100644 --- a/delta/ml/train.py +++ b/delta/ml/train.py @@ -20,6 +20,7 @@ """ import os +import sys import tempfile import shutil @@ -75,10 +76,13 @@ def _prep_datasets(ids, tc, chunk_size, output_size): validation = None else: if vlabel: - vimagery = ImageryDataset(vimg, vlabel, chunk_size, output_size, tc.chunk_stride) + vimagery = ImageryDataset(vimg, vlabel, chunk_size, output_size, tc.chunk_stride, + resume_mode=False) else: - vimagery = AutoencoderDataset(vimg, chunk_size, tc.chunk_stride) - validation = vimagery.dataset().batch(tc.batch_size).take(tc.validation.steps) + vimagery = AutoencoderDataset(vimg, chunk_size, tc.chunk_stride, resume_mode=False) + validation = vimagery.dataset().batch(tc.batch_size) + if tc.validation.steps: + validation = validation.take(tc.validation.steps) #validation = validation.prefetch(4)#tf.data.experimental.AUTOTUNE) else: @@ -133,6 +137,8 @@ def on_train_batch_end(self, batch, logs=None): old = filename filename = os.path.join(self.temp_dir, 'latest.h5') os.rename(old, filename) + tf.print('Recording checkpoint: ' + filename, + output_stream=sys.stdout) mlflow.log_artifact(filename, 'checkpoints') os.remove(filename) @@ -208,15 +214,17 @@ def train(model_fn, dataset : ImageryDataset, training_spec): if config.mlflow.enabled(): mcb = _mlflow_train_setup(model, dataset, training_spec) callbacks.append(mcb) + print('Using mlflow folder: ' + mlflow.get_artifact_uri()) try: - print(training_spec.validation) history = model.fit(ds, epochs=training_spec.epochs, callbacks=callbacks, validation_data=validation, validation_steps=training_spec.validation.steps if training_spec.validation else None, - steps_per_epoch=training_spec.steps)#, verbose=2) + steps_per_epoch=training_spec.steps, + verbose=1) + if config.mlflow.enabled(): model_path = os.path.join(mcb.temp_dir, 'final_model.h5') print('\nFinished, saving model to %s.' % (mlflow.get_artifact_uri() + '/final_model.h5')) diff --git a/delta/subcommands/train.py b/delta/subcommands/train.py index 50e321b4..e94a7491 100644 --- a/delta/subcommands/train.py +++ b/delta/subcommands/train.py @@ -21,6 +21,7 @@ import sys import time +import os #import logging #logging.getLogger("tensorflow").setLevel(logging.DEBUG) @@ -36,6 +37,15 @@ #tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.DEBUG) def main(options): + + log_folder = config.dataset.log_folder() + if log_folder: + if not options.resume: # Start fresh and clear the read logs + os.system('rm ' + log_folder + '/*') + print('Dataset progress recording in: ' + log_folder) + else: + print('Resuming dataset progress recorded in: ' + log_folder) + start_time = time.time() images = config.dataset.images() if not images: @@ -43,14 +53,18 @@ def main(options): return 1 tc = config.train.spec() if options.autoencoder: - ids = imagery_dataset.AutoencoderDataset(images, config.train.network.chunk_size(), tc.chunk_stride) + ids = imagery_dataset.AutoencoderDataset(images, config.train.network.chunk_size(), + tc.chunk_stride, resume_mode=options.resume, + log_folder=log_folder) else: labels = config.dataset.labels() if not labels: print('No labels specified.', file=sys.stderr) return 1 ids = imagery_dataset.ImageryDataset(images, labels, config.train.network.chunk_size(), - config.train.network.output_size(), tc.chunk_stride) + config.train.network.output_size(), tc.chunk_stride, + resume_mode=options.resume, + log_folder=log_folder) try: if options.resume is not None: diff --git a/scripts/fetch/fetch_hdds_images.py b/scripts/fetch/fetch_hdds_images.py index a54a90fb..112e813d 100755 --- a/scripts/fetch/fetch_hdds_images.py +++ b/scripts/fetch/fetch_hdds_images.py @@ -368,7 +368,7 @@ def main(argsIn): #pylint: disable=R0914,R0912 #raise Exception('DEBUG') print('Finished processing dataset: ' + full_name) -# os.system('touch ' + done_flag_path) # Mark this dataset as finished + os.system('touch ' + done_flag_path) # Mark this dataset as finished #raise Exception('DEBUG') #if not os.path.exists(output_path): From bc13df6a0e4b1029f25e7a4863be7787aff8f338 Mon Sep 17 00:00:00 2001 From: Scott Date: Tue, 7 Jul 2020 22:02:44 -0700 Subject: [PATCH 25/42] Cleanup, test fixes --- delta/config/config.py | 3 +-- delta/config/delta.yaml | 2 +- delta/imagery/imagery_dataset.py | 28 +++------------------------- delta/ml/train.py | 4 ++-- setup.py | 2 +- 5 files changed, 8 insertions(+), 31 deletions(-) diff --git a/delta/config/config.py b/delta/config/config.py index a9b877cf..35b2d303 100644 --- a/delta/config/config.py +++ b/delta/config/config.py @@ -181,9 +181,9 @@ def load(self, yaml_file: str = None, yaml_str: str = None): Loads a config file, then updates the default configuration with the loaded values. """ - print("Loading config file: " + yaml_file) base_path = None if yaml_file: + print("Loading config file: " + yaml_file) if not os.path.exists(yaml_file): raise Exception('Config file does not exist: ' + yaml_file) with open(yaml_file, 'r') as f: @@ -214,7 +214,6 @@ def initialize(self, options, config_files = None): """ self.reset() - print('initialize with config_files = ' + str(config_files)) if config_files is None: dirs = appdirs.AppDirs('delta', 'nasa') config_files = [os.path.join(dirs.site_config_dir, 'delta.yaml'), diff --git a/delta/config/delta.yaml b/delta/config/delta.yaml index 86529fad..fdb2c424 100644 --- a/delta/config/delta.yaml +++ b/delta/config/delta.yaml @@ -19,7 +19,7 @@ io: limit: 8 dataset: - log_folder: ~ + log_folder: ~ # Storage location for any record keeping files about the input dataset images: type: tiff # preprocess the images when loading (i.e., scaling) diff --git a/delta/imagery/imagery_dataset.py b/delta/imagery/imagery_dataset.py index 31c19b7a..1b9c1e6f 100644 --- a/delta/imagery/imagery_dataset.py +++ b/delta/imagery/imagery_dataset.py @@ -163,7 +163,6 @@ def _load_images(self, is_labels, data_type): """ ds_input = self._tile_images() def load_tile(image_index, x1, y1, x2, y2): - #tf.print("load_tile", output_stream=sys.stdout) img = tf.py_function(functools.partial(self._load_tensor_imagery, is_labels), [image_index, [x1, y1, x2, y2]], data_type) @@ -174,11 +173,10 @@ def load_tile(image_index, x1, y1, x2, y2): # - Would be better to handle this somehow but it is not clear if TF supports that. ret = ret.apply(tf.data.experimental.ignore_errors()) - return ret#.prefetch(4)#tf.data.experimental.AUTOTUNE) + return ret def _chunk_image(self, image): """Split up a tensor image into tensor chunks""" - #tf.print("chunk_image", output_stream=sys.stdout) ksizes = [1, self._chunk_size, self._chunk_size, 1] # Size of the chunks strides = [1, self._chunk_stride, self._chunk_stride, 1] # Spacing between chunk starts rates = [1, 1, 1, 1] @@ -207,8 +205,7 @@ def data(self): Unbatched dataset of image chunks. """ ret = self._load_images(False, self._data_type) - ret = ret.map(self._chunk_image, num_parallel_calls=tf.data.experimental.AUTOTUNE)#config.io.threads()) - #ret = ret.prefetch(4)#tf.data.experimental.AUTOTUNE) + ret = ret.map(self._chunk_image, num_parallel_calls=tf.data.experimental.AUTOTUNE) return ret.unbatch() def labels(self): @@ -216,8 +213,7 @@ def labels(self): Unbatched dataset of labels. """ label_set = self._load_images(True, self._label_type) - label_set = label_set.map(self._reshape_labels, num_parallel_calls=tf.data.experimental.AUTOTUNE)#config.threads()) #pylint: disable=C0301 - #label_set = label_set.prefetch(4)#tf.data.experimental.AUTOTUNE) + label_set = label_set.map(self._reshape_labels, num_parallel_calls=tf.data.experimental.AUTOTUNE) #pylint: disable=C0301 return label_set.unbatch() def dataset(self): @@ -225,29 +221,11 @@ def dataset(self): Return the underlying TensorFlow dataset object that this class creates. """ - #def double_chunk(image, label): - # #di = tf.data.Dataset.from_tensors(image) - # #dl = tf.data.Dataset.from_tensors(label) - # #di = di.map(self._chunk_image, num_parallel_calls=tf.data.experimental.AUTOTUNE) - # #dl = dl.map(self._reshape_labels, num_parallel_calls=tf.data.experimental.AUTOTUNE) - # #return tf.data.Dataset.zip((di.unbatch(), dl.unbatch())) - # ret = tf.data.Dataset.from_tensors(self._chunk_image(image)) - # label_set = tf.data.Dataset.from_tensors(tf.cast(self._reshape_labels(label), tf.float32)) - # return tf.data.Dataset.zip((ret.unbatch(), label_set.unbatch())) - - #ret_data = self._load_images(False, self._data_type) - #label_set = self._load_images(True, self._label_type) - #ds = tf.data.Dataset.zip((ret_data, label_set)) - #ds = ds.interleave(double_chunk, num_parallel_calls=tf.data.experimental.AUTOTUNE) - #ds = ds.unbatch() - # Pair the data and labels in our dataset ds = tf.data.Dataset.zip((self.data(), self.labels())) # ignore labels with no data if self._labels.nodata_value(): ds = ds.filter(lambda x, y: tf.math.not_equal(y, self._labels.nodata_value())) - #ds = ds.prefetch(tf.data.experimental.AUTOTUNE) - print(ds) return ds def num_bands(self): diff --git a/delta/ml/train.py b/delta/ml/train.py index 9a7ce473..2f5a8793 100644 --- a/delta/ml/train.py +++ b/delta/ml/train.py @@ -42,9 +42,9 @@ def _devices(num_gpus): ''' devs = None if num_gpus == 0: - devs = [x.name for x in tf.config.experimental.list_logical_devices('CPU')] + devs = [x.name for x in tf.config.list_logical_devices('CPU')] else: - devs = [x.name for x in tf.config.experimental.list_logical_devices('GPU')] + devs = [x.name for x in tf.config.list_logical_devices('GPU')] assert len(devs) >= num_gpus,\ "Requested %d GPUs with only %d available." % (num_gpus, len(devs)) if num_gpus > 0: diff --git a/setup.py b/setup.py index 22581930..ced33d45 100644 --- a/setup.py +++ b/setup.py @@ -50,7 +50,7 @@ 'numpy', 'scipy', 'matplotlib', - 'tensorflow-gpu==2.1', + 'tensorflow==2.1', 'mlflow', 'portalocker', 'appdirs', From 3a93428e1d81e30406147989dee8c5e13ed88f2d Mon Sep 17 00:00:00 2001 From: Scott Date: Wed, 8 Jul 2020 11:31:13 -0700 Subject: [PATCH 26/42] Disable log output --- delta/config/config.py | 2 +- delta/imagery/imagery_dataset.py | 4 +++- delta/ml/ml_config.py | 2 +- delta/ml/train.py | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/delta/config/config.py b/delta/config/config.py index 35b2d303..3eda4866 100644 --- a/delta/config/config.py +++ b/delta/config/config.py @@ -183,7 +183,7 @@ def load(self, yaml_file: str = None, yaml_str: str = None): """ base_path = None if yaml_file: - print("Loading config file: " + yaml_file) + #print("Loading config file: " + yaml_file) if not os.path.exists(yaml_file): raise Exception('Config file does not exist: ' + yaml_file) with open(yaml_file, 'r') as f: diff --git a/delta/imagery/imagery_dataset.py b/delta/imagery/imagery_dataset.py index 1b9c1e6f..2330b7f2 100644 --- a/delta/imagery/imagery_dataset.py +++ b/delta/imagery/imagery_dataset.py @@ -111,6 +111,7 @@ def tile_generator(): for i in range(len(self._images)): if self._resume_mode: + # TODO: Improve feature to work with multiple epochs # Skip images which we have already read some number of tiles from if self._get_image_read_count(self._images[i]) > config.io.resume_cutoff(): continue @@ -262,7 +263,8 @@ def __init__(self, images, chunk_size, chunk_stride=1, resume_mode=False, log_fo """ The images are used as labels as well. """ - super(AutoencoderDataset, self).__init__(images, None, chunk_size, chunk_size, chunk_stride=chunk_stride, resume_mode=resume_mode, log_folder=log_folder) + super(AutoencoderDataset, self).__init__(images, None, chunk_size, chunk_size, chunk_stride=chunk_stride, + resume_mode=resume_mode, log_folder=log_folder) self._labels = self._images self._output_dims = self.num_bands() diff --git a/delta/ml/ml_config.py b/delta/ml/ml_config.py index f7bdf0b4..a5c0b51b 100644 --- a/delta/ml/ml_config.py +++ b/delta/ml/ml_config.py @@ -121,7 +121,7 @@ def as_dict(self) -> dict: yaml_file = pkg_resources.resource_filename('delta', resource) if not os.path.exists(yaml_file): raise ValueError('Model yaml_file does not exist: ' + yaml_file) - print('Opening model file: ' + yaml_file) + #print('Opening model file: ' + yaml_file) with open(yaml_file, 'r') as f: return yaml.safe_load(f) return self._config_dict diff --git a/delta/ml/train.py b/delta/ml/train.py index 2f5a8793..8fb111c8 100644 --- a/delta/ml/train.py +++ b/delta/ml/train.py @@ -214,7 +214,7 @@ def train(model_fn, dataset : ImageryDataset, training_spec): if config.mlflow.enabled(): mcb = _mlflow_train_setup(model, dataset, training_spec) callbacks.append(mcb) - print('Using mlflow folder: ' + mlflow.get_artifact_uri()) + #print('Using mlflow folder: ' + mlflow.get_artifact_uri()) try: history = model.fit(ds, From f89461b8b73c746bbd741b47fa26cabc80ae719a Mon Sep 17 00:00:00 2001 From: Scott Date: Wed, 8 Jul 2020 14:09:55 -0700 Subject: [PATCH 27/42] Bump up TF version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ced33d45..1a9a62ca 100644 --- a/setup.py +++ b/setup.py @@ -50,7 +50,7 @@ 'numpy', 'scipy', 'matplotlib', - 'tensorflow==2.1', + 'tensorflow>=2.1', 'mlflow', 'portalocker', 'appdirs', From 2b0834d8063bd66f391eeb203fc83b796316eca0 Mon Sep 17 00:00:00 2001 From: Scott Date: Wed, 8 Jul 2020 14:22:45 -0700 Subject: [PATCH 28/42] Linter fixes --- delta/imagery/sources/tiff.py | 2 +- delta/imagery/sources/worldview.py | 2 +- delta/subcommands/classify.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/delta/imagery/sources/tiff.py b/delta/imagery/sources/tiff.py index 98f125a8..3b868370 100644 --- a/delta/imagery/sources/tiff.py +++ b/delta/imagery/sources/tiff.py @@ -260,7 +260,7 @@ def _prep(self, paths): os.system(cmd) return [output_path] -def numpy_dtype_to_gdal_type(dtype): +def numpy_dtype_to_gdal_type(dtype): #pylint: disable=R0911 if dtype == np.uint8: return gdal.GDT_Byte if dtype == np.uint16: diff --git a/delta/imagery/sources/worldview.py b/delta/imagery/sources/worldview.py index 2d9c9242..7db15733 100644 --- a/delta/imagery/sources/worldview.py +++ b/delta/imagery/sources/worldview.py @@ -70,7 +70,7 @@ def _unpack(self, paths): # Get the folder where this will be stored from the cache manager unpack_folder = config.io.cache.manager().register_item(self._name) - with portalocker.Lock(paths, 'r', timeout=300) as unused: + with portalocker.Lock(paths, 'r', timeout=300) as unused: #pylint: disable=W0612 # Check if we already unpacked this data (tif_path, imd_path) = _get_files_from_unpack_folder(unpack_folder) diff --git a/delta/subcommands/classify.py b/delta/subcommands/classify.py index ec652a07..661194e6 100644 --- a/delta/subcommands/classify.py +++ b/delta/subcommands/classify.py @@ -76,7 +76,7 @@ def main(options): [0x00, 0xff, 0xff], # TODO: Label and clean up colormap [0xff, 0x00, 0xff], [0xff, 0xff, 0x00]], - dtype=np.uint8) + dtype=np.uint8) error_colors = np.array([[0x0, 0x0, 0x0], [0xFF, 0x00, 0x00]], dtype=np.uint8) if options.noColormap: @@ -108,7 +108,7 @@ def main(options): predictor = predict.ImagePredictor(model, output_image, True, (ae_convert, np.uint8, 3)) else: predictor = predict.LabelPredictor(model, output_image, True, colormap=colors, prob_image=prob_image, - error_image=error_image, error_colors=error_colors) + error_image=error_image, error_colors=error_colors) try: if cpuOnly: From fffb8e054c24eb52ea4b567d568bc85cc8d76aa3 Mon Sep 17 00:00:00 2001 From: Scott Date: Thu, 9 Jul 2020 13:55:30 -0700 Subject: [PATCH 29/42] Fix test --- tests/test_imagery_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_imagery_dataset.py b/tests/test_imagery_dataset.py index b194414c..e788cd88 100644 --- a/tests/test_imagery_dataset.py +++ b/tests/test_imagery_dataset.py @@ -134,7 +134,7 @@ def model_fn(): output_image = npy.NumpyImageWriter() predictor = predict.LabelPredictor(model, output_image=output_image) predictor.predict(npy.NumpyImage(test_image)) - assert sum(sum(np.logical_xor(output_image.buffer(), test_label))) < 200 # very easy test since we don't train much + assert sum(sum(np.logical_xor(output_image.buffer()[:,:,0], test_label))) < 200 # very easy test since we don't train much @pytest.fixture(scope="function") def autoencoder(all_sources): From 35c7d8c4aab50682195915548eaecfd0ff9da69f Mon Sep 17 00:00:00 2001 From: Scott Date: Thu, 9 Jul 2020 16:16:50 -0700 Subject: [PATCH 30/42] Lint fix --- tests/test_imagery_dataset.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_imagery_dataset.py b/tests/test_imagery_dataset.py index e788cd88..d7bfb135 100644 --- a/tests/test_imagery_dataset.py +++ b/tests/test_imagery_dataset.py @@ -134,7 +134,8 @@ def model_fn(): output_image = npy.NumpyImageWriter() predictor = predict.LabelPredictor(model, output_image=output_image) predictor.predict(npy.NumpyImage(test_image)) - assert sum(sum(np.logical_xor(output_image.buffer()[:,:,0], test_label))) < 200 # very easy test since we don't train much + # very easy test since we don't train much + assert sum(sum(np.logical_xor(output_image.buffer()[:,:,0], test_label))) < 200 @pytest.fixture(scope="function") def autoencoder(all_sources): From 90c5bc6dbe35267eea7752938af8e30ef8741842 Mon Sep 17 00:00:00 2001 From: Brian Coltin Date: Wed, 15 Jul 2020 16:02:00 -0700 Subject: [PATCH 31/42] Separate Arguments in Configs (#4) * Remove top-level tensorflow import, add tests for this. * Separate register_arg method for config, more flexible. --- bin/delta | 26 +------------ delta/config/config.py | 47 ++++++++++++++++------- delta/config/modules.py | 33 ++++++++++++++++ delta/imagery/imagery_config.py | 45 ++++++++++++---------- delta/ml/ml_config.py | 67 ++++++++++++++++++++------------- delta/ml/train.py | 2 +- delta/subcommands/commands.py | 5 --- delta/subcommands/main.py | 44 ++++++++++++++++++++++ tests/conftest.py | 9 ++--- tests/test_config.py | 3 -- 10 files changed, 183 insertions(+), 98 deletions(-) create mode 100644 delta/config/modules.py create mode 100644 delta/subcommands/main.py diff --git a/bin/delta b/bin/delta index 8cfb07ed..16084d19 100755 --- a/bin/delta +++ b/bin/delta @@ -18,30 +18,8 @@ # limitations under the License. import sys -import argparse -from delta.config import config -from delta.subcommands import commands - -def main(args): - parser = argparse.ArgumentParser(description='DELTA Machine Learning Toolkit') - subparsers = parser.add_subparsers() - - for d in commands.SETUP_COMMANDS: - d(subparsers) - - try: - options = parser.parse_args(args[1:]) - except argparse.ArgumentError: - parser.print_help(sys.stderr) - sys.exit(1) - - if not hasattr(options, 'function'): - parser.print_help(sys.stderr) - sys.exit(1) - - config.initialize(options) - return options.function(options) +from delta.subcommands import main if __name__ == "__main__": - sys.exit(main(sys.argv)) + sys.exit(main.main(sys.argv)) diff --git a/delta/config/config.py b/delta/config/config.py index 3eda4866..aba3b0ae 100644 --- a/delta/config/config.py +++ b/delta/config/config.py @@ -35,6 +35,9 @@ def validate_positive(num, _): raise ValueError('%d is not positive' % (num)) return num +class _NotSpecified: #pylint:disable=too-few-public-methods + pass + class DeltaConfigComponent: """ DELTA configuration component. @@ -78,7 +81,7 @@ def register_component(self, component, name : str, attr_name = None): attr_name = name setattr(self, attr_name, component) - def register_field(self, name : str, types, accessor = None, cmd_arg = None, validate_fn = None, desc = None): + def register_field(self, name : str, types, accessor = None, validate_fn = None, desc = None): """ Register a field in this component of the configuration. @@ -92,7 +95,6 @@ def register_field(self, name : str, types, accessor = None, cmd_arg = None, val self._fields.append(name) self._validate[name] = validate_fn self._types[name] = types - self._cmd_args[name] = cmd_arg self._descs[name] = desc if accessor: def access(self) -> types: @@ -101,6 +103,28 @@ def access(self) -> types: access.__doc__ = desc setattr(self.__class__, accessor, access) + def register_arg(self, field, argname, **kwargs): + """ + Registers a command line argument in this component. + + field is the (registered) field this argument modifies. + argname is the name of the flag on the command line (i.e., '--flag') + **kwargs are arguments to ArgumentParser.add_argument. + + If help and type are not specified, will use the ones for the field. + If default is not specified, will use the value from the config files. + """ + assert field in self._fields, 'Field %s not registered.' % (field) + if 'help' not in kwargs: + kwargs['help'] = self._descs[field] + if 'type' not in kwargs: + kwargs['type'] = self._types[field] + elif kwargs['type'] is None: + del kwargs['type'] + if 'default' not in kwargs: + kwargs['default'] = _NotSpecified + self._cmd_args[argname] = (field, kwargs) + def export(self) -> str: """ Returns a YAML string of all configuration options. @@ -139,12 +163,9 @@ def setup_arg_parser(self, parser, components = None) -> None: """ if self._section_header is not None: parser = parser.add_argument_group(self._section_header) - for name in self._fields: - c = self._cmd_args[name] - if c is None: - continue - parser.add_argument(c, dest=c.replace('-', '_'), required=False, - type=self._types[name], help=self._descs[name]) + for (arg, value) in self._cmd_args.items(): + (field, kwargs) = value + parser.add_argument(arg, dest=field, **kwargs) for (name, c) in self._components.items(): if components is None or name in components: @@ -157,14 +178,12 @@ def parse_args(self, options): configuration values. """ d = {} - for name in self._fields: - c = self._cmd_args[name] - if c is None: + for (field, _) in self._cmd_args.values(): + if not hasattr(options, field) or getattr(options, field) is None: continue - n = c.replace('-', '_') - if not hasattr(options, n) or getattr(options, n) is None: + if getattr(options, field) is _NotSpecified: continue - d[name] = getattr(options, n) + d[field] = getattr(options, field) self._load_dict(d, None) for c in self._components.values(): diff --git a/delta/config/modules.py b/delta/config/modules.py new file mode 100644 index 00000000..2b53419d --- /dev/null +++ b/delta/config/modules.py @@ -0,0 +1,33 @@ +# Copyright © 2020, United States Government, as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All rights reserved. +# +# The DELTA (Deep Earth Learning, Tools, and Analysis) platform is +# licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Registers all config modules. +""" + +import delta.imagery.imagery_config +import delta.ml.ml_config + +_config_initialized = False +def register_all(): + global _config_initialized #pylint: disable=global-statement + # needed to call twice when testing subcommands and when not + if _config_initialized: + return + delta.imagery.imagery_config.register() + delta.ml.ml_config.register() + _config_initialized = True diff --git a/delta/imagery/imagery_config.py b/delta/imagery/imagery_config.py index 8b2f8ecf..b641009d 100644 --- a/delta/imagery/imagery_config.py +++ b/delta/imagery/imagery_config.py @@ -177,8 +177,8 @@ def load_images_labels(images_comp, labels_comp): class ImagePreprocessConfig(DeltaConfigComponent): def __init__(self): super().__init__() - self.register_field('enabled', bool, 'enabled', None, None, 'Turn on preprocessing.') - self.register_field('scale_factor', (float, str), 'scale_factor', None, None, 'Image scale factor.') + self.register_field('enabled', bool, 'enabled', None, 'Turn on preprocessing.') + self.register_field('scale_factor', (float, str), 'scale_factor', None, 'Image scale factor.') def _validate_paths(paths, base_dir): out = [] @@ -189,15 +189,18 @@ def _validate_paths(paths, base_dir): class ImageSetConfig(DeltaConfigComponent): def __init__(self, name=None): super().__init__() - self.register_field('type', str, 'type', '--' + name + '-type' if name else None, None, 'Image type.') - self.register_field('files', list, None, None, _validate_paths, 'List of image files.') - self.register_field('file_list', list, None, '--' + name + '-file-list' if name else None, - validate_path, 'File listing image files.') - self.register_field('directory', str, None, '--' + name + '-dir' if name else None, - validate_path, 'Directory of image files.') - self.register_field('extension', str, None, '--' + name + '-extension' if name else None, - None, 'Image file extension.') - self.register_field('nodata_value', float, None, None, None, 'Value of pixels to ignore.') + self.register_field('type', str, 'type', None, 'Image type.') + self.register_field('files', list, None, _validate_paths, 'List of image files.') + self.register_field('file_list', list, None, validate_path, 'File listing image files.') + self.register_field('directory', str, None, validate_path, 'Directory of image files.') + self.register_field('extension', str, None, None, 'Image file extension.') + self.register_field('nodata_value', float, None, None, 'Value of pixels to ignore.') + + if name: + self.register_arg('type', '--' + name + '-type') + self.register_arg('file_list', '--' + name + '-file-list') + self.register_arg('directory', '--' + name + '-dir') + self.register_arg('extension', '--' + name + '-extension') self.register_component(ImagePreprocessConfig(), 'preprocess') self._name = name @@ -222,7 +225,7 @@ def __init__(self): self.register_component(ImageSetConfig('label'), 'labels', '__label_comp') self.__images = None self.__labels = None - self.register_field('log_folder', str, 'log_folder', None, validate_path, + self.register_field('log_folder', str, 'log_folder', validate_path, 'Directory where dataset progress is recorded.') def reset(self): @@ -251,8 +254,8 @@ def labels(self) -> ImageSet: class CacheConfig(DeltaConfigComponent): def __init__(self): super().__init__() - self.register_field('dir', str, None, None, validate_path, 'Cache directory.') - self.register_field('limit', int, None, None, validate_positive, 'Number of items to cache.') + self.register_field('dir', str, None, validate_path, 'Cache directory.') + self.register_field('limit', int, None, validate_positive, 'Number of items to cache.') self._cache_manager = None @@ -274,16 +277,20 @@ def manager(self) -> disk_folder_cache.DiskCache: class IOConfig(DeltaConfigComponent): def __init__(self): super().__init__() - self.register_field('threads', int, 'threads', '--threads', None, 'Number of threads to use.') - self.register_field('block_size_mb', int, 'block_size_mb', '--block-size-mb', validate_positive, + self.register_field('threads', int, 'threads', None, 'Number of threads to use.') + self.register_field('block_size_mb', int, 'block_size_mb', validate_positive, 'Size of an image block to load in memory at once.') - self.register_field('interleave_images', int, 'interleave_images', None, validate_positive, + self.register_field('interleave_images', int, 'interleave_images', validate_positive, 'Number of images to interleave at a time when training.') - self.register_field('tile_ratio', float, 'tile_ratio', '--tile-ratio', validate_positive, + self.register_field('tile_ratio', float, 'tile_ratio', validate_positive, 'Width to height ratio of blocks to load in images.') - self.register_field('resume_cutoff', int, 'resume_cutoff', None, None, + self.register_field('resume_cutoff', int, 'resume_cutoff', None, 'When resuming a dataset, skip images where we have read this many tiles.') + self.register_arg('threads', '--threads') + self.register_arg('block_size_mb', '--block-size-mb') + self.register_arg('tile_ratio', '--tile-ratio') + self.register_component(CacheConfig(), 'cache') def register(): diff --git a/delta/ml/ml_config.py b/delta/ml/ml_config.py index a5c0b51b..4d650f5b 100644 --- a/delta/ml/ml_config.py +++ b/delta/ml/ml_config.py @@ -18,18 +18,17 @@ """ Configuration options specific to machine learning. """ +# Please do not put any tensorflow imports in this file as it will greatly slow loading +# when tensorflow isn't needed import os.path import appdirs import pkg_resources import yaml -import tensorflow.keras.losses - from delta.imagery.imagery_config import ImageSet, ImageSetConfig, load_images_labels import delta.config as config - def loss_function_factory(loss_spec): ''' loss_function_factory - Creates a loss function object, if an object is specified in the @@ -39,6 +38,7 @@ def loss_function_factory(loss_spec): with the keras interface (e.g. 'categorical_crossentropy') or an object defined by a dict of the form {'LossFunctionName': {'arg1':arg1_val, ...,'argN',argN_val}} ''' + import tensorflow.keras.losses # pylint: disable=import-outside-toplevel if isinstance(loss_spec, str): return loss_spec @@ -94,10 +94,10 @@ def __init__(self, batch_size, epochs, loss_function, metrics, validation=None, class NetworkModelConfig(config.DeltaConfigComponent): def __init__(self): super().__init__() - self.register_field('yaml_file', str, 'yaml_file', None, config.validate_path, + self.register_field('yaml_file', str, 'yaml_file', config.validate_path, 'A YAML file describing the network to train.') - self.register_field('params', dict, None, None, None, None) - self.register_field('layers', list, None, None, None, None) + self.register_field('params', dict, None, None, None) + self.register_field('layers', list, None, None, None) # overwrite model entirely if updated (don't want combined layers from multiple files) def _load_dict(self, d : dict, base_dir): @@ -129,12 +129,16 @@ def as_dict(self) -> dict: class NetworkConfig(config.DeltaConfigComponent): def __init__(self): super().__init__() - self.register_field('chunk_size', int, 'chunk_size', '--chunk-size', config.validate_positive, + self.register_field('chunk_size', int, 'chunk_size', config.validate_positive, 'Width of an image chunk to input to the neural network.') - self.register_field('output_size', int, 'output_size', '--output-size', config.validate_positive, + self.register_field('output_size', int, 'output_size', config.validate_positive, 'Width of an image chunk to output from the neural network.') - self.register_field('classes', int, 'classes', '--classes', config.validate_positive, + self.register_field('classes', int, 'classes', config.validate_positive, 'Number of label classes.') + + self.register_arg('chunk_size', '--chunk-size') + self.register_arg('output_size', '--output-size') + self.register_arg('classes', '--classes') self.register_component(NetworkModelConfig(), 'model') def setup_arg_parser(self, parser, components = None) -> None: @@ -144,9 +148,9 @@ def setup_arg_parser(self, parser, components = None) -> None: class ValidationConfig(config.DeltaConfigComponent): def __init__(self): super().__init__() - self.register_field('steps', int, 'steps', None, config.validate_positive, + self.register_field('steps', int, 'steps', config.validate_positive, 'If from training, validate for this many steps.') - self.register_field('from_training', bool, 'from_training', None, None, + self.register_field('from_training', bool, 'from_training', None, 'Take validation data from training data.') self.register_component(ImageSetConfig(), 'images', '__image_comp') self.register_component(ImageSetConfig(), 'labels', '__label_comp') @@ -179,16 +183,21 @@ def labels(self) -> ImageSet: class TrainingConfig(config.DeltaConfigComponent): def __init__(self): super().__init__() - self.register_field('chunk_stride', int, None, '--chunk-stride', config.validate_positive, + self.register_field('chunk_stride', int, None, config.validate_positive, 'Pixels to skip when iterating over chunks. A value of 1 means to take every chunk.') - self.register_field('epochs', int, None, '--epochs', config.validate_positive, + self.register_field('epochs', int, None, config.validate_positive, 'Number of times to repeat training on the dataset.') - self.register_field('batch_size', int, None, '--batch-size', config.validate_positive, + self.register_field('batch_size', int, None, config.validate_positive, 'Features to group into each training batch.') - self.register_field('loss_function', (str, list), None, None, None, 'Keras loss function.') - self.register_field('metrics', list, None, None, None, 'List of metrics to apply.') - self.register_field('steps', int, None, '--steps', config.validate_positive, 'Batches to train per epoch.') - self.register_field('optimizer', str, None, None, None, 'Keras optimizer to use.') + self.register_field('loss_function', (str, list), None, None, 'Keras loss function.') + self.register_field('metrics', list, None, None, 'List of metrics to apply.') + self.register_field('steps', int, None, config.validate_positive, 'Batches to train per epoch.') + self.register_field('optimizer', str, None, None, 'Keras optimizer to use.') + + self.register_arg('chunk_stride', '--chunk-stride') + self.register_arg('epochs', '--epochs') + self.register_arg('batch_size', '--batch-size') + self.register_arg('steps', '--steps') self.register_component(ValidationConfig(), 'validation') self.register_component(NetworkConfig(), 'network') self.__training = None @@ -223,19 +232,22 @@ def spec(self) -> TrainingSpec: class MLFlowCheckpointsConfig(config.DeltaConfigComponent): def __init__(self): super().__init__() - self.register_field('frequency', int, 'frequency', None, None, + self.register_field('frequency', int, 'frequency', None, 'Frequency in batches to store neural network checkpoints.') - self.register_field('save_latest', bool, 'save_latest', None, None, + self.register_field('save_latest', bool, 'save_latest', None, 'If true, only keep the most recent checkpoint.') class MLFlowConfig(config.DeltaConfigComponent): def __init__(self): super().__init__() - self.register_field('enabled', bool, 'enabled', None, None, 'Enable MLFlow.') - self.register_field('uri', str, None, None, None, 'URI to store MLFlow data.') - self.register_field('frequency', int, 'frequency', None, config.validate_positive, + self.register_field('enabled', bool, 'enabled', None, 'Enable MLFlow.') + self.register_field('uri', str, None, None, 'URI to store MLFlow data.') + self.register_field('frequency', int, 'frequency', config.validate_positive, 'Frequency to store metrics.') - self.register_field('experiment_name', str, 'experiment', None, None, 'Experiment name in MLFlow.') + self.register_field('experiment_name', str, 'experiment', None, 'Experiment name in MLFlow.') + + self.register_arg('enabled', '--disable-mlflow', action='store_const', const=False, type=None) + self.register_arg('enabled', '--enable-mlflow', action='store_const', const=True, type=None) self.register_component(MLFlowCheckpointsConfig(), 'checkpoints') def uri(self) -> str: @@ -250,8 +262,8 @@ def uri(self) -> str: class TensorboardConfig(config.DeltaConfigComponent): def __init__(self): super().__init__() - self.register_field('enabled', bool, 'enabled', None, None, 'Enable Tensorboard.') - self.register_field('dir', str, None, None, None, 'Directory to store Tensorboard data.') + self.register_field('enabled', bool, 'enabled', None, 'Enable Tensorboard.') + self.register_field('dir', str, None, None, 'Directory to store Tensorboard data.') def dir(self) -> str: """ @@ -270,7 +282,8 @@ def register(): """ if not hasattr(config.config, 'general'): config.config.register_component(config.DeltaConfigComponent('General'), 'general') - config.config.general.register_field('gpus', int, 'gpus', '--gpus', None, 'Number of gpus to use.') + config.config.general.register_field('gpus', int, 'gpus', None, 'Number of gpus to use.') + config.config.general.register_arg('gpus', '--gpus') config.config.register_component(TrainingConfig(), 'train') config.config.register_component(MLFlowConfig(), 'mlflow') diff --git a/delta/ml/train.py b/delta/ml/train.py index 8fb111c8..3a31e7b2 100644 --- a/delta/ml/train.py +++ b/delta/ml/train.py @@ -129,7 +129,7 @@ def on_train_batch_end(self, batch, logs=None): for k in logs.keys(): if k in ('batch', 'size'): continue - mlflow.log_metric(k, logs[k].item(), step=batch) + mlflow.log_metric(k, logs[k], step=batch) if config.mlflow.checkpoints.frequency() and batch % config.mlflow.checkpoints.frequency() == 0: filename = os.path.join(self.temp_dir, '%d.h5' % (batch)) self.model.save(filename, save_format='h5') diff --git a/delta/subcommands/commands.py b/delta/subcommands/commands.py index 10042038..5e56b825 100644 --- a/delta/subcommands/commands.py +++ b/delta/subcommands/commands.py @@ -18,15 +18,10 @@ """ Lists all avaiable commands. """ -import delta.imagery.imagery_config -import delta.ml.ml_config from delta.config import config #pylint:disable=import-outside-toplevel -delta.imagery.imagery_config.register() -delta.ml.ml_config.register() - # we put this here because tensorflow takes so long to load, we don't do it unless we have to def main_classify(options): from . import classify diff --git a/delta/subcommands/main.py b/delta/subcommands/main.py new file mode 100644 index 00000000..15bca996 --- /dev/null +++ b/delta/subcommands/main.py @@ -0,0 +1,44 @@ +# Copyright © 2020, United States Government, as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All rights reserved. +# +# The DELTA (Deep Earth Learning, Tools, and Analysis) platform is +# licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import argparse + +from delta.config import config +import delta.config.modules +from delta.subcommands import commands + +def main(args): + delta.config.modules.register_all() + parser = argparse.ArgumentParser(description='DELTA Machine Learning Toolkit') + subparsers = parser.add_subparsers() + + for d in commands.SETUP_COMMANDS: + d(subparsers) + + try: + options = parser.parse_args(args[1:]) + except argparse.ArgumentError: + parser.print_help(sys.stderr) + sys.exit(1) + + if not hasattr(options, 'function'): + parser.print_help(sys.stderr) + sys.exit(1) + + config.initialize(options) + return options.function(options) diff --git a/tests/conftest.py b/tests/conftest.py index 056fcdb6..d7bffbf0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -21,6 +21,7 @@ import os import random import shutil +import sys import tempfile import zipfile @@ -29,12 +30,10 @@ from delta.imagery.sources import tiff -import delta.imagery.imagery_config -import delta.ml.ml_config +import delta.config.modules +delta.config.modules.register_all() -# initialize config files -delta.imagery.imagery_config.register() -delta.ml.ml_config.register() +assert 'tensorflow' not in sys.modules, 'For speed of command line tool, tensorflow should not be imported by config!' def generate_tile(width=32, height=32, blocks=50): """Generate a widthXheightX3 image, with blocks pixels surrounded by ones and the rest zeros in band 0""" diff --git a/tests/test_config.py b/tests/test_config.py index cdfe8fa3..58be8b67 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -26,8 +26,6 @@ from delta.config import config from delta.ml import model_parser -#pylint: disable=import-outside-toplevel - def test_general(): config.reset() @@ -309,7 +307,6 @@ def test_argparser(): assert config.train.network.chunk_size() == 5 im = config.dataset.images() - print(im.preprocess()) assert im.preprocess() is not None assert im.type() == 'tiff' assert len(im) == 1 From ce97b71a0776230617cfd619437c2f11b49329c1 Mon Sep 17 00:00:00 2001 From: ScottMcMichael Date: Thu, 16 Jul 2020 13:46:08 -0700 Subject: [PATCH 32/42] Add option for stop on input error (#5) * Add option to stop/bypass input errors * Make stop on input error defalt true --- delta/config/delta.yaml | 1 + delta/imagery/imagery_dataset.py | 74 ++++++++++++++++++++------------ delta/imagery/utilities.py | 10 ++--- delta/ml/ml_config.py | 7 +++ 4 files changed, 59 insertions(+), 33 deletions(-) diff --git a/delta/config/delta.yaml b/delta/config/delta.yaml index fdb2c424..d0cb9453 100644 --- a/delta/config/delta.yaml +++ b/delta/config/delta.yaml @@ -1,6 +1,7 @@ general: # negative is all gpus: -1 + stop_on_input_error: true # If false skip past bad input files without halting training io: threads: 1 diff --git a/delta/imagery/imagery_dataset.py b/delta/imagery/imagery_dataset.py index 2330b7f2..e449d7a8 100644 --- a/delta/imagery/imagery_dataset.py +++ b/delta/imagery/imagery_dataset.py @@ -24,15 +24,13 @@ import sys import os import portalocker - +import numpy as np import tensorflow as tf from delta.config import config from delta.imagery import rectangle from delta.imagery.sources import loader -#import numpy as np - class ImageryDataset: """Create dataset with all files as described in the provided config file. """ @@ -97,11 +95,20 @@ def _load_tensor_imagery(self, is_labels, image_index, bbox): f.write(str(bbox) + '\n') # TODO: What to write and when to clear it? - image = loader.load_image(data, image_index.numpy()) - w = int(bbox[2]) - h = int(bbox[3]) - rect = rectangle.Rectangle(int(bbox[0]), int(bbox[1]), w, h) - r = image.read(rect) + try: + image = loader.load_image(data, image_index.numpy()) + w = int(bbox[2]) + h = int(bbox[3]) + rect = rectangle.Rectangle(int(bbox[0]), int(bbox[1]), w, h) + r = image.read(rect) + except Exception as e: #pylint: disable=W0703 + print('Caught exception loading tile from image: ' + data[image_index.numpy()] + ' -> ' + str(e) + + '\nSkipping tile: ' + str(bbox)) + if config.general.stop_on_input_error(): + print('Aborting processing, set --bypass-input-errors to bypass this error.') + raise + # Else just skip this tile + r = np.zeros(shape=(0,0,0), dtype=np.float32) return r def _tile_images(self): @@ -116,25 +123,36 @@ def tile_generator(): if self._get_image_read_count(self._images[i]) > config.io.resume_cutoff(): continue - img = loader.load_image(self._images, i) - if self._labels: # If we have labels make sure they are the same size as the input images - label = loader.load_image(self._labels, i) - if label.size() != img.size(): - raise Exception('Label file ' + self._labels[i] + ' with size ' + str(label.size()) - + ' does not match input image size of ' + str(img.size())) - # w * h * bands * 4 * chunk * chunk = max_block_bytes - tile_width = int(math.sqrt(max_block_bytes / img.num_bands() / self._data_type.size / - config.io.tile_ratio())) - tile_height = int(config.io.tile_ratio() * tile_width) - min_block_size = self._chunk_size ** 2 * config.io.tile_ratio() * img.num_bands() * 4 - if max_block_bytes < min_block_size: - print('Warning: max_block_bytes=%g MB, but %g MB is recommended (minimum: %g MB)' % ( \ - max_block_bytes / 1024 / 1024, min_block_size * 2 / 1024 / 1024, min_block_size / 1024/ 1024), - file=sys.stderr) - if tile_width < self._chunk_size or tile_height < self._chunk_size: - raise ValueError('max_block_bytes is too low.') - tiles = img.tiles(tile_width, tile_height, min_width=self._chunk_size, min_height=self._chunk_size, - overlap=self._chunk_size - 1) + try: + img = loader.load_image(self._images, i) + + if self._labels: # If we have labels make sure they are the same size as the input images + label = loader.load_image(self._labels, i) + if label.size() != img.size(): + raise Exception('Label file ' + self._labels[i] + ' with size ' + str(label.size()) + + ' does not match input image size of ' + str(img.size())) + # w * h * bands * 4 * chunk * chunk = max_block_bytes + tile_width = int(math.sqrt(max_block_bytes / img.num_bands() / self._data_type.size / + config.io.tile_ratio())) + tile_height = int(config.io.tile_ratio() * tile_width) + min_block_size = self._chunk_size ** 2 * config.io.tile_ratio() * img.num_bands() * 4 + if max_block_bytes < min_block_size: + print('Warning: max_block_bytes=%g MB, but %g MB is recommended (minimum: %g MB)' + % (max_block_bytes / 1024 / 1024, + min_block_size * 2 / 1024 / 1024, min_block_size / 1024/ 1024), + file=sys.stderr) + if tile_width < self._chunk_size or tile_height < self._chunk_size: + raise ValueError('max_block_bytes is too low.') + tiles = img.tiles(tile_width, tile_height, min_width=self._chunk_size, min_height=self._chunk_size, + overlap=self._chunk_size - 1) + except Exception as e: #pylint: disable=W0703 + print('Caught exception tiling image: ' + self._images[i] + ' -> ' + str(e) + + '\nWill not load any tiles from this image') + if config.general.stop_on_input_error(): + print('Aborting processing, set --bypass-input-errors to bypass this error.') + raise + tiles = [] # Else move past this image without loading any tiles + random.Random(0).shuffle(tiles) # gives consistent random ordering so labels will match tgs.append((i, tiles)) if not tgs: @@ -172,7 +190,7 @@ def load_tile(image_index, x1, y1, x2, y2): # Don't let the entire session be taken down by one bad dataset input. # - Would be better to handle this somehow but it is not clear if TF supports that. - ret = ret.apply(tf.data.experimental.ignore_errors()) +# ret = ret.apply(tf.data.experimental.ignore_errors()) return ret diff --git a/delta/imagery/utilities.py b/delta/imagery/utilities.py index 35b067d4..b64ca206 100644 --- a/delta/imagery/utilities.py +++ b/delta/imagery/utilities.py @@ -39,11 +39,11 @@ def unpack_to_folder(compressed_path, unpack_folder): else: # Assume a tar file with tarfile.TarFile(compressed_path, 'r') as tf: tf.extractall(tmpdir) - except: - shutil.rmtree(tmpdir) - raise - # make this atomic so we don't have incomplete data - os.rename(tmpdir, unpack_folder) + except Exception as e: + shutil.rmtree(tmpdir) # Clear any partially unpacked results + raise RuntimeError('Caught exception unpacking compressed file: ' + compressed_path + + '\n' + str(e)) + os.rename(tmpdir, unpack_folder) # Clean up def progress_bar(text, fill_amount, prefix = '', length = 80): #pylint: disable=W0613 """ diff --git a/delta/ml/ml_config.py b/delta/ml/ml_config.py index 4d650f5b..907ce205 100644 --- a/delta/ml/ml_config.py +++ b/delta/ml/ml_config.py @@ -282,8 +282,15 @@ def register(): """ if not hasattr(config.config, 'general'): config.config.register_component(config.DeltaConfigComponent('General'), 'general') + config.config.general.register_field('gpus', int, 'gpus', None, 'Number of gpus to use.') config.config.general.register_arg('gpus', '--gpus') + config.config.general.register_field('stop_on_input_error', bool, 'stop_on_input_error', None, + 'If false, skip past bad input images.') + config.config.general.register_arg('stop_on_input_error', '--bypass-input-errors', + action='store_const', const=False, type=None) + config.config.general.register_arg('stop_on_input_error', '--stop-on-input-error', + action='store_const', const=True, type=None) config.config.register_component(TrainingConfig(), 'train') config.config.register_component(MLFlowConfig(), 'mlflow') From ccd44ec415e09e75ace47b567c0d35a070f2bf52 Mon Sep 17 00:00:00 2001 From: Brian Coltin Date: Fri, 17 Jul 2020 17:25:10 -0700 Subject: [PATCH 33/42] Support worldview files with subdirectories of same name. (#9) --- delta/imagery/sources/worldview.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/delta/imagery/sources/worldview.py b/delta/imagery/sources/worldview.py index 7db15733..4254f2d8 100644 --- a/delta/imagery/sources/worldview.py +++ b/delta/imagery/sources/worldview.py @@ -82,6 +82,14 @@ def _unpack(self, paths): tf.print('Unpacking file ' + paths + ' to folder ' + unpack_folder, output_stream=sys.stdout) utilities.unpack_to_folder(paths, unpack_folder) + # some worldview zip files have a subdirectory with the name of the image + if not os.path.exists(os.path.join(unpack_folder, 'vendor_metadata')): + subdir = os.path.join(unpack_folder, os.path.splitext(os.path.basename(paths))[0]) + if not os.path.exists(os.path.join(subdir, 'vendor_metadata')): + raise Exception('vendor_metadata not found in %s.' % (paths)) + for filename in os.listdir(subdir): + os.rename(os.path.join(subdir, filename), os.path.join(unpack_folder, filename)) + os.rmdir(subdir) (tif_path, imd_path) = _get_files_from_unpack_folder(unpack_folder) return (tif_path, imd_path) From 81a9e436e3c137a75df5e4b87a1f2eff75e00318 Mon Sep 17 00:00:00 2001 From: Brian Coltin Date: Mon, 20 Jul 2020 12:34:49 -0700 Subject: [PATCH 34/42] Fix memory issues with classify. (#10) * Support worldview files with subdirectories of same name. * Fix memory use in classify. --- delta/imagery/sources/delta_image.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/delta/imagery/sources/delta_image.py b/delta/imagery/sources/delta_image.py index 4e877812..fb984196 100644 --- a/delta/imagery/sources/delta_image.py +++ b/delta/imagery/sources/delta_image.py @@ -127,7 +127,6 @@ def roi_generator(self, requested_rois: Iterator[rectangle.Rectangle]) -> Iterat # gdal doesn't work reading multithreading. But this let's a thread # take care of IO input while we do computation. - exe = concurrent.futures.ThreadPoolExecutor(1) jobs = [] total_rois = len(block_rois) @@ -148,18 +147,25 @@ def roi_generator(self, requested_rois: Iterator[rectangle.Rectangle]) -> Iterat continue applicable_rois.append(block_rois.pop(index)) - buf = exe.submit(functools.partial(self.read, read_roi)) - jobs.append((buf, read_roi, applicable_rois)) + jobs.append((read_roi, applicable_rois)) + # only do a few reads ahead since otherwise we will exhaust our memory + pending = [] + exe = concurrent.futures.ThreadPoolExecutor(1) + NUM_AHEAD = 2 + for i in range(min(NUM_AHEAD, len(jobs))): + pending.append(exe.submit(functools.partial(self.read, jobs[i][0]))) num_remaining = total_rois - for (buf_exe, read_roi, rois) in jobs: - buf = buf_exe.result() + for (i, (read_roi, rois)) in enumerate(jobs): + buf = pending.pop(0).result() for roi in rois: x0 = roi.min_x - read_roi.min_x y0 = roi.min_y - read_roi.min_y num_remaining -= 1 yield (roi, buf[x0:x0 + roi.width(), y0:y0 + roi.height(), :], (total_rois - num_remaining, total_rois)) + if i + NUM_AHEAD < len(jobs): + pending.append(exe.submit(functools.partial(self.read, jobs[i + NUM_AHEAD][0]))) def process_rois(self, requested_rois: Iterator[rectangle.Rectangle], callback_function: Callable[[rectangle.Rectangle, np.ndarray], None], From 2e5700215497b3aac9e8f584a13bae0993d2bab1 Mon Sep 17 00:00:00 2001 From: Brian Coltin Date: Wed, 22 Jul 2020 16:31:58 -0700 Subject: [PATCH 35/42] Rename save_latest to be more clear. (#11) --- delta/config/README.md | 2 +- delta/config/delta.yaml | 4 ++-- delta/ml/ml_config.py | 2 +- delta/ml/train.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/delta/config/README.md b/delta/config/README.md index f3985eaa..e7ff0f42 100644 --- a/delta/config/README.md +++ b/delta/config/README.md @@ -107,7 +107,7 @@ Used in the `delta train` and `delta mlflow_ui` commands to keep track of traini networks from different stages of training. * `frequency`: Frequency in batches to save a checkpoint. Networks can require a fair amount of disk space, so don't save too often. - * `save_latest`: If true, only keep the network file from the most recent checkpoint. + * `only_save_latest`: If true, only keep the network file from the most recent checkpoint. TensorBoard ----------- diff --git a/delta/config/delta.yaml b/delta/config/delta.yaml index d0cb9453..729f1c83 100644 --- a/delta/config/delta.yaml +++ b/delta/config/delta.yaml @@ -96,8 +96,8 @@ mlflow: experiment_name: Default # rate in batches to save model checkpoints checkpoints: - frequency: 10000 - save_latest: true + frequency: 10000 + only_save_latest: true tensorboard: enabled: false diff --git a/delta/ml/ml_config.py b/delta/ml/ml_config.py index 907ce205..d794dd25 100644 --- a/delta/ml/ml_config.py +++ b/delta/ml/ml_config.py @@ -234,7 +234,7 @@ def __init__(self): super().__init__() self.register_field('frequency', int, 'frequency', None, 'Frequency in batches to store neural network checkpoints.') - self.register_field('save_latest', bool, 'save_latest', None, + self.register_field('only_save_latest', bool, 'only_save_latest', None, 'If true, only keep the most recent checkpoint.') class MLFlowConfig(config.DeltaConfigComponent): diff --git a/delta/ml/train.py b/delta/ml/train.py index 3a31e7b2..9468ef6e 100644 --- a/delta/ml/train.py +++ b/delta/ml/train.py @@ -133,7 +133,7 @@ def on_train_batch_end(self, batch, logs=None): if config.mlflow.checkpoints.frequency() and batch % config.mlflow.checkpoints.frequency() == 0: filename = os.path.join(self.temp_dir, '%d.h5' % (batch)) self.model.save(filename, save_format='h5') - if config.mlflow.checkpoints.save_latest(): + if config.mlflow.checkpoints.only_save_latest(): old = filename filename = os.path.join(self.temp_dir, 'latest.h5') os.rename(old, filename) From 9478cea1f869870f9567b2d7cc6f1b474eb0642b Mon Sep 17 00:00:00 2001 From: Brian Coltin Date: Wed, 5 Aug 2020 17:31:11 -0700 Subject: [PATCH 36/42] Fix unit tests for tensorflow 2.3. (#16) --- delta/ml/predict.py | 12 ++++++------ delta/ml/train.py | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/delta/ml/predict.py b/delta/ml/predict.py index 989545bc..94252c88 100644 --- a/delta/ml/predict.py +++ b/delta/ml/predict.py @@ -64,8 +64,8 @@ def _process_block(self, pred_image, x, y, labels): """ def _predict_array(self, data): - net_input_shape = self._model.get_input_shape_at(0)[1:] - net_output_shape = self._model.get_output_shape_at(0)[1:] + net_input_shape = self._model.input_shape[1:] + net_output_shape = self._model.output_shape[1:] assert net_input_shape[2] == data.shape[2],\ 'Model expects %d input channels, data has %d channels' % (net_input_shape[2], data.shape[2]) @@ -101,8 +101,8 @@ def predict(self, image, label=None, input_bounds=None): Results are limited to `input_bounds`. Returns output, the meaning of which depends on the subclass. """ - net_input_shape = self._model.get_input_shape_at(0)[1:] - net_output_shape = self._model.get_output_shape_at(0)[1:] + net_input_shape = self._model.input_shape[1:] + net_output_shape = self._model.output_shape[1:] offset_r = -net_input_shape[0] + net_output_shape[0] offset_c = -net_input_shape[1] + net_output_shape[1] block_size_x = net_input_shape[0] * (_TILE_SIZE // net_input_shape[0]) @@ -166,7 +166,7 @@ def __init__(self, model, output_image=None, show_progress=False, self._errors = None def _initialize(self, shape, label, image): - net_output_shape = self._model.get_output_shape_at(0)[1:] + net_output_shape = self._model.output_shape[1:] self._num_classes = net_output_shape[-1] if label: self._errors = np.zeros(shape, dtype=np.bool) @@ -244,7 +244,7 @@ def __init__(self, model, output_image=None, show_progress=False, transform=None self._transform = transform def _initialize(self, shape, label, image): - net_output_shape = self._model.get_output_shape_at(0)[1:] + net_output_shape = self._model.output_shape[1:] if self._output_image is not None: dtype = np.float32 if self._transform is None else self._transform[1] bands = net_output_shape[-1] if self._transform is None else self._transform[2] diff --git a/delta/ml/train.py b/delta/ml/train.py index 9468ef6e..6d31f96d 100644 --- a/delta/ml/train.py +++ b/delta/ml/train.py @@ -180,8 +180,8 @@ def train(model_fn, dataset : ImageryDataset, training_spec): model.compile(optimizer=training_spec.optimizer, loss=loss, metrics=training_spec.metrics) - input_shape = model.get_input_at(0).shape - output_shape = model.get_output_at(0).shape + input_shape = model.input_shape + output_shape = model.output_shape chunk_size = input_shape[1] assert len(input_shape) == 4, 'Input to network is wrong shape.' From 99e74c3bc5cb789e71ad883789f39c9090f92265 Mon Sep 17 00:00:00 2001 From: Michael Furlong Date: Thu, 6 Aug 2020 19:46:29 +0000 Subject: [PATCH 37/42] Added script for extracting DELTA configs from .h5 files (#14) * Added script to display the number of labels in a label geotif image and how many times each label shows up * Added a script to extract neural network specification from .h5 files into DELTA yaml config files --- scripts/label-img-info | 19 +++++++++++++++++++ scripts/model2config | 24 ++++++++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100755 scripts/label-img-info create mode 100644 scripts/model2config diff --git a/scripts/label-img-info b/scripts/label-img-info new file mode 100755 index 00000000..5bc326d3 --- /dev/null +++ b/scripts/label-img-info @@ -0,0 +1,19 @@ +#!/usr/bin/env python + +import sys +import pathlib +import numpy as np +from osgeo import gdal + +if __name__=='__main__': + assert len(sys.argv) > 1, 'Need to supply a file' + filename = pathlib.Path(sys.argv[1]) + + tif_file = gdal.Open(str(filename)) + assert tif_file is not None, f'Could not open file {filename}' + tif_data = tif_file.ReadAsArray() + unique_labels = np.unique(tif_data) + print(np.any(np.isnan(tif_data)), tif_data.min(), tif_data.max(), tif_data.shape, unique_labels) + print(np.histogram(tif_data, bins=len(unique_labels))) + + diff --git a/scripts/model2config b/scripts/model2config new file mode 100644 index 00000000..51c4162b --- /dev/null +++ b/scripts/model2config @@ -0,0 +1,24 @@ +#!/usr/bin/env python + +import tensorflow as tf +from argparse import ArgumentParser +import pathlib + +parser = ArgumentParser(description='Converts a neural network in a *.h5 file to the DELTA configuration langauge') +parser.add_argument('model_name', type=pathlib.Path, help='The model to convert') + +args = parser.parse_args() + +a = tf.keras.models.load_model(args.model_name) + +for l in a.layers: + print('\t- ', type(l).__name__) + configs = l.get_config() + if isinstance(l.input, list): + print('\t\t- input: ['+ ', '.join([x.name.replace('/Identity:0','') for x in l.input])+ ']') + else: + print('\t\t- input:', l.input.name.replace('/Identity:0','')) + for k in configs.keys(): + if isinstance(configs[k], dict) or configs[k] is None: + continue + print(f'\t\t- {k}: {configs[k]}') From ffce6d0f4cb10c18ac1dacce958d57fb0db9c386 Mon Sep 17 00:00:00 2001 From: Brian Coltin Date: Mon, 10 Aug 2020 19:49:23 -0700 Subject: [PATCH 38/42] Configurable class pixel values, names, and display colors (#17) * Move classes from network to dataset. * Extended classes config for colors and names and arbitrary numbers. * Ignore nodata labelled pixels in predict evaluation. --- delta/config/delta.yaml | 19 ++++++- delta/imagery/imagery_config.py | 93 ++++++++++++++++++++++++++++++--- delta/ml/ml_config.py | 9 ++-- delta/ml/model_parser.py | 2 +- delta/ml/predict.py | 23 ++++++-- delta/subcommands/classify.py | 27 ++++------ tests/test_config.py | 51 +++++++++++++++--- 7 files changed, 186 insertions(+), 38 deletions(-) diff --git a/delta/config/delta.yaml b/delta/config/delta.yaml index 729f1c83..bb62b472 100644 --- a/delta/config/delta.yaml +++ b/delta/config/delta.yaml @@ -44,11 +44,28 @@ dataset: file_list: ~ files: ~ + # can either be a list of classes or the number of classes, + # if the labels are 0, 1, 2, 3 + classes: 4 + # labels are 1, 2, 3, 4; with names and display colors + #classes: + # - 1: + # name: Water + # color: 0x67a9cf + # - 2: + # name: No Water + # color: 0xf6eff7 + # - 3: + # name: Maybe Water + # color: 0xbdc9e1 + # - 4: + # name: Cloud + # color: 0x02818a + train: network: chunk_size: 16 output_size: 8 - classes: 4 model: yaml_file: networks/convpool.yaml params: ~ diff --git a/delta/imagery/imagery_config.py b/delta/imagery/imagery_config.py index b641009d..182fc2f0 100644 --- a/delta/imagery/imagery_config.py +++ b/delta/imagery/imagery_config.py @@ -142,9 +142,10 @@ def __preprocess_function(image_comp): return None return lambda data, _, dummy: data / np.float32(f) -def load_images_labels(images_comp, labels_comp): +def load_images_labels(images_comp, labels_comp, classes_comp): ''' - Takes two configuration subsections and returns (image set, label set) + Takes two configuration subsections and returns (image set, label set). Also takes classes + configuration to apply preprocessing function to labels. ''' images_dict = images_comp._config_dict #pylint:disable=protected-access labels_dict = labels_comp._config_dict #pylint:disable=protected-access @@ -170,7 +171,10 @@ def load_images_labels(images_comp, labels_comp): if len(labels) != len(images): raise ValueError('%d images found, but %d labels found.' % (len(images), len(labels))) - pre = __preprocess_function(labels_comp) + pre = pre_orig = __preprocess_function(labels_comp) + conv = classes_comp.classes_to_indices_func() + if conv is not None: + pre = lambda data, _, dummy: conv(pre_orig(data, _, dummy) if pre_orig is not None else data) return (imageset, ImageSet(labels, labels_dict['type'], pre, labels_dict['nodata_value'])) @@ -194,7 +198,7 @@ def __init__(self, name=None): self.register_field('file_list', list, None, validate_path, 'File listing image files.') self.register_field('directory', str, None, validate_path, 'Directory of image files.') self.register_field('extension', str, None, None, 'Image file extension.') - self.register_field('nodata_value', float, None, None, 'Value of pixels to ignore.') + self.register_field('nodata_value', (float, int), None, None, 'Value of pixels to ignore.') if name: self.register_arg('type', '--' + name + '-type') @@ -218,6 +222,80 @@ def parse_args(self, options): if hasattr(options, self._name) and getattr(options, self._name) is not None: self._config_dict['files'] = [getattr(options, self._name)] +class LabelClass: + def __init__(self, value, name=None, color=None): + color_order = [0x1f77b4, 0xff7f0e, 0x2ca02c, 0xd62728, 0x9467bd, 0x8c564b, \ + 0xe377c2, 0x7f7f7f, 0xbcbd22, 0x17becf] + if name is None: + name = 'Class ' + str(value) + if color is None: + color = color_order[value] if value < len(color_order) else 0 + self.value = value + self.name = name + self.color = color + def __repr__(self): + return 'Color: ' + self.name + +class ClassesConfig(DeltaConfigComponent): + def __init__(self): + super().__init__() + self._classes = [] + self._conversions = [] + + def __iter__(self): + return self._classes.__iter__() + + def __len__(self): + return len(self._classes) + + # overwrite model entirely if updated (don't want combined layers from multiple files) + def _load_dict(self, d : dict, base_dir): + if not d: + return + self._classes = [] + if isinstance(d, int): + for i in range(d): + self._classes.append(LabelClass(i)) + elif isinstance(d, list): + for (i, c) in enumerate(d): + if isinstance(c, int): # just pixel value + self._classes.append(LabelClass(i)) + else: + keys = c.keys() + assert len(keys) == 1, 'Dict should have name of pixel value.' + k = next(iter(keys)) + assert isinstance(k, int), 'Class label value must be int.' + inner_dict = c[k] + self._classes.append(LabelClass(k, str(inner_dict.get('name')), inner_dict.get('color'))) + else: + raise ValueError('Expected classes to be an int or list in config, was ' + str(d)) + # make sure the order is consistent for same values, and create preprocessing function + self._conversions = [] + self._classes = sorted(self._classes, key=lambda x: x.value) + for (i, v) in enumerate(self._classes): + if v.value != i: + self._conversions.append(v.value) + + def classes_to_indices_func(self): + if not self._conversions: + return None + def convert(data): + assert isinstance(data, np.ndarray) + for (i, c) in enumerate(self._conversions): + data[data == c] = i + return data + return convert + + def indices_to_classes_func(self): + if not self._conversions: + return None + def convert(data): + assert isinstance(data, np.ndarray) + for (i, c) in reversed(list(enumerate(self._conversions))): + data[data == i] = c + return data + return convert + class DatasetConfig(DeltaConfigComponent): def __init__(self): super().__init__('Dataset') @@ -227,6 +305,7 @@ def __init__(self): self.__labels = None self.register_field('log_folder', str, 'log_folder', validate_path, 'Directory where dataset progress is recorded.') + self.register_component(ClassesConfig(), 'classes') def reset(self): super().reset() @@ -239,7 +318,8 @@ def images(self) -> ImageSet: """ if self.__images is None: (self.__images, self.__labels) = load_images_labels(self._components['images'], - self._components['labels']) + self._components['labels'], + self._components['classes']) return self.__images def labels(self) -> ImageSet: @@ -248,7 +328,8 @@ def labels(self) -> ImageSet: """ if self.__labels is None: (self.__images, self.__labels) = load_images_labels(self._components['images'], - self._components['labels']) + self._components['labels'], + self._components['classes']) return self.__labels class CacheConfig(DeltaConfigComponent): diff --git a/delta/ml/ml_config.py b/delta/ml/ml_config.py index d794dd25..9eed4efb 100644 --- a/delta/ml/ml_config.py +++ b/delta/ml/ml_config.py @@ -133,12 +133,9 @@ def __init__(self): 'Width of an image chunk to input to the neural network.') self.register_field('output_size', int, 'output_size', config.validate_positive, 'Width of an image chunk to output from the neural network.') - self.register_field('classes', int, 'classes', config.validate_positive, - 'Number of label classes.') self.register_arg('chunk_size', '--chunk-size') self.register_arg('output_size', '--output-size') - self.register_arg('classes', '--classes') self.register_component(NetworkModelConfig(), 'model') def setup_arg_parser(self, parser, components = None) -> None: @@ -168,7 +165,8 @@ def images(self) -> ImageSet: """ if self.__images is None: (self.__images, self.__labels) = load_images_labels(self._components['images'], - self._components['labels']) + self._components['labels'], + config.dataset.classes) return self.__images def labels(self) -> ImageSet: @@ -177,7 +175,8 @@ def labels(self) -> ImageSet: """ if self.__labels is None: (self.__images, self.__labels) = load_images_labels(self._components['images'], - self._components['labels']) + self._components['labels'], + config.dataset.classes) return self.__labels class TrainingConfig(config.DeltaConfigComponent): diff --git a/delta/ml/model_parser.py b/delta/ml/model_parser.py index 0d6fc078..ff9c15d6 100644 --- a/delta/ml/model_parser.py +++ b/delta/ml/model_parser.py @@ -146,7 +146,7 @@ def config_model(num_bands: int) -> Callable[[], tensorflow.keras.models.Sequent """ in_data_shape = (config.train.network.chunk_size(), config.train.network.chunk_size(), num_bands) out_data_shape = (config.train.network.output_size(), config.train.network.output_size(), - config.train.network.classes()) + len(config.dataset.classes)) params_exposed = {'out_shape' : out_data_shape, 'out_dims' : out_data_shape[0] * out_data_shape[1] * out_data_shape[2], diff --git a/delta/ml/predict.py b/delta/ml/predict.py index 94252c88..6c3e752e 100644 --- a/delta/ml/predict.py +++ b/delta/ml/predict.py @@ -72,7 +72,7 @@ def _predict_array(self, data): out_shape = (data.shape[0] - net_input_shape[0] + net_output_shape[0], data.shape[1] - net_input_shape[1] + net_output_shape[1]) - out_type = self._model.get_output_at(0).dtype + out_type = tf.dtypes.as_dtype(self._model.dtype) image = tf.convert_to_tensor(data) image = tf.expand_dims(image, 0) chunks = tf.image.extract_patches(image, [1, net_input_shape[0], net_input_shape[1], 1], @@ -141,11 +141,12 @@ def callback_function(roi, data): raise return self._complete() + class LabelPredictor(Predictor): """ Predicts integer labels for an image. """ - def __init__(self, model, output_image=None, show_progress=False, + def __init__(self, model, output_image=None, show_progress=False, nodata_value=None, # pylint:disable=too-many-arguments colormap=None, prob_image=None, error_image=None, error_colors=None): """ output_image, prob_image, and error_image are all DeltaImageWriter's. @@ -155,6 +156,16 @@ def __init__(self, model, output_image=None, show_progress=False, self._confusion_matrix = None self._num_classes = None self._output_image = output_image + if colormap is not None: + # convert python list to numpy array + if not isinstance(colormap, np.ndarray): + a = np.zeros(shape=(len(colormap), 3), dtype=np.uint8) + for (i, v) in enumerate(colormap): + a[i][0] = (v >> 16) & 0xFF + a[i][1] = (v >> 8) & 0xFF + a[i][2] = v & 0xFF + colormap = a + self._nodata_value = nodata_value self._colormap = colormap self._prob_image = prob_image self._error_image = error_image @@ -215,7 +226,13 @@ def _process_block(self, pred_image, x, y, labels): self._output_image.write(pred_image, x, y) if labels is not None: - self._error_image.write(self._error_colors[(labels != pred_image).astype(int)], x, y) + eimg = self._error_colors[(labels != pred_image).astype(int)] + if self._nodata_value is not None: + valid = (labels != self._nodata_value) + eimg[np.logical_not(valid)] = np.zeros(eimg.shape[-1:], dtype=eimg.dtype) + labels = labels[valid] + pred_image = pred_image[valid] + self._error_image.write(eimg, x, y) cm = tf.math.confusion_matrix(np.ndarray.flatten(labels), np.ndarray.flatten(pred_image), self._num_classes) diff --git a/delta/subcommands/classify.py b/delta/subcommands/classify.py index 661194e6..45ec4dc2 100644 --- a/delta/subcommands/classify.py +++ b/delta/subcommands/classify.py @@ -33,16 +33,18 @@ import delta.imagery.imagery_config import delta.ml.ml_config -def save_confusion(cm, filename): +def save_confusion(cm, class_labels, filename): f = plt.figure() ax = f.add_subplot(1, 1, 1) image = ax.imshow(cm, interpolation='nearest', cmap=plt.get_cmap('inferno')) ax.set_title('Confusion Matrix') f.colorbar(image) + ax.set_xlim(-0.5, cm.shape[0] - 0.5) + ax.set_ylim(-0.5, cm.shape[0] - 0.5) ax.set_xticks(range(cm.shape[0])) ax.set_yticks(range(cm.shape[0])) - ax.set_xlim(-0.5, cm.shape[0]-0.5) - ax.set_ylim(-0.5, cm.shape[0]-0.5) + ax.set_xticklabels(class_labels) + ax.set_yticklabels(class_labels) m = cm.max() total = cm.sum() @@ -51,7 +53,7 @@ def save_confusion(cm, filename): ax.text(j, i, '%d\n%.2g%%' % (cm[i, j], cm[i, j] / total * 100), horizontalalignment='center', color='white' if cm[i, j] < m / 2 else 'black') ax.set_ylabel('True Label') - ax.set_xlabel('Predicated Label') + ax.set_xlabel('Predicted Label') f.savefig(filename) def ae_convert(data): @@ -68,15 +70,7 @@ def main(options): else: model = tf.keras.models.load_model(options.model, custom_objects=delta.ml.layers.ALL_LAYERS) - colors = np.array([[0x0, 0x0, 0x0], - [0x67, 0xa9, 0xcf], - [0xf6, 0xef, 0xf7], - [0xbd, 0xc9, 0xe1], - [0x02, 0x81, 0x8a], - [0x00, 0xff, 0xff], # TODO: Label and clean up colormap - [0xff, 0x00, 0xff], - [0xff, 0xff, 0x00]], - dtype=np.uint8) + colors = list(map(lambda x: x.color, config.dataset.classes)) error_colors = np.array([[0x0, 0x0, 0x0], [0xFF, 0x00, 0x00]], dtype=np.uint8) if options.noColormap: @@ -107,8 +101,9 @@ def main(options): label = image predictor = predict.ImagePredictor(model, output_image, True, (ae_convert, np.uint8, 3)) else: - predictor = predict.LabelPredictor(model, output_image, True, colormap=colors, prob_image=prob_image, - error_image=error_image, error_colors=error_colors) + predictor = predict.LabelPredictor(model, output_image, True, labels.nodata_value(), colormap=colors, + prob_image=prob_image, error_image=error_image, + error_colors=error_colors) try: if cpuOnly: @@ -123,7 +118,7 @@ def main(options): if labels: cm = predictor.confusion_matrix() print('%.2g%% Correct: %s' % (np.sum(np.diag(cm)) / np.sum(cm) * 100, path)) - save_confusion(cm, 'confusion_' + base_name + '.pdf') + save_confusion(cm, map(lambda x: x.name, config.dataset.classes), 'confusion_' + base_name + '.pdf') if options.autoencoder: tiff.write_tiff('orig_' + base_name + '.tiff', ae_convert(image.read()), diff --git a/tests/test_config.py b/tests/test_config.py index 58be8b67..ed6e0f28 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -21,6 +21,7 @@ import pytest import yaml +import numpy as np import tensorflow as tf from delta.config import config @@ -92,6 +93,43 @@ def test_images_files(): assert len(im) == 1 assert im[0] == file_path +def test_classes(): + config.reset() + test_str = ''' + dataset: + classes: 2 + ''' + config.load(yaml_str=test_str) + assert len(config.dataset.classes) == 2 + for (i, c) in enumerate(config.dataset.classes): + assert c.value == i + config.reset() + test_str = ''' + dataset: + classes: + - 2: + name: 2 + color: 2 + - 1: + name: 1 + color: 1 + - 5: + name: 5 + color: 5 + ''' + config.load(yaml_str=test_str) + assert config.dataset.classes + values = [1, 2, 5] + for (i, c) in enumerate(config.dataset.classes): + e = values[i] + assert c.value == e + assert c.name == str(e) + assert c.color == e + arr = np.array(values) + ind = config.dataset.classes.classes_to_indices_func()(arr) + assert np.max(ind) == 2 + assert (config.dataset.classes.indices_to_classes_func()(ind) == values).all() + def test_model_from_dict(): config.reset() test_str = ''' @@ -169,20 +207,20 @@ def test_pretrained_layer(): def test_network_file(): config.reset() test_str = ''' + dataset: + classes: 3 train: network: chunk_size: 5 - classes: 3 model: yaml_file: networks/convpool.yaml ''' config.load(yaml_str=test_str) assert config.train.network.chunk_size() == 5 - assert config.train.network.classes() == 3 model = model_parser.config_model(2)() assert model.input_shape == (None, config.train.network.chunk_size(), config.train.network.chunk_size(), 2) assert model.output_shape == (None, config.train.network.output_size(), - config.train.network.output_size(), config.train.network.classes()) + config.train.network.output_size(), len(config.dataset.classes)) def test_validate(): config.reset() @@ -205,11 +243,12 @@ def test_validate(): def test_network_inline(): config.reset() test_str = ''' + dataset: + classes: 3 train: network: chunk_size: 5 output_size: 1 - classes: 3 model: params: v1 : 10 @@ -225,10 +264,10 @@ def test_network_inline(): ''' config.load(yaml_str=test_str) assert config.train.network.chunk_size() == 5 - assert config.train.network.classes() == 3 + assert len(config.dataset.classes) == 3 model = model_parser.config_model(2)() assert model.input_shape == (None, config.train.network.chunk_size(), config.train.network.chunk_size(), 2) - assert model.output_shape == (None, config.train.network.classes()) + assert model.output_shape == (None, len(config.dataset.classes)) def test_train(): config.reset() From ad011356805de78725e2684df9b176722b68e354 Mon Sep 17 00:00:00 2001 From: Brian Coltin Date: Thu, 13 Aug 2020 10:39:00 -0700 Subject: [PATCH 39/42] Add config file to h5 files saved by delta. (#20) Also fix bug causing crash with validation. --- delta/config/config.py | 18 +++++++++++++----- delta/imagery/imagery_config.py | 1 + delta/ml/io.py | 32 ++++++++++++++++++++++++++++++++ delta/ml/ml_config.py | 22 +++++++--------------- delta/ml/model_parser.py | 2 +- delta/ml/train.py | 7 ++++--- delta/subcommands/train.py | 3 ++- scripts/model2config | 10 +++++++++- setup.py | 3 ++- 9 files changed, 71 insertions(+), 27 deletions(-) create mode 100644 delta/ml/io.py mode change 100644 => 100755 scripts/model2config diff --git a/delta/config/config.py b/delta/config/config.py index aba3b0ae..bbb2127f 100644 --- a/delta/config/config.py +++ b/delta/config/config.py @@ -125,14 +125,22 @@ def register_arg(self, field, argname, **kwargs): kwargs['default'] = _NotSpecified self._cmd_args[argname] = (field, kwargs) + def to_dict(self) -> dict: + """ + Returns a dictionary representing the config object. + """ + if isinstance(self._config_dict, dict): + exp = self._config_dict.copy() + for (name, c) in self._components.items(): + exp[name] = c.to_dict() + return exp + return self._config_dict + def export(self) -> str: """ - Returns a YAML string of all configuration options. + Returns a YAML string of all configuration options, from to_dict. """ - exp = self._config_dict.copy() - for (name, c) in self._components.items(): - exp[name] = c.export() - return yaml.dump(exp) + return yaml.dump(self.to_dict()) def _set_field(self, name : str, value : str, base_dir : str): if name not in self._fields: diff --git a/delta/imagery/imagery_config.py b/delta/imagery/imagery_config.py index 182fc2f0..dbf7a85d 100644 --- a/delta/imagery/imagery_config.py +++ b/delta/imagery/imagery_config.py @@ -252,6 +252,7 @@ def __len__(self): def _load_dict(self, d : dict, base_dir): if not d: return + self._config_dict = d self._classes = [] if isinstance(d, int): for i in range(d): diff --git a/delta/ml/io.py b/delta/ml/io.py new file mode 100644 index 00000000..12e4733f --- /dev/null +++ b/delta/ml/io.py @@ -0,0 +1,32 @@ +# Copyright © 2020, United States Government, as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All rights reserved. +# +# The DELTA (Deep Earth Learning, Tools, and Analysis) platform is +# licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Functions for IO specific to ML. +""" + +import h5py + +from delta.config import config + +def save_model(model, filename): + """ + Save a model. Includes DELTA configuration. + """ + model.save(filename, save_format='h5') + with h5py.File(filename, 'r+') as f: + f.attrs['delta'] = config.export() diff --git a/delta/ml/ml_config.py b/delta/ml/ml_config.py index 9eed4efb..10ea3bcf 100644 --- a/delta/ml/ml_config.py +++ b/delta/ml/ml_config.py @@ -106,25 +106,17 @@ def _load_dict(self, d : dict, base_dir): self._config_dict['layers'] = None elif 'layers' in d: self._config_dict['yaml_file'] = None - - def as_dict(self) -> dict: - """ - Returns a dictionary representing the network model for use by `delta.ml.model_parser`. - """ - yaml_file = self._config_dict['yaml_file'] - if yaml_file is not None: - if self._config_dict['layers'] is not None: - raise ValueError('Specified both yaml file and layers in model.') - + if 'yaml_file' in d and 'layers' in d and d['yaml_file'] is not None and d['layers'] is not None: + raise ValueError('Specified both yaml file and layers in model.') + if 'yaml_file' in d and d['yaml_file'] is not None: + yaml_file = d['yaml_file'] resource = os.path.join('config', yaml_file) if not os.path.exists(yaml_file) and pkg_resources.resource_exists('delta', resource): yaml_file = pkg_resources.resource_filename('delta', resource) if not os.path.exists(yaml_file): raise ValueError('Model yaml_file does not exist: ' + yaml_file) - #print('Opening model file: ' + yaml_file) with open(yaml_file, 'r') as f: - return yaml.safe_load(f) - return self._config_dict + self._config_dict.update(yaml.safe_load(f)) class NetworkConfig(config.DeltaConfigComponent): def __init__(self): @@ -166,7 +158,7 @@ def images(self) -> ImageSet: if self.__images is None: (self.__images, self.__labels) = load_images_labels(self._components['images'], self._components['labels'], - config.dataset.classes) + config.config.dataset.classes) return self.__images def labels(self) -> ImageSet: @@ -176,7 +168,7 @@ def labels(self) -> ImageSet: if self.__labels is None: (self.__images, self.__labels) = load_images_labels(self._components['images'], self._components['labels'], - config.dataset.classes) + config.config.dataset.classes) return self.__labels class TrainingConfig(config.DeltaConfigComponent): diff --git a/delta/ml/model_parser.py b/delta/ml/model_parser.py index ff9c15d6..3ba37ff3 100644 --- a/delta/ml/model_parser.py +++ b/delta/ml/model_parser.py @@ -154,4 +154,4 @@ def config_model(num_bands: int) -> Callable[[], tensorflow.keras.models.Sequent 'in_dims' : in_data_shape[0] * in_data_shape[1] * in_data_shape[2], 'num_bands' : in_data_shape[2]} - return model_from_dict(config.train.network.model.as_dict(), params_exposed) + return model_from_dict(config.train.network.model.to_dict(), params_exposed) diff --git a/delta/ml/train.py b/delta/ml/train.py index 6d31f96d..38dbf56a 100644 --- a/delta/ml/train.py +++ b/delta/ml/train.py @@ -31,6 +31,7 @@ from delta.imagery.imagery_dataset import ImageryDataset from delta.imagery.imagery_dataset import AutoencoderDataset from .layers import DeltaLayer +from .io import save_model def _devices(num_gpus): ''' @@ -132,7 +133,7 @@ def on_train_batch_end(self, batch, logs=None): mlflow.log_metric(k, logs[k], step=batch) if config.mlflow.checkpoints.frequency() and batch % config.mlflow.checkpoints.frequency() == 0: filename = os.path.join(self.temp_dir, '%d.h5' % (batch)) - self.model.save(filename, save_format='h5') + save_model(self.model, filename) if config.mlflow.checkpoints.only_save_latest(): old = filename filename = os.path.join(self.temp_dir, 'latest.h5') @@ -228,7 +229,7 @@ def train(model_fn, dataset : ImageryDataset, training_spec): if config.mlflow.enabled(): model_path = os.path.join(mcb.temp_dir, 'final_model.h5') print('\nFinished, saving model to %s.' % (mlflow.get_artifact_uri() + '/final_model.h5')) - model.save(model_path, save_format='h5') + save_model(model, model_path) mlflow.log_artifact(model_path) os.remove(model_path) mlflow.log_param('Status', 'Completed') @@ -238,7 +239,7 @@ def train(model_fn, dataset : ImageryDataset, training_spec): mlflow.end_run('FAILED') model_path = os.path.join(mcb.temp_dir, 'aborted_model.h5') print('\nAborting, saving current model to %s.' % (mlflow.get_artifact_uri() + '/aborted_model.h5')) - model.save(model_path, save_format='h5') + save_model(model, model_path) mlflow.log_artifact(model_path) os.remove(model_path) raise diff --git a/delta/subcommands/train.py b/delta/subcommands/train.py index e94a7491..85b0ee31 100644 --- a/delta/subcommands/train.py +++ b/delta/subcommands/train.py @@ -33,6 +33,7 @@ from delta.ml.train import train from delta.ml.model_parser import config_model from delta.ml.layers import ALL_LAYERS +from delta.ml.io import save_model #tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.DEBUG) @@ -74,7 +75,7 @@ def main(options): model, _ = train(model, ids, tc) if options.model is not None: - model.save(options.model) + save_model(model, options.model) except KeyboardInterrupt: print() print('Training cancelled.') diff --git a/scripts/model2config b/scripts/model2config old mode 100644 new mode 100755 index 51c4162b..3b74504c --- a/scripts/model2config +++ b/scripts/model2config @@ -2,6 +2,7 @@ import tensorflow as tf from argparse import ArgumentParser +import h5py import pathlib parser = ArgumentParser(description='Converts a neural network in a *.h5 file to the DELTA configuration langauge') @@ -9,8 +10,15 @@ parser.add_argument('model_name', type=pathlib.Path, help='The model to convert' args = parser.parse_args() -a = tf.keras.models.load_model(args.model_name) +print('Configuration File') +with h5py.File(args.model_name, 'r') as f: + if 'delta' not in f.attrs: + print(' - Not Available\n') + else: + print('\n' + f.attrs['delta'] + '\n') +a = tf.keras.models.load_model(args.model_name) +print('Network Structure') for l in a.layers: print('\t- ', type(l).__name__) configs = l.get_config() diff --git a/setup.py b/setup.py index 1a9a62ca..36ba9160 100644 --- a/setup.py +++ b/setup.py @@ -54,7 +54,8 @@ 'mlflow', 'portalocker', 'appdirs', - 'gdal' + 'gdal', + 'h5py' ], scripts=scripts, include_package_data = True, From 2c08f79a306b0108aeaca5411b8c963bd4cd6cf5 Mon Sep 17 00:00:00 2001 From: Brian Coltin Date: Fri, 14 Aug 2020 15:28:45 -0700 Subject: [PATCH 40/42] Add class weights for training. (#22) --- delta/config/delta.yaml | 5 +++++ delta/imagery/imagery_config.py | 17 +++++++++++++++-- delta/imagery/imagery_dataset.py | 7 ++++++- delta/ml/train.py | 5 +---- tests/test_config.py | 5 +++++ tests/test_imagery_dataset.py | 2 +- 6 files changed, 33 insertions(+), 8 deletions(-) diff --git a/delta/config/delta.yaml b/delta/config/delta.yaml index bb62b472..82dbf2f8 100644 --- a/delta/config/delta.yaml +++ b/delta/config/delta.yaml @@ -48,19 +48,24 @@ dataset: # if the labels are 0, 1, 2, 3 classes: 4 # labels are 1, 2, 3, 4; with names and display colors + # weight is optional, but required for either all or none #classes: # - 1: # name: Water # color: 0x67a9cf + # weight: 5.0 # - 2: # name: No Water # color: 0xf6eff7 + # weight: 1.0 # - 3: # name: Maybe Water # color: 0xbdc9e1 + # weight: 1.0 # - 4: # name: Cloud # color: 0x02818a + # weight: 1.0 train: network: diff --git a/delta/imagery/imagery_config.py b/delta/imagery/imagery_config.py index dbf7a85d..9720a37b 100644 --- a/delta/imagery/imagery_config.py +++ b/delta/imagery/imagery_config.py @@ -223,7 +223,7 @@ def parse_args(self, options): self._config_dict['files'] = [getattr(options, self._name)] class LabelClass: - def __init__(self, value, name=None, color=None): + def __init__(self, value, name=None, color=None, weight=None): color_order = [0x1f77b4, 0xff7f0e, 0x2ca02c, 0xd62728, 0x9467bd, 0x8c564b, \ 0xe377c2, 0x7f7f7f, 0xbcbd22, 0x17becf] if name is None: @@ -233,6 +233,8 @@ def __init__(self, value, name=None, color=None): self.value = value self.name = name self.color = color + self.weight = weight + def __repr__(self): return 'Color: ' + self.name @@ -267,7 +269,8 @@ def _load_dict(self, d : dict, base_dir): k = next(iter(keys)) assert isinstance(k, int), 'Class label value must be int.' inner_dict = c[k] - self._classes.append(LabelClass(k, str(inner_dict.get('name')), inner_dict.get('color'))) + self._classes.append(LabelClass(k, str(inner_dict.get('name')), + inner_dict.get('color'), inner_dict.get('weight'))) else: raise ValueError('Expected classes to be an int or list in config, was ' + str(d)) # make sure the order is consistent for same values, and create preprocessing function @@ -277,6 +280,16 @@ def _load_dict(self, d : dict, base_dir): if v.value != i: self._conversions.append(v.value) + def weights(self): + weights = [] + for c in self._classes: + if c.weight is not None: + weights.append(c.weight) + if not weights: + return None + assert len(weights) == len(self._classes), 'For class weights, either all or none must be specified.' + return weights + def classes_to_indices_func(self): if not self._conversions: return None diff --git a/delta/imagery/imagery_dataset.py b/delta/imagery/imagery_dataset.py index e449d7a8..00bae651 100644 --- a/delta/imagery/imagery_dataset.py +++ b/delta/imagery/imagery_dataset.py @@ -235,9 +235,11 @@ def labels(self): label_set = label_set.map(self._reshape_labels, num_parallel_calls=tf.data.experimental.AUTOTUNE) #pylint: disable=C0301 return label_set.unbatch() - def dataset(self): + def dataset(self, class_weights=None): """ Return the underlying TensorFlow dataset object that this class creates. + + class_weights: a list of weights to apply to the samples in each class, if specified. """ # Pair the data and labels in our dataset @@ -245,6 +247,9 @@ def dataset(self): # ignore labels with no data if self._labels.nodata_value(): ds = ds.filter(lambda x, y: tf.math.not_equal(y, self._labels.nodata_value())) + if class_weights is not None: + lookup = tf.constant(class_weights) + ds = ds.map(lambda x, y: (x, y, tf.gather(lookup, tf.cast(y, tf.int32), axis=None))) return ds def num_bands(self): diff --git a/delta/ml/train.py b/delta/ml/train.py index 38dbf56a..6e76843b 100644 --- a/delta/ml/train.py +++ b/delta/ml/train.py @@ -20,7 +20,6 @@ """ import os -import sys import tempfile import shutil @@ -62,7 +61,7 @@ def _strategy(devices): return strategy def _prep_datasets(ids, tc, chunk_size, output_size): - ds = ids.dataset() + ds = ids.dataset(config.dataset.classes.weights()) ds = ds.batch(tc.batch_size) #ds = ds.cache() ds = ds.prefetch(tf.data.experimental.AUTOTUNE) @@ -138,8 +137,6 @@ def on_train_batch_end(self, batch, logs=None): old = filename filename = os.path.join(self.temp_dir, 'latest.h5') os.rename(old, filename) - tf.print('Recording checkpoint: ' + filename, - output_stream=sys.stdout) mlflow.log_artifact(filename, 'checkpoints') os.remove(filename) diff --git a/tests/test_config.py b/tests/test_config.py index ed6e0f28..5586e471 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -103,6 +103,7 @@ def test_classes(): assert len(config.dataset.classes) == 2 for (i, c) in enumerate(config.dataset.classes): assert c.value == i + assert config.dataset.classes.weights() is None config.reset() test_str = ''' dataset: @@ -110,12 +111,15 @@ def test_classes(): - 2: name: 2 color: 2 + weight: 5.0 - 1: name: 1 color: 1 + weight: 1.0 - 5: name: 5 color: 5 + weight: 2.0 ''' config.load(yaml_str=test_str) assert config.dataset.classes @@ -125,6 +129,7 @@ def test_classes(): assert c.value == e assert c.name == str(e) assert c.color == e + assert config.dataset.classes.weights() == [1.0, 5.0, 2.0] arr = np.array(values) ind = config.dataset.classes.classes_to_indices_func()(arr) assert np.max(ind) == 2 diff --git a/tests/test_imagery_dataset.py b/tests/test_imagery_dataset.py index d7bfb135..fece4a32 100644 --- a/tests/test_imagery_dataset.py +++ b/tests/test_imagery_dataset.py @@ -127,7 +127,7 @@ def model_fn(): model, _ = train.train(model_fn, dataset, TrainingSpec(100, 5, 'sparse_categorical_crossentropy', ['accuracy'])) ret = model.evaluate(x=dataset.dataset().batch(1000)) - assert ret[1] > 0.90 + assert ret[1] > 0.70 (test_image, test_label) = conftest.generate_tile() test_label = test_label[1:-1, 1:-1] From bc4dafc4e11abe6e83bcdf063a12dee5cdc263c8 Mon Sep 17 00:00:00 2001 From: Michael Furlong Date: Tue, 18 Aug 2020 23:53:14 +0000 Subject: [PATCH 41/42] Worldview caching name problems (#23) Modified: the caching for the worldview images. Right now it assumes that the zip files that contain the .tif files will be named with a particular format. I changed it to query the zipfile for *.tif files, and if there is one (and only one) in there, queries that file name for the date, sensor, etc. --- delta/imagery/sources/worldview.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/delta/imagery/sources/worldview.py b/delta/imagery/sources/worldview.py index 4254f2d8..fb1b0923 100644 --- a/delta/imagery/sources/worldview.py +++ b/delta/imagery/sources/worldview.py @@ -20,6 +20,7 @@ """ import math +import zipfile import functools import os import sys @@ -101,10 +102,20 @@ def _prep(self, paths): TODO: Apply TOA conversion! """ assert isinstance(paths, str) - parts = os.path.basename(paths).split('_') + (_, ext) = os.path.splitext(paths) + assert '.zip' in ext, f'Error: Was assuming a zip file. Found {paths}' + + zip_file = zipfile.ZipFile(paths, 'r') + tif_names = list(filter(lambda x: '.tif' in x, zip_file.namelist())) + assert len(tif_names) > 0, f'Error: no tif files in the file {paths}' + assert len(tif_names) == 1, f'Error: too many tif files in {paths}: {tif_names}' + tif_name = tif_names[0] + + + parts = os.path.basename(tif_name).split('_') self._sensor = parts[0][0:4] self._date = parts[2][6:14] - self._name = os.path.splitext(os.path.basename(paths))[0] + self._name = os.path.splitext(os.path.basename(tif_name))[0] (tif_path, imd_path) = self._unpack(paths) From 7e59a5de31b0cf95a7b16af7e6ba3b19fb621359 Mon Sep 17 00:00:00 2001 From: Brian Coltin Date: Thu, 20 Aug 2020 12:33:54 -0700 Subject: [PATCH 42/42] Generate gh-pages Branch on Push to Master (#24) --- .github/workflows/docs.yaml | 50 +++++++++++++++++++++++++++++++++++++ scripts/docs.sh | 3 ++- 2 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/docs.yaml diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml new file mode 100644 index 00000000..25ba65f5 --- /dev/null +++ b/.github/workflows/docs.yaml @@ -0,0 +1,50 @@ +# Build documentation and commit to gh-pages branch. + +name: Build and Push Documentation to gh-pages Branch + +on: + push: + branches: [ 'master'] + +jobs: + build_and_push_docs: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + path: repo/ + - name: Checkout gh-pages + uses: actions/checkout@v2 + with: + path: docs/ + ref: gh-pages + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Install pdoc3 + run: | + python3 -m pip install pdoc3 + - name: Install DELTA + run: | + cd repo + ./scripts/setup.sh + python3 -m pip install . + - name: Build Documentation + run: | + ./repo/scripts/docs.sh ./docs/ + - name: Commit and Push + run: | + cd repo + EMAIL=`git show -s --format='%ae' HEAD` + NAME=`git show -s --format='%an' HEAD` + cd .. + cd docs/ + git add . + git config user.email "$EMAIL" + git config user.name "$NAME" + git commit -m "Automatic update for $GITHUB_SHA." + git push origin gh-pages + diff --git a/scripts/docs.sh b/scripts/docs.sh index 0af389d9..c8b68735 100755 --- a/scripts/docs.sh +++ b/scripts/docs.sh @@ -2,5 +2,6 @@ SCRIPT=$(readlink -f "$0") SCRIPTPATH=$(dirname "$SCRIPT") +OUT_DIR=$(readlink -m ${1:-./html}) cd $SCRIPTPATH/.. -pdoc3 --html -c show_type_annotations=True delta --force +pdoc3 --html -c show_type_annotations=True delta --force -o $OUT_DIR