From 4c480f60aae3a93332b23ae65d890f2bf82adbc1 Mon Sep 17 00:00:00 2001 From: Anne Haley Date: Wed, 19 Jun 2024 14:45:57 +0000 Subject: [PATCH 01/10] Make file_item relation optional (a MapLayer can exist on a Dataset without a FileItem) --- uvdat/core/admin.py | 14 ++++++-- .../core/migrations/0003_fileitem_optional.py | 36 +++++++++++++++++++ uvdat/core/models/dataset.py | 4 +-- uvdat/core/models/map_layers.py | 6 ++-- uvdat/core/models/simulations.py | 4 +-- uvdat/core/rest/dataset.py | 2 +- uvdat/core/rest/map_layers.py | 4 +-- uvdat/core/rest/serializers.py | 25 +++++++------ uvdat/core/tasks/dataset.py | 4 +-- uvdat/core/tasks/map_layers.py | 4 +-- uvdat/core/tasks/networks.py | 14 ++++---- uvdat/core/tasks/regions.py | 2 +- uvdat/core/tasks/simulations.py | 4 +-- web/src/components/OptionsDrawerContents.vue | 8 ++--- web/src/components/SimulationsPanel.vue | 2 +- web/src/types.ts | 1 + 16 files changed, 93 insertions(+), 41 deletions(-) create mode 100644 uvdat/core/migrations/0003_fileitem_optional.py diff --git a/uvdat/core/admin.py b/uvdat/core/admin.py index 22506892..0276cc7e 100644 --- a/uvdat/core/admin.py +++ b/uvdat/core/admin.py @@ -43,14 +43,24 @@ class RasterMapLayerAdmin(admin.ModelAdmin): list_display = ['id', 'get_dataset_name', 'index'] def get_dataset_name(self, obj): - return obj.file_item.dataset.name + return obj.dataset.name class VectorMapLayerAdmin(admin.ModelAdmin): list_display = ['id', 'get_dataset_name', 'index'] def get_dataset_name(self, obj): - return obj.file_item.dataset.name + return obj.dataset.name + + +class VectorFeatureAdmin(admin.ModelAdmin): + list_display = ['id', 'get_dataset_name', 'get_map_layer_index'] + + def get_dataset_name(self, obj): + return obj.map_layer.dataset.name + + def get_map_layer_index(self, obj): + return obj.map_layer.index class VectorFeatureAdmin(admin.ModelAdmin): diff --git a/uvdat/core/migrations/0003_fileitem_optional.py b/uvdat/core/migrations/0003_fileitem_optional.py new file mode 100644 index 00000000..bb5b74ee --- /dev/null +++ b/uvdat/core/migrations/0003_fileitem_optional.py @@ -0,0 +1,36 @@ +# Generated by Django 4.1 on 2024-06-24 20:13 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0002_vector_features'), + ] + + operations = [ + migrations.RemoveField( + model_name='rastermaplayer', + name='file_item', + ), + migrations.RemoveField( + model_name='vectormaplayer', + name='file_item', + ), + migrations.AddField( + model_name='rastermaplayer', + name='dataset', + field=models.ForeignKey( + null=True, on_delete=django.db.models.deletion.CASCADE, to='core.dataset' + ), + ), + migrations.AddField( + model_name='vectormaplayer', + name='dataset', + field=models.ForeignKey( + null=True, on_delete=django.db.models.deletion.CASCADE, to='core.dataset' + ), + ), + ] diff --git a/uvdat/core/models/dataset.py b/uvdat/core/models/dataset.py index 81e0f8bd..81e12f55 100644 --- a/uvdat/core/models/dataset.py +++ b/uvdat/core/models/dataset.py @@ -84,8 +84,8 @@ def get_map_layers(self): from uvdat.core.models import RasterMapLayer, VectorMapLayer if self.dataset_type == self.DatasetType.RASTER: - return RasterMapLayer.objects.filter(file_item__dataset=self) + return RasterMapLayer.objects.filter(dataset=self) if self.dataset_type == self.DatasetType.VECTOR: - return VectorMapLayer.objects.filter(file_item__dataset=self) + return VectorMapLayer.objects.filter(dataset=self) raise NotImplementedError(f'Dataset Type {self.dataset_type}') diff --git a/uvdat/core/models/map_layers.py b/uvdat/core/models/map_layers.py index f90ba7a2..34cb0bf8 100644 --- a/uvdat/core/models/map_layers.py +++ b/uvdat/core/models/map_layers.py @@ -9,17 +9,17 @@ import large_image from s3_file_field import S3FileField -from .file_item import FileItem +from .dataset import Dataset class AbstractMapLayer(TimeStampedModel): - file_item = models.ForeignKey(FileItem, on_delete=models.CASCADE, null=True) + dataset = models.ForeignKey(Dataset, on_delete=models.CASCADE, null=True) metadata = models.JSONField(blank=True, null=True) default_style = models.JSONField(blank=True, null=True) index = models.IntegerField(null=True) def is_in_context(self, context_id): - return self.file_item.is_in_context(context_id) + return self.dataset.is_in_context(context_id) class Meta: abstract = True diff --git a/uvdat/core/models/simulations.py b/uvdat/core/models/simulations.py index 2d9bee05..2cec181c 100644 --- a/uvdat/core/models/simulations.py +++ b/uvdat/core/models/simulations.py @@ -69,12 +69,12 @@ def run(self, **kwargs): { 'name': 'elevation_data', 'type': RasterMapLayer, - 'options_query': {'file_item__dataset__category': 'elevation'}, + 'options_query': {'dataset__category': 'elevation'}, }, { 'name': 'flood_area', 'type': VectorMapLayer, - 'options_query': {'file_item__dataset__category': 'flood'}, + 'options_query': {'dataset__category': 'flood'}, }, ], }, diff --git a/uvdat/core/rest/dataset.py b/uvdat/core/rest/dataset.py index a354f775..17512825 100644 --- a/uvdat/core/rest/dataset.py +++ b/uvdat/core/rest/dataset.py @@ -23,7 +23,7 @@ def get_queryset(self): @action(detail=True, methods=['get']) def map_layers(self, request, **kwargs): dataset: Dataset = self.get_object() - map_layers = list(dataset.get_map_layers().select_related('file_item__dataset')) + map_layers = list(dataset.get_map_layers().select_related('dataset')) # Set serializer based on dataset type if dataset.dataset_type == Dataset.DatasetType.RASTER: diff --git a/uvdat/core/rest/map_layers.py b/uvdat/core/rest/map_layers.py index a9ed6289..fb56fcff 100644 --- a/uvdat/core/rest/map_layers.py +++ b/uvdat/core/rest/map_layers.py @@ -74,7 +74,7 @@ class RasterMapLayerViewSet(ModelViewSet, LargeImageFileDetailMixin): - queryset = RasterMapLayer.objects.select_related('file_item__dataset').all() + queryset = RasterMapLayer.objects.select_related('dataset').all() serializer_class = RasterMapLayerSerializer FILE_FIELD_NAME = 'cloud_optimized_geotiff' @@ -91,7 +91,7 @@ def get_raster_data(self, request, resolution: str = '1', **kwargs): class VectorMapLayerViewSet(ModelViewSet): - queryset = VectorMapLayer.objects.select_related('file_item__dataset').all() + queryset = VectorMapLayer.objects.select_related('dataset').all() serializer_class = VectorMapLayerSerializer def retrieve(self, request, *args, **kwargs): diff --git a/uvdat/core/rest/serializers.py b/uvdat/core/rest/serializers.py index d015473c..2641f7bd 100644 --- a/uvdat/core/rest/serializers.py +++ b/uvdat/core/rest/serializers.py @@ -56,9 +56,12 @@ class AbstractMapLayerSerializer(serializers.Serializer): file_item = serializers.SerializerMethodField('get_file_item') def get_name(self, obj: VectorMapLayer | RasterMapLayer): - if obj.file_item is None: - return None - return obj.file_item.name + if obj.dataset: + for file_item in obj.dataset.source_files.all(): + if file_item.index == obj.index: + return file_item.name + return f'{obj.dataset.name} Layer {obj.index}' + return None def get_type(self, obj: VectorMapLayer | RasterMapLayer): if isinstance(obj, VectorMapLayer): @@ -66,17 +69,19 @@ def get_type(self, obj: VectorMapLayer | RasterMapLayer): return 'raster' def get_dataset_id(self, obj: VectorMapLayer | RasterMapLayer): - if obj.file_item and obj.file_item.dataset: - return obj.file_item.dataset.id + if obj.dataset: + return obj.dataset.id return None def get_file_item(self, obj: VectorMapLayer | RasterMapLayer): - if obj.file_item is None: + if obj.dataset is None: return None - return { - 'id': obj.file_item.id, - 'name': obj.file_item.name, - } + for file_item in obj.dataset.source_files.all(): + if file_item.index == obj.index: + return { + 'id': file_item.id, + 'name': file_item.name, + } class RasterMapLayerSerializer(serializers.ModelSerializer, AbstractMapLayerSerializer): diff --git a/uvdat/core/tasks/dataset.py b/uvdat/core/tasks/dataset.py index 65d932c0..08bb50a4 100644 --- a/uvdat/core/tasks/dataset.py +++ b/uvdat/core/tasks/dataset.py @@ -32,7 +32,7 @@ def convert_dataset( region = dataset.classification == Dataset.Classification.REGION if dataset.dataset_type == dataset.DatasetType.RASTER: - RasterMapLayer.objects.filter(file_item__dataset=dataset).delete() + RasterMapLayer.objects.filter(dataset=dataset).delete() for file_to_convert in FileItem.objects.filter(dataset=dataset): create_raster_map_layer( file_to_convert, @@ -40,7 +40,7 @@ def convert_dataset( ) elif dataset.dataset_type == dataset.DatasetType.VECTOR: - VectorMapLayer.objects.filter(file_item__dataset=dataset).delete() + VectorMapLayer.objects.filter(dataset=dataset).delete() if network: NetworkNode.objects.filter(dataset=dataset).delete() NetworkEdge.objects.filter(dataset=dataset).delete() diff --git a/uvdat/core/tasks/map_layers.py b/uvdat/core/tasks/map_layers.py index e70c7493..31aa342f 100644 --- a/uvdat/core/tasks/map_layers.py +++ b/uvdat/core/tasks/map_layers.py @@ -51,7 +51,7 @@ def create_raster_map_layer(file_item, style_options): import large_image_converter new_map_layer = RasterMapLayer.objects.create( - file_item=file_item, + dataset=file_item.dataset, metadata={}, default_style=style_options, index=file_item.index, @@ -124,7 +124,7 @@ def create_raster_map_layer(file_item, style_options): def create_vector_map_layer(file_item, style_options): """Save a VectorMapLayer from a FileItem's contents.""" new_map_layer = VectorMapLayer.objects.create( - file_item=file_item, + dataset=file_item.dataset, metadata={}, default_style=style_options, index=file_item.index, diff --git a/uvdat/core/tasks/networks.py b/uvdat/core/tasks/networks.py index c27c39e8..3517854c 100644 --- a/uvdat/core/tasks/networks.py +++ b/uvdat/core/tasks/networks.py @@ -77,12 +77,12 @@ def create_network(vector_map_layer, network_options): try: from_node_obj = NetworkNode.objects.get( - dataset=vector_map_layer.file_item.dataset, + dataset=vector_map_layer.dataset, name=current_node_name, ) except NetworkNode.DoesNotExist: from_node_obj = NetworkNode.objects.create( - dataset=vector_map_layer.file_item.dataset, + dataset=vector_map_layer.dataset, name=current_node_name, location=Point( current_node_coordinates.x, @@ -106,12 +106,12 @@ def create_network(vector_map_layer, network_options): try: to_node_obj = NetworkNode.objects.get( - dataset=vector_map_layer.file_item.dataset, + dataset=vector_map_layer.dataset, name=next_node_name, ) except NetworkNode.DoesNotExist: to_node_obj = NetworkNode.objects.create( - dataset=vector_map_layer.file_item.dataset, + dataset=vector_map_layer.dataset, name=next_node_name, location=Point( next_node_coordinates.x, @@ -139,7 +139,7 @@ def create_network(vector_map_layer, network_options): try: NetworkEdge.objects.get( - dataset=vector_map_layer.file_item.dataset, + dataset=vector_map_layer.dataset, name=f'{current_node_name} - {next_node_name}', ) except NetworkEdge.DoesNotExist: @@ -153,7 +153,7 @@ def create_network(vector_map_layer, network_options): ) ) NetworkEdge.objects.create( - dataset=vector_map_layer.file_item.dataset, + dataset=vector_map_layer.dataset, name=f'{current_node_name} - {next_node_name}', from_node=from_node_obj, to_node=to_node_obj, @@ -161,7 +161,7 @@ def create_network(vector_map_layer, network_options): metadata=metadata, ) # rewrite vector_map_layer geojson_data with updated features - vector_map_layer.write_geojson_data(geojson_from_network(vector_map_layer.file_item.dataset)) + vector_map_layer.write_geojson_data(geojson_from_network(vector_map_layer.dataset)) vector_map_layer.metadata['network'] = True vector_map_layer.save() diff --git a/uvdat/core/tasks/regions.py b/uvdat/core/tasks/regions.py index 2af73b3b..26031109 100644 --- a/uvdat/core/tasks/regions.py +++ b/uvdat/core/tasks/regions.py @@ -101,7 +101,7 @@ def create_source_regions(vector_map_layer, region_options): name=name, boundary=GEOSGeometry(str(geometry)), metadata=properties, - dataset=vector_map_layer.file_item.dataset, + dataset=vector_map_layer.dataset, ) region.save() region_count += 1 diff --git a/uvdat/core/tasks/simulations.py b/uvdat/core/tasks/simulations.py index e926206d..28726c1c 100644 --- a/uvdat/core/tasks/simulations.py +++ b/uvdat/core/tasks/simulations.py @@ -58,8 +58,8 @@ def flood_scenario_1(simulation_result_id, network_dataset, elevation_data, floo if ( not network_dataset.get_network() - or elevation_data.file_item.dataset.category != 'elevation' - or flood_area.file_item.dataset.category != 'flood' + or elevation_data.dataset.category != 'elevation' + or flood_area.dataset.category != 'flood' ): result.error_message = 'Invalid dataset selected.' result.save() diff --git a/web/src/components/OptionsDrawerContents.vue b/web/src/components/OptionsDrawerContents.vue index e3b8bb72..61bab760 100644 --- a/web/src/components/OptionsDrawerContents.vue +++ b/web/src/components/OptionsDrawerContents.vue @@ -26,8 +26,8 @@ export default { const applyToAll = ref(false); const zIndex = ref(); - const currentFileItemName = computed(() => { - return currentMapLayer.value?.file_item?.name; + const currentLayerName = computed(() => { + return currentMapLayer.value?.name; }); function getCurrentMapLayer() { @@ -150,7 +150,7 @@ export default { currentMapLayerIndex, currentMapLayer, currentDataset, - currentFileItemName, + currentLayerName, rasterColormaps, opacity, colormap, @@ -202,7 +202,7 @@ export default { :max="currentDataset?.map_layers.length - 1" /> - Current layer file name: {{ currentFileItemName || "Untitled" }} + Current layer name: {{ currentLayerName || "Untitled" }} diff --git a/web/src/components/SimulationsPanel.vue b/web/src/components/SimulationsPanel.vue index 8144db4c..73d9ea93 100644 --- a/web/src/components/SimulationsPanel.vue +++ b/web/src/components/SimulationsPanel.vue @@ -96,7 +96,7 @@ export default { | VectorMapLayer | RasterMapLayer | undefined; - } else if (selectedOption.file_item && selectedOption.type) { + } else if (selectedOption.index && selectedOption.type) { // Object is layer mapLayer = await getOrCreateLayerFromID( selectedOption.id, diff --git a/web/src/types.ts b/web/src/types.ts index ffce2160..43e9fc11 100644 --- a/web/src/types.ts +++ b/web/src/types.ts @@ -84,6 +84,7 @@ export interface NetworkEdge { export interface AbstractMapLayer { id: number; + name: string; file_item?: { id: number; name: string; From 6e0e132d5024518e99c462df4aac286f3c8f68d0 Mon Sep 17 00:00:00 2001 From: Anne Haley Date: Mon, 8 Jul 2024 23:57:13 +0000 Subject: [PATCH 02/10] Add Network model; allow multiple networks on a single dataset --- sample_data/datasets.json | 5 - sample_data/ingest_sample_data.py | 12 --- uvdat/core/admin.py | 23 ++++- .../core/migrations/0003_fileitem_optional.py | 36 -------- .../migrations/0004_files_and_networks.py | 91 +++++++++++++++++++ uvdat/core/models/__init__.py | 3 +- uvdat/core/models/dataset.py | 29 ------ uvdat/core/models/networks.py | 29 +++++- uvdat/core/models/simulations.py | 16 ++-- uvdat/core/rest/dataset.py | 28 ++++-- uvdat/core/rest/serializers.py | 12 +++ uvdat/core/tasks/chart.py | 5 +- uvdat/core/tasks/dataset.py | 12 +-- uvdat/core/tasks/networks.py | 49 +++++----- uvdat/core/tasks/osmnx.py | 26 +++--- uvdat/core/tasks/simulations.py | 33 +++---- uvdat/core/tests/test_load_roads.py | 27 ++++++ uvdat/core/tests/test_populate.py | 4 +- web/src/components/NodeAnimation.vue | 4 +- web/src/components/SimulationsPanel.vue | 2 +- web/src/types.ts | 1 - 21 files changed, 267 insertions(+), 180 deletions(-) delete mode 100644 uvdat/core/migrations/0003_fileitem_optional.py create mode 100644 uvdat/core/migrations/0004_files_and_networks.py create mode 100644 uvdat/core/tests/test_load_roads.py diff --git a/sample_data/datasets.json b/sample_data/datasets.json index 7dd34202..c39782f5 100644 --- a/sample_data/datasets.json +++ b/sample_data/datasets.json @@ -32,11 +32,6 @@ "path": "boston/commuter_rail.zip" } ], - "network_options": { - "connection_column": "LINE_BRNCH", - "connection_column_delimiter": "/", - "node_id_column": "STATION" - }, "style_options": { "color_property": "gray", "outline": "black" diff --git a/sample_data/ingest_sample_data.py b/sample_data/ingest_sample_data.py index 18c4108d..fb6f11b6 100644 --- a/sample_data/ingest_sample_data.py +++ b/sample_data/ingest_sample_data.py @@ -105,23 +105,12 @@ def ingest_datasets(include_large=False, dataset_indexes=None): with open('sample_data/datasets.json') as datasets_json: data = json.load(datasets_json) for index, dataset in enumerate(data): - # Grab fields specific to dataset classification - network_options = dataset.get('network_options') - region_options = dataset.get('region_options') - if dataset_indexes is None or index in dataset_indexes: print('\t- ', dataset['name']) existing = Dataset.objects.filter(name=dataset['name']) if existing.count(): dataset_for_conversion = existing.first() else: - # Determine classification - classification = Dataset.Classification.OTHER - if network_options: - classification = Dataset.Classification.NETWORK - elif region_options: - classification = Dataset.Classification.REGION - # Create dataset new_dataset = Dataset.objects.create( name=dataset['name'], @@ -129,7 +118,6 @@ def ingest_datasets(include_large=False, dataset_indexes=None): category=dataset['category'], dataset_type=dataset.get('type', 'vector').upper(), metadata=dataset.get('metadata', {}), - classification=classification, ) print('\t', f'Dataset {new_dataset.name} created.') for index, file_info in enumerate(dataset.get('files', [])): diff --git a/uvdat/core/admin.py b/uvdat/core/admin.py index 0276cc7e..625a6619 100644 --- a/uvdat/core/admin.py +++ b/uvdat/core/admin.py @@ -6,6 +6,7 @@ Dataset, DerivedRegion, FileItem, + Network, NetworkEdge, NetworkNode, RasterMapLayer, @@ -84,18 +85,31 @@ def get_source_region_names(self, obj): return ', '.join(r.name for r in obj.source_regions.all()) -class NetworkEdgeAdmin(admin.ModelAdmin): - list_display = ['id', 'name', 'get_dataset_name'] +class NetworkAdmin(admin.ModelAdmin): + list_display = ['id', 'category', 'get_dataset_name'] def get_dataset_name(self, obj): return obj.dataset.name +class NetworkEdgeAdmin(admin.ModelAdmin): + list_display = ['id', 'name', 'get_network_id', 'get_dataset_name'] + + def get_network_id(self, obj): + return obj.network.id + + def get_dataset_name(self, obj): + return obj.network.dataset.name + + class NetworkNodeAdmin(admin.ModelAdmin): - list_display = ['id', 'name', 'get_dataset_name', 'get_adjacent_node_names'] + list_display = ['id', 'name', 'get_network_id', 'get_dataset_name', 'get_adjacent_node_names'] + + def get_network_id(self, obj): + return obj.network.id def get_dataset_name(self, obj): - return obj.dataset.name + return obj.network.dataset.name def get_adjacent_node_names(self, obj): return ', '.join(n.name for n in obj.get_adjacent_nodes()) @@ -114,6 +128,7 @@ class SimulationResultAdmin(admin.ModelAdmin): admin.site.register(VectorFeature, VectorFeatureAdmin) admin.site.register(SourceRegion, SourceRegionAdmin) admin.site.register(DerivedRegion, DerivedRegionAdmin) +admin.site.register(Network, NetworkAdmin) admin.site.register(NetworkNode, NetworkNodeAdmin) admin.site.register(NetworkEdge, NetworkEdgeAdmin) admin.site.register(SimulationResult, SimulationResultAdmin) diff --git a/uvdat/core/migrations/0003_fileitem_optional.py b/uvdat/core/migrations/0003_fileitem_optional.py deleted file mode 100644 index bb5b74ee..00000000 --- a/uvdat/core/migrations/0003_fileitem_optional.py +++ /dev/null @@ -1,36 +0,0 @@ -# Generated by Django 4.1 on 2024-06-24 20:13 - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('core', '0002_vector_features'), - ] - - operations = [ - migrations.RemoveField( - model_name='rastermaplayer', - name='file_item', - ), - migrations.RemoveField( - model_name='vectormaplayer', - name='file_item', - ), - migrations.AddField( - model_name='rastermaplayer', - name='dataset', - field=models.ForeignKey( - null=True, on_delete=django.db.models.deletion.CASCADE, to='core.dataset' - ), - ), - migrations.AddField( - model_name='vectormaplayer', - name='dataset', - field=models.ForeignKey( - null=True, on_delete=django.db.models.deletion.CASCADE, to='core.dataset' - ), - ), - ] diff --git a/uvdat/core/migrations/0004_files_and_networks.py b/uvdat/core/migrations/0004_files_and_networks.py new file mode 100644 index 00000000..2cf90181 --- /dev/null +++ b/uvdat/core/migrations/0004_files_and_networks.py @@ -0,0 +1,91 @@ +# Generated by Django 4.1 on 2024-07-08 20:01 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0003_nonunique_network_names'), + ] + + operations = [ + migrations.RemoveField( + model_name='dataset', + name='classification', + ), + migrations.RemoveField( + model_name='networkedge', + name='dataset', + ), + migrations.RemoveField( + model_name='networknode', + name='dataset', + ), + migrations.RemoveField( + model_name='rastermaplayer', + name='file_item', + ), + migrations.RemoveField( + model_name='vectormaplayer', + name='file_item', + ), + migrations.AddField( + model_name='rastermaplayer', + name='dataset', + field=models.ForeignKey( + null=True, on_delete=django.db.models.deletion.CASCADE, to='core.dataset' + ), + ), + migrations.AddField( + model_name='vectormaplayer', + name='dataset', + field=models.ForeignKey( + null=True, on_delete=django.db.models.deletion.CASCADE, to='core.dataset' + ), + ), + migrations.CreateModel( + name='Network', + fields=[ + ( + 'id', + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name='ID' + ), + ), + ('category', models.CharField(max_length=25)), + ('metadata', models.JSONField(blank=True, null=True)), + ( + 'dataset', + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name='networks', + to='core.dataset', + ), + ), + ], + ), + migrations.AddField( + model_name='networkedge', + name='network', + field=models.ForeignKey( + default=None, + on_delete=django.db.models.deletion.CASCADE, + related_name='edges', + to='core.network', + ), + preserve_default=False, + ), + migrations.AddField( + model_name='networknode', + name='network', + field=models.ForeignKey( + default=None, + on_delete=django.db.models.deletion.CASCADE, + related_name='nodes', + to='core.network', + ), + preserve_default=False, + ), + ] diff --git a/uvdat/core/models/__init__.py b/uvdat/core/models/__init__.py index 1e5a3232..88a0b109 100644 --- a/uvdat/core/models/__init__.py +++ b/uvdat/core/models/__init__.py @@ -3,7 +3,7 @@ from .dataset import Dataset from .file_item import FileItem from .map_layers import RasterMapLayer, VectorFeature, VectorMapLayer -from .networks import NetworkEdge, NetworkNode +from .networks import Network, NetworkEdge, NetworkNode from .regions import DerivedRegion, SourceRegion from .simulations import SimulationResult @@ -17,6 +17,7 @@ VectorFeature, SourceRegion, DerivedRegion, + Network, NetworkEdge, NetworkNode, SimulationResult, diff --git a/uvdat/core/models/dataset.py b/uvdat/core/models/dataset.py index 81e12f55..19819c6b 100644 --- a/uvdat/core/models/dataset.py +++ b/uvdat/core/models/dataset.py @@ -6,11 +6,6 @@ class DatasetType(models.TextChoices): VECTOR = 'VECTOR', 'Vector' RASTER = 'RASTER', 'Raster' - class Classification(models.TextChoices): - NETWORK = 'Network' - REGION = 'Region' - OTHER = 'Other' - name = models.CharField(max_length=255, unique=True) description = models.TextField(null=True, blank=True) category = models.CharField(max_length=25) @@ -20,9 +15,6 @@ class Classification(models.TextChoices): max_length=max(len(choice[0]) for choice in DatasetType.choices), choices=DatasetType.choices, ) - classification = models.CharField( - max_length=16, choices=Classification.choices, default=Classification.OTHER - ) def is_in_context(self, context_id): from uvdat.core.models import Context @@ -58,27 +50,6 @@ def get_regions(self): return SourceRegion.objects.filter(dataset=self) - def get_network(self): - from uvdat.core.models import NetworkEdge, NetworkNode - - network = { - 'nodes': NetworkNode.objects.filter(dataset=self), - 'edges': NetworkEdge.objects.filter(dataset=self), - } - if len(network.get('nodes')) == 0 and len(network.get('edges')) == 0: - return None - return network - - def get_network_graph(self): - from uvdat.core.tasks.networks import get_dataset_network_graph - - return get_dataset_network_graph(self) - - def get_network_gcc(self, exclude_nodes): - from uvdat.core.tasks.networks import get_dataset_network_gcc - - return get_dataset_network_gcc(self, exclude_nodes) - def get_map_layers(self): """Return a queryset of either RasterMapLayer, or VectorMapLayer.""" from uvdat.core.models import RasterMapLayer, VectorMapLayer diff --git a/uvdat/core/models/networks.py b/uvdat/core/models/networks.py index a7972cae..8f4ef129 100644 --- a/uvdat/core/models/networks.py +++ b/uvdat/core/models/networks.py @@ -1,18 +1,39 @@ +import networkx as nx from django.contrib.gis.db import models as geo_models from django.db import models from .dataset import Dataset +class Network(models.Model): + dataset = models.ForeignKey(Dataset, on_delete=models.CASCADE, related_name='networks') + category = models.CharField(max_length=25) + metadata = models.JSONField(blank=True, null=True) + + def is_in_context(self, context_id): + return self.dataset.is_in_context(context_id) + + def get_graph(self): + from uvdat.core.tasks.networks import get_network_graph + + return get_network_graph(self) + + def get_gcc(self, exclude_nodes): + graph = self.get_graph() + graph.remove_nodes_from(exclude_nodes) + gcc = max(nx.connected_components(graph), key=len) + return list(gcc) + + class NetworkNode(models.Model): name = models.CharField(max_length=255) - dataset = models.ForeignKey(Dataset, on_delete=models.CASCADE, related_name='network_nodes') + network = models.ForeignKey(Network, on_delete=models.CASCADE, related_name='nodes') metadata = models.JSONField(blank=True, null=True) capacity = models.IntegerField(null=True) location = geo_models.PointField() def is_in_context(self, context_id): - return self.dataset.is_in_context(context_id) + return self.network.is_in_context(context_id) def get_adjacent_nodes(self) -> models.QuerySet: entering_node_ids = ( @@ -32,7 +53,7 @@ def get_adjacent_nodes(self) -> models.QuerySet: class NetworkEdge(models.Model): name = models.CharField(max_length=255) - dataset = models.ForeignKey(Dataset, on_delete=models.CASCADE, related_name='network_edges') + network = models.ForeignKey(Network, on_delete=models.CASCADE, related_name='edges') metadata = models.JSONField(blank=True, null=True) capacity = models.IntegerField(null=True) line_geometry = geo_models.LineStringField() @@ -41,4 +62,4 @@ class NetworkEdge(models.Model): to_node = models.ForeignKey(NetworkNode, related_name='+', on_delete=models.CASCADE) def is_in_context(self, context_id): - return self.dataset.is_in_context(context_id) + return self.network.is_in_context(context_id) diff --git a/uvdat/core/models/simulations.py b/uvdat/core/models/simulations.py index 2cec181c..e6bf4e12 100644 --- a/uvdat/core/models/simulations.py +++ b/uvdat/core/models/simulations.py @@ -4,8 +4,8 @@ from uvdat.core.tasks import simulations as uvdat_simulations from .context import Context -from .dataset import Dataset from .map_layers import RasterMapLayer, VectorMapLayer +from .networks import Network class SimulationResult(TimeStampedModel): @@ -47,7 +47,7 @@ def run(self, **kwargs): AVAILABLE_SIMULATIONS = { 'FLOOD_1': { 'description': """ - Provide a network dataset, elevation dataset, and flood dataset + Provide a network, elevation dataset, and flood dataset to determine which network nodes go out of service when the target flood occurs. """, @@ -55,15 +55,15 @@ def run(self, **kwargs): 'func': uvdat_simulations.flood_scenario_1, 'args': [ { - 'name': 'network_dataset', - 'type': Dataset, + 'name': 'network', + 'type': Network, 'options_annotations': { - 'network_nodes_count': models.Count('network_nodes'), - 'network_edges_count': models.Count('network_edges'), + 'nodes_count': models.Count('nodes'), + 'edges_count': models.Count('edges'), }, 'options_query': { - 'network_nodes_count__gte': 1, - 'network_edges_count__gte': 1, + 'nodes_count__gte': 1, + 'edges_count__gte': 1, }, }, { diff --git a/uvdat/core/rest/dataset.py b/uvdat/core/rest/dataset.py index 17512825..80a06194 100644 --- a/uvdat/core/rest/dataset.py +++ b/uvdat/core/rest/dataset.py @@ -5,7 +5,7 @@ from rest_framework.response import Response from rest_framework.viewsets import ModelViewSet -from uvdat.core.models import Dataset +from uvdat.core.models import Dataset, NetworkEdge, NetworkNode from uvdat.core.rest import serializers as uvdat_serializers from uvdat.core.tasks.chart import add_gcc_chart_datum @@ -46,17 +46,16 @@ def convert(self, request, **kwargs): @action(detail=True, methods=['get']) def network(self, request, **kwargs): dataset = self.get_object() - network = dataset.get_network() return HttpResponse( json.dumps( { 'nodes': [ uvdat_serializers.NetworkNodeSerializer(n).data - for n in network.get('nodes') + for n in NetworkNode.objects.filter(network__dataset=dataset) ], 'edges': [ uvdat_serializers.NetworkEdgeSerializer(e).data - for e in network.get('edges') + for e in NetworkEdge.objects.filter(network__dataset=dataset) ], } ), @@ -70,10 +69,19 @@ def gcc(self, request, **kwargs): exclude_nodes = request.query_params.get('exclude_nodes', []) exclude_nodes = exclude_nodes.split(',') exclude_nodes = [int(n) for n in exclude_nodes if len(n)] - excluded_node_names = [ - n.name for n in dataset.get_network().get('nodes') if n.id in exclude_nodes - ] - gcc = dataset.get_network_gcc(exclude_nodes) - add_gcc_chart_datum(dataset, context_id, excluded_node_names, len(gcc)) - return HttpResponse(json.dumps(gcc), status=200) + # TODO: improve this for datasets with multiple networks; + # this currently returns the gcc for the network with the most excluded nodes + results = [] + for network in dataset.networks.all(): + excluded_node_names = [ + n.name for n in network.nodes.all() if n.id in exclude_nodes + ] + gcc = network.get_gcc(exclude_nodes) + results.append(dict(excluded=excluded_node_names, gcc=gcc)) + if len(results): + results.sort(key=lambda r: len(r.get('excluded')), reverse=True) + gcc = results[0].get('gcc') + excluded = results[0].get('excluded') + add_gcc_chart_datum(dataset, context_id, excluded, len(gcc)) + return HttpResponse(json.dumps(gcc), status=200) diff --git a/uvdat/core/rest/serializers.py b/uvdat/core/rest/serializers.py index 2641f7bd..a4d33bb8 100644 --- a/uvdat/core/rest/serializers.py +++ b/uvdat/core/rest/serializers.py @@ -9,6 +9,7 @@ Dataset, DerivedRegion, FileItem, + Network, NetworkEdge, NetworkNode, RasterMapLayer, @@ -173,6 +174,17 @@ class Meta: operation = serializers.ChoiceField(choices=DerivedRegion.VectorOperation.choices) +class NetworkSerializer(serializers.ModelSerializer): + class Meta: + model = Network + fields = '__all__' + + name = serializers.SerializerMethodField('get_name') + + def get_name(self, obj): + return obj.dataset.name + + class NetworkNodeSerializer(serializers.ModelSerializer): class Meta: model = NetworkNode diff --git a/uvdat/core/tasks/chart.py b/uvdat/core/tasks/chart.py index 8804874b..536c6292 100644 --- a/uvdat/core/tasks/chart.py +++ b/uvdat/core/tasks/chart.py @@ -45,6 +45,9 @@ def convert_chart(chart_id, conversion_options): def get_gcc_chart(dataset, context_id): chart_name = f'{dataset.name} Greatest Connected Component Sizes' + max_nodes = 0 + for network in dataset.networks.all(): + max_nodes += network.nodes.count() try: return Chart.objects.get(name=chart_name) except Chart.DoesNotExist: @@ -63,7 +66,7 @@ def get_gcc_chart(dataset, context_id): 'chart_title': 'Size of Greatest Connected Component over Period', 'x_title': 'Step when Excluded Nodes Changed', 'y_title': 'Number of Nodes', - 'y_range': [0, dataset.network_nodes.count()], + 'y_range': [0, max_nodes], }, ) print('\t', f'Chart {chart.name} created.') diff --git a/uvdat/core/tasks/dataset.py b/uvdat/core/tasks/dataset.py index 08bb50a4..493b6407 100644 --- a/uvdat/core/tasks/dataset.py +++ b/uvdat/core/tasks/dataset.py @@ -27,10 +27,6 @@ def convert_dataset( dataset.processing = True dataset.save() - # Determine network/region classificaton - network = dataset.classification == Dataset.Classification.NETWORK - region = dataset.classification == Dataset.Classification.REGION - if dataset.dataset_type == dataset.DatasetType.RASTER: RasterMapLayer.objects.filter(dataset=dataset).delete() for file_to_convert in FileItem.objects.filter(dataset=dataset): @@ -41,18 +37,14 @@ def convert_dataset( elif dataset.dataset_type == dataset.DatasetType.VECTOR: VectorMapLayer.objects.filter(dataset=dataset).delete() - if network: - NetworkNode.objects.filter(dataset=dataset).delete() - NetworkEdge.objects.filter(dataset=dataset).delete() - elif region: - SourceRegion.objects.filter(dataset=dataset).delete() + SourceRegion.objects.filter(dataset=dataset).delete() for file_to_convert in FileItem.objects.filter(dataset=dataset): vector_map_layer = create_vector_map_layer( file_to_convert, style_options=style_options, ) - if network: + if network_options: create_network(vector_map_layer, network_options) elif region_options: create_source_regions(vector_map_layer, region_options) diff --git a/uvdat/core/tasks/networks.py b/uvdat/core/tasks/networks.py index 3517854c..b742d2ce 100644 --- a/uvdat/core/tasks/networks.py +++ b/uvdat/core/tasks/networks.py @@ -6,7 +6,7 @@ import numpy import shapely -from uvdat.core.models import NetworkEdge, NetworkNode +from uvdat.core.models import Network, NetworkEdge, NetworkNode NODE_RECOVERY_MODES = [ 'random', @@ -21,6 +21,13 @@ def create_network(vector_map_layer, network_options): + # Overwrite previous results + dataset = vector_map_layer.dataset + Network.objects.filter(dataset=dataset).delete() + network = Network.objects.create( + dataset=dataset, category=dataset.category, metadata={'source': 'Parsed from GeoJSON.'} + ) + connection_column = network_options.get('connection_column') connection_column_delimiter = network_options.get('connection_column_delimiter') node_id_column = network_options.get('node_id_column') @@ -77,12 +84,12 @@ def create_network(vector_map_layer, network_options): try: from_node_obj = NetworkNode.objects.get( - dataset=vector_map_layer.dataset, + network=network, name=current_node_name, ) except NetworkNode.DoesNotExist: from_node_obj = NetworkNode.objects.create( - dataset=vector_map_layer.dataset, + network=network, name=current_node_name, location=Point( current_node_coordinates.x, @@ -106,12 +113,12 @@ def create_network(vector_map_layer, network_options): try: to_node_obj = NetworkNode.objects.get( - dataset=vector_map_layer.dataset, + network=network, name=next_node_name, ) except NetworkNode.DoesNotExist: to_node_obj = NetworkNode.objects.create( - dataset=vector_map_layer.dataset, + network=network, name=next_node_name, location=Point( next_node_coordinates.x, @@ -139,7 +146,7 @@ def create_network(vector_map_layer, network_options): try: NetworkEdge.objects.get( - dataset=vector_map_layer.dataset, + network=network, name=f'{current_node_name} - {next_node_name}', ) except NetworkEdge.DoesNotExist: @@ -153,7 +160,7 @@ def create_network(vector_map_layer, network_options): ) ) NetworkEdge.objects.create( - dataset=vector_map_layer.dataset, + network=network, name=f'{current_node_name} - {next_node_name}', from_node=from_node_obj, to_node=to_node_obj, @@ -170,7 +177,7 @@ def geojson_from_network(dataset): total_nodes = 0 total_edges = 0 new_feature_set = [] - for n in NetworkNode.objects.filter(dataset=dataset): + for n in NetworkNode.objects.filter(network__dataset=dataset): node_as_feature = { 'id': n.id, 'type': 'Feature', @@ -183,7 +190,7 @@ def geojson_from_network(dataset): new_feature_set.append(node_as_feature) total_nodes += 1 - for e in NetworkEdge.objects.filter(dataset=dataset): + for e in NetworkEdge.objects.filter(network__dataset=dataset): edge_as_feature = { 'id': e.id, 'type': 'Feature', @@ -206,11 +213,19 @@ def geojson_from_network(dataset): return new_geodata.to_json() -def get_dataset_network_graph(dataset): - db_representation = dataset.get_network() +def get_network_graph(network): + from uvdat.core.models import NetworkEdge, NetworkNode + + network = { + 'nodes': NetworkNode.objects.filter(network=network), + 'edges': NetworkEdge.objects.filter(network=network), + } + if len(network.get('nodes')) == 0 and len(network.get('edges')) == 0: + return None + # Construct adj list edge_list: dict[int, list[int]] = {} - for e in db_representation.get('edges'): + for e in network.get('edges'): if e.from_node.id not in edge_list: edge_list[e.from_node.id] = [] edge_list[e.from_node.id].append(e.to_node.id) @@ -220,16 +235,6 @@ def get_dataset_network_graph(dataset): return graph_representation -def get_dataset_network_gcc(dataset, exclude_nodes: list[int]) -> list[int]: - # Create graph, remove nodes, get GCC - graph = dataset.get_network_graph() - graph.remove_nodes_from(exclude_nodes) - gcc = max(nx.connected_components(graph), key=len) - - # Return GCC's list of nodes - return list(gcc) - - # Authored by Jack Watson # Takes in a second argument, measure, which is a string specifying the centrality # measure to calculate. diff --git a/uvdat/core/tasks/osmnx.py b/uvdat/core/tasks/osmnx.py index e1cf9028..bc660b7e 100644 --- a/uvdat/core/tasks/osmnx.py +++ b/uvdat/core/tasks/osmnx.py @@ -5,7 +5,7 @@ from django.core.files.base import ContentFile import osmnx -from uvdat.core.models import Context, Dataset, FileItem, NetworkEdge, NetworkNode, VectorMapLayer +from uvdat.core.models import Context, Dataset, Network, NetworkEdge, NetworkNode, VectorMapLayer from uvdat.core.tasks.map_layers import save_vector_features from uvdat.core.tasks.networks import geojson_from_network @@ -21,9 +21,7 @@ def get_or_create_road_dataset(context, location): context.datasets.add(dataset) print('Clearing previous results...') - FileItem.objects.filter(dataset=dataset).delete() - NetworkNode.objects.filter(dataset=dataset).delete() - NetworkEdge.objects.filter(dataset=dataset).delete() + Network.objects.filter(dataset=dataset).delete() return dataset @@ -39,6 +37,9 @@ def metadata_for_row(row): def load_roads(context_id, location): context = Context.objects.get(id=context_id) dataset = get_or_create_road_dataset(context, location) + network = Network.objects.create( + dataset=dataset, category="roads", metadata={'source': 'Fetched with OSMnx.'} + ) print(f'Fetching road data for {location}...') roads = osmnx.graph_from_place(location, network_type='drive') @@ -64,21 +65,21 @@ def load_roads(context_id, location): ].iloc[0] start_node, created = NetworkNode.objects.get_or_create( - dataset=dataset, + network=network, name='{:0.5f}/{:0.5f}'.format(*start), location=Point(*start), ) start_node.metadata = metadata_for_row(start_node_data) start_node.save() end_node, created = NetworkNode.objects.get_or_create( - dataset=dataset, + network=network, name='{:0.5f}/{:0.5f}'.format(*end), location=Point(*end), ) end_node.metadata = metadata_for_row(end_node_data) end_node.save() edge, created = NetworkEdge.objects.get_or_create( - dataset=dataset, + network=network, name=edge_name, directed=edge_data['oneway'], from_node=start_node, @@ -88,14 +89,11 @@ def load_roads(context_id, location): edge.metadata = metadata_for_row(edge_data) edge.save() - geojson = geojson_from_network(dataset) - file_item = FileItem.objects.create( - name=f'{location} Roads', dataset=dataset, file_type='geojson' - ) - file_item.file.save('roads.geojson', ContentFile(json.dumps(geojson).encode())) vector_map_layer = VectorMapLayer.objects.create( - file_item=file_item, metadata={'network': True} + dataset=dataset, + metadata=dict(network=True), + index=0, ) - vector_map_layer.write_geojson_data(geojson) + vector_map_layer.write_geojson_data(geojson_from_network(dataset)) save_vector_features(vector_map_layer) print('Done.') diff --git a/uvdat/core/tasks/simulations.py b/uvdat/core/tasks/simulations.py index 28726c1c..13406b41 100644 --- a/uvdat/core/tasks/simulations.py +++ b/uvdat/core/tasks/simulations.py @@ -7,12 +7,7 @@ import large_image import shapely -from uvdat.core.models import Dataset -from uvdat.core.tasks.networks import ( - NODE_RECOVERY_MODES, - get_dataset_network_graph, - sort_graph_centrality, -) +from uvdat.core.tasks.networks import NODE_RECOVERY_MODES, get_network_graph, sort_graph_centrality def get_network_node_elevations(network_nodes, elevation_data): @@ -43,21 +38,21 @@ def get_network_node_elevations(network_nodes, elevation_data): @shared_task -def flood_scenario_1(simulation_result_id, network_dataset, elevation_data, flood_area): - from uvdat.core.models import RasterMapLayer, SimulationResult, VectorMapLayer +def flood_scenario_1(simulation_result_id, network, elevation_data, flood_area): + from uvdat.core.models import Network, RasterMapLayer, SimulationResult, VectorMapLayer result = SimulationResult.objects.get(id=simulation_result_id) try: - network_dataset = Dataset.objects.get(id=network_dataset) + network = Network.objects.get(id=network) elevation_data = RasterMapLayer.objects.get(id=elevation_data) flood_area = VectorMapLayer.objects.get(id=flood_area) - except Dataset.DoesNotExist: - result.error_message = 'Dataset not found.' + except: + result.error_message = 'Object not found.' result.save() return if ( - not network_dataset.get_network() + network.nodes.count() < 1 or elevation_data.dataset.category != 'elevation' or flood_area.dataset.category != 'flood' ): @@ -66,7 +61,7 @@ def flood_scenario_1(simulation_result_id, network_dataset, elevation_data, floo return node_failures = [] - network_nodes = network_dataset.network_nodes.all() + network_nodes = network.nodes.all() flood_geodata = flood_area.read_geojson_data() flood_areas = [ shapely.geometry.shape(feature['geometry']) for feature in flood_geodata['features'] @@ -85,7 +80,7 @@ def flood_scenario_1(simulation_result_id, network_dataset, elevation_data, floo @shared_task def recovery_scenario(simulation_result_id, node_failure_simulation_result, recovery_mode): - from uvdat.core.models import SimulationResult + from uvdat.core.models import Network, SimulationResult result = SimulationResult.objects.get(id=simulation_result_id) try: @@ -106,14 +101,14 @@ def recovery_scenario(simulation_result_id, node_failure_simulation_result, reco if recovery_mode == 'random': random.shuffle(node_recoveries) else: - dataset_id = node_failure_simulation_result.input_args['network_dataset'] + network_id = node_failure_simulation_result.input_args['network'] try: - dataset = Dataset.objects.get(id=dataset_id) - except Dataset.DoesNotExist: - result.error_message = 'Dataset not found.' + network = Network.objects.get(id=network_id) + except Network.DoesNotExist: + result.error_message = 'Network not found.' result.save() return - graph = get_dataset_network_graph(dataset) + graph = get_network_graph(network) nodes_sorted, edge_list = sort_graph_centrality(graph, recovery_mode) node_recoveries.sort(key=lambda n: nodes_sorted.index(n)) diff --git a/uvdat/core/tests/test_load_roads.py b/uvdat/core/tests/test_load_roads.py new file mode 100644 index 00000000..6c2d77d6 --- /dev/null +++ b/uvdat/core/tests/test_load_roads.py @@ -0,0 +1,27 @@ +from django.core.management import call_command +from django.contrib.gis.geos import Point +import pytest + +from uvdat.core.models import Context, Dataset + + +@pytest.mark.django_db +def test_load_roads(): + + context = Context.objects.create( + name='Road Test', + default_map_zoom=10, + default_map_center=Point(42, -71) + ) + + call_command( + 'load_roads', + 'Boston', + context_id=context.id, + ) + + dataset = Dataset.objects.get(name='Boston Road Network') + assert dataset is not None + assert dataset.networks.count() == 1 + assert dataset.networks.first().nodes.count() == 11026 + assert dataset.networks.first().edges.count() == 25274 diff --git a/uvdat/core/tests/test_populate.py b/uvdat/core/tests/test_populate.py index 3b3b8add..479628b8 100644 --- a/uvdat/core/tests/test_populate.py +++ b/uvdat/core/tests/test_populate.py @@ -7,6 +7,7 @@ Dataset, DerivedRegion, FileItem, + Network, NetworkEdge, NetworkNode, RasterMapLayer, @@ -33,10 +34,11 @@ def test_populate(): ) assert Chart.objects.all().count() == 1 - assert Context.objects.all().count() == 3 + assert Context.objects.all().count() == 2 assert Dataset.objects.all().count() == 4 assert DerivedRegion.objects.all().count() == 0 assert FileItem.objects.all().count() == 7 + assert Network.objects.all().count() == 1 assert NetworkEdge.objects.all().count() == 164 assert NetworkNode.objects.all().count() == 158 assert RasterMapLayer.objects.all().count() == 1 diff --git a/web/src/components/NodeAnimation.vue b/web/src/components/NodeAnimation.vue index bd29d21b..4f590c50 100644 --- a/web/src/components/NodeAnimation.vue +++ b/web/src/components/NodeAnimation.vue @@ -39,8 +39,8 @@ export default { }); async function findCurrentNetworkDataset() { - currentNetworkDataset.value = selectedDatasets.value.find( - (d) => d.classification === "Network" + currentNetworkDataset.value = selectedDatasets.value.find((d) => + d.map_layers?.some((l) => l.metadata?.network) ); if (currentNetworkDataset.value && !currentNetworkDataset.value.network) { fetchDatasetNetwork(currentNetworkDataset.value); diff --git a/web/src/components/SimulationsPanel.vue b/web/src/components/SimulationsPanel.vue index 73d9ea93..6c963505 100644 --- a/web/src/components/SimulationsPanel.vue +++ b/web/src/components/SimulationsPanel.vue @@ -155,7 +155,7 @@ export default { watch(activeResult, () => { if (activeResult.value) { populateActiveResultInputs(); - if (!activeResult.value.output_data) { + if (!outputPoll.value && !activeResult.value.output_data) { outputPoll.value = setInterval(pollForActiveResultOutput, 3000); } } diff --git a/web/src/types.ts b/web/src/types.ts index 43e9fc11..d838a4a4 100644 --- a/web/src/types.ts +++ b/web/src/types.ts @@ -10,7 +10,6 @@ export interface Dataset { dataset_type: "vector" | "raster"; map_layers?: (VectorMapLayer | RasterMapLayer)[]; current_layer_index?: number; - classification: "Network" | "Region" | "Other"; network: { nodes: NetworkNode[]; edges: NetworkEdge[]; From 87279bce6a19bec2f4979c7f381913734ad9d655 Mon Sep 17 00:00:00 2001 From: Anne Haley Date: Mon, 29 Jul 2024 13:39:58 +0000 Subject: [PATCH 03/10] fix: Small bug fixes --- uvdat/core/models/networks.py | 2 ++ uvdat/core/rest/chart.py | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/uvdat/core/models/networks.py b/uvdat/core/models/networks.py index 8f4ef129..6a83b1cc 100644 --- a/uvdat/core/models/networks.py +++ b/uvdat/core/models/networks.py @@ -21,6 +21,8 @@ def get_graph(self): def get_gcc(self, exclude_nodes): graph = self.get_graph() graph.remove_nodes_from(exclude_nodes) + if graph.number_of_nodes == 0 or nx.number_connected_components(graph) == 0: + return [] gcc = max(nx.connected_components(graph), key=len) return list(gcc) diff --git a/uvdat/core/rest/chart.py b/uvdat/core/rest/chart.py index 794b3373..a7ec8700 100644 --- a/uvdat/core/rest/chart.py +++ b/uvdat/core/rest/chart.py @@ -12,7 +12,8 @@ class ChartViewSet(GenericViewSet, mixins.ListModelMixin): serializer_class = ChartSerializer def get_queryset(self, **kwargs): - context_id = kwargs.get('context') + request = self.request + context_id = request.query_params.get('context') if context_id: return Chart.objects.filter(context__id=context_id) return Chart.objects.all() From 95c8c8a729538bc515676c8600a35f28a2612dab Mon Sep 17 00:00:00 2001 From: Anne Haley Date: Mon, 29 Jul 2024 18:52:48 +0000 Subject: [PATCH 04/10] test: Update OSMnx test expected number of nodes and edges --- uvdat/core/tests/test_load_roads.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/uvdat/core/tests/test_load_roads.py b/uvdat/core/tests/test_load_roads.py index 6c2d77d6..2d6d57dd 100644 --- a/uvdat/core/tests/test_load_roads.py +++ b/uvdat/core/tests/test_load_roads.py @@ -23,5 +23,7 @@ def test_load_roads(): dataset = Dataset.objects.get(name='Boston Road Network') assert dataset is not None assert dataset.networks.count() == 1 - assert dataset.networks.first().nodes.count() == 11026 - assert dataset.networks.first().edges.count() == 25274 + # check if nodes and edges surpass a minimum amount + # (exact amounts are expected to change over time) + assert dataset.networks.first().nodes.count() > 10000 + assert dataset.networks.first().edges.count() > 20000 From 53e54e5dfdb6c530f8238e7be671afb2dbcc13d0 Mon Sep 17 00:00:00 2001 From: Anne Haley Date: Mon, 29 Jul 2024 19:05:33 +0000 Subject: [PATCH 05/10] feat: Add rest viewsets for file item and network models --- uvdat/core/rest/__init__.py | 6 ++++++ uvdat/core/rest/file_item.py | 9 +++++++++ uvdat/core/rest/network.py | 23 +++++++++++++++++++++++ uvdat/urls.py | 8 ++++++++ 4 files changed, 46 insertions(+) create mode 100644 uvdat/core/rest/file_item.py create mode 100644 uvdat/core/rest/network.py diff --git a/uvdat/core/rest/__init__.py b/uvdat/core/rest/__init__.py index 88706ca7..6bc2c50e 100644 --- a/uvdat/core/rest/__init__.py +++ b/uvdat/core/rest/__init__.py @@ -1,15 +1,21 @@ from .chart import ChartViewSet from .context import ContextViewSet from .dataset import DatasetViewSet +from .file_item import FileItemViewSet from .map_layers import RasterMapLayerViewSet, VectorMapLayerViewSet +from .network import NetworkEdgeViewSet, NetworkNodeViewSet, NetworkViewSet from .regions import DerivedRegionViewSet, SourceRegionViewSet from .simulations import SimulationViewSet __all__ = [ ContextViewSet, ChartViewSet, + FileItemViewSet, RasterMapLayerViewSet, VectorMapLayerViewSet, + NetworkViewSet, + NetworkNodeViewSet, + NetworkEdgeViewSet, DatasetViewSet, SourceRegionViewSet, DerivedRegionViewSet, diff --git a/uvdat/core/rest/file_item.py b/uvdat/core/rest/file_item.py new file mode 100644 index 00000000..1e8e8154 --- /dev/null +++ b/uvdat/core/rest/file_item.py @@ -0,0 +1,9 @@ +from rest_framework.viewsets import ModelViewSet + +from uvdat.core.models import FileItem +from uvdat.core.rest.serializers import FileItemSerializer + + +class FileItemViewSet(ModelViewSet): + queryset = FileItem.objects.all() + serializer_class = FileItemSerializer diff --git a/uvdat/core/rest/network.py b/uvdat/core/rest/network.py new file mode 100644 index 00000000..3e42cbae --- /dev/null +++ b/uvdat/core/rest/network.py @@ -0,0 +1,23 @@ +from rest_framework.viewsets import ModelViewSet + +from uvdat.core.models import Network, NetworkEdge, NetworkNode +from uvdat.core.rest.serializers import ( + NetworkEdgeSerializer, + NetworkNodeSerializer, + NetworkSerializer, +) + + +class NetworkViewSet(ModelViewSet): + queryset = Network.objects.all() + serializer_class = NetworkSerializer + + +class NetworkNodeViewSet(ModelViewSet): + queryset = NetworkNode.objects.all() + serializer_class = NetworkNodeSerializer + + +class NetworkEdgeViewSet(ModelViewSet): + queryset = NetworkEdge.objects.all() + serializer_class = NetworkEdgeSerializer diff --git a/uvdat/urls.py b/uvdat/urls.py index 5be1d17c..06f204ea 100644 --- a/uvdat/urls.py +++ b/uvdat/urls.py @@ -10,6 +10,10 @@ ContextViewSet, DatasetViewSet, DerivedRegionViewSet, + FileItemViewSet, + NetworkEdgeViewSet, + NetworkNodeViewSet, + NetworkViewSet, RasterMapLayerViewSet, SimulationViewSet, SourceRegionViewSet, @@ -26,9 +30,13 @@ router.register(r'contexts', ContextViewSet, basename='contexts') router.register(r'datasets', DatasetViewSet, basename='datasets') +router.register(r'files', FileItemViewSet, basename='files') router.register(r'charts', ChartViewSet, basename='charts') router.register(r'rasters', RasterMapLayerViewSet, basename='rasters') router.register(r'vectors', VectorMapLayerViewSet, basename='vectors') +router.register(r'networks', NetworkViewSet, basename='networks') +router.register(r'nodes', NetworkNodeViewSet, basename='nodes') +router.register(r'edges', NetworkEdgeViewSet, basename='edges') router.register(r'source-regions', SourceRegionViewSet, basename='source-regions') router.register(r'derived-regions', DerivedRegionViewSet, basename='derived-regions') router.register(r'simulations', SimulationViewSet, basename='simulations') From d7c6ef6f6bb2915305a495e649a07cf02456b1cc Mon Sep 17 00:00:00 2001 From: Anne Haley Date: Fri, 2 Aug 2024 15:03:33 +0000 Subject: [PATCH 06/10] style: Lint fixes --- uvdat/core/admin.py | 4 ---- uvdat/core/models/networks.py | 2 +- uvdat/core/rest/dataset.py | 4 +--- uvdat/core/tasks/dataset.py | 10 +--------- uvdat/core/tasks/osmnx.py | 5 +---- uvdat/core/tasks/simulations.py | 2 +- uvdat/core/tests/test_load_roads.py | 6 ++---- 7 files changed, 7 insertions(+), 26 deletions(-) diff --git a/uvdat/core/admin.py b/uvdat/core/admin.py index 625a6619..346aab6a 100644 --- a/uvdat/core/admin.py +++ b/uvdat/core/admin.py @@ -64,10 +64,6 @@ def get_map_layer_index(self, obj): return obj.map_layer.index -class VectorFeatureAdmin(admin.ModelAdmin): - list_display = ['id', 'map_layer'] - - class SourceRegionAdmin(admin.ModelAdmin): list_display = ['id', 'name', 'get_dataset_name'] diff --git a/uvdat/core/models/networks.py b/uvdat/core/models/networks.py index 6a83b1cc..5b9cb83d 100644 --- a/uvdat/core/models/networks.py +++ b/uvdat/core/models/networks.py @@ -1,6 +1,6 @@ -import networkx as nx from django.contrib.gis.db import models as geo_models from django.db import models +import networkx as nx from .dataset import Dataset diff --git a/uvdat/core/rest/dataset.py b/uvdat/core/rest/dataset.py index 80a06194..590639e3 100644 --- a/uvdat/core/rest/dataset.py +++ b/uvdat/core/rest/dataset.py @@ -74,9 +74,7 @@ def gcc(self, request, **kwargs): # this currently returns the gcc for the network with the most excluded nodes results = [] for network in dataset.networks.all(): - excluded_node_names = [ - n.name for n in network.nodes.all() if n.id in exclude_nodes - ] + excluded_node_names = [n.name for n in network.nodes.all() if n.id in exclude_nodes] gcc = network.get_gcc(exclude_nodes) results.append(dict(excluded=excluded_node_names, gcc=gcc)) if len(results): diff --git a/uvdat/core/tasks/dataset.py b/uvdat/core/tasks/dataset.py index 493b6407..b7db2ce2 100644 --- a/uvdat/core/tasks/dataset.py +++ b/uvdat/core/tasks/dataset.py @@ -1,14 +1,6 @@ from celery import shared_task -from uvdat.core.models import ( - Dataset, - FileItem, - NetworkEdge, - NetworkNode, - RasterMapLayer, - SourceRegion, - VectorMapLayer, -) +from uvdat.core.models import Dataset, FileItem, RasterMapLayer, SourceRegion, VectorMapLayer from uvdat.core.tasks.map_layers import save_vector_features from .map_layers import create_raster_map_layer, create_vector_map_layer diff --git a/uvdat/core/tasks/osmnx.py b/uvdat/core/tasks/osmnx.py index bc660b7e..ce35daaa 100644 --- a/uvdat/core/tasks/osmnx.py +++ b/uvdat/core/tasks/osmnx.py @@ -1,8 +1,5 @@ -import json - from celery import shared_task from django.contrib.gis.geos import LineString, Point -from django.core.files.base import ContentFile import osmnx from uvdat.core.models import Context, Dataset, Network, NetworkEdge, NetworkNode, VectorMapLayer @@ -38,7 +35,7 @@ def load_roads(context_id, location): context = Context.objects.get(id=context_id) dataset = get_or_create_road_dataset(context, location) network = Network.objects.create( - dataset=dataset, category="roads", metadata={'source': 'Fetched with OSMnx.'} + dataset=dataset, category='roads', metadata={'source': 'Fetched with OSMnx.'} ) print(f'Fetching road data for {location}...') diff --git a/uvdat/core/tasks/simulations.py b/uvdat/core/tasks/simulations.py index 13406b41..d552f8c1 100644 --- a/uvdat/core/tasks/simulations.py +++ b/uvdat/core/tasks/simulations.py @@ -46,7 +46,7 @@ def flood_scenario_1(simulation_result_id, network, elevation_data, flood_area): network = Network.objects.get(id=network) elevation_data = RasterMapLayer.objects.get(id=elevation_data) flood_area = VectorMapLayer.objects.get(id=flood_area) - except: + except Exception: result.error_message = 'Object not found.' result.save() return diff --git a/uvdat/core/tests/test_load_roads.py b/uvdat/core/tests/test_load_roads.py index 2d6d57dd..572de76e 100644 --- a/uvdat/core/tests/test_load_roads.py +++ b/uvdat/core/tests/test_load_roads.py @@ -1,5 +1,5 @@ -from django.core.management import call_command from django.contrib.gis.geos import Point +from django.core.management import call_command import pytest from uvdat.core.models import Context, Dataset @@ -9,9 +9,7 @@ def test_load_roads(): context = Context.objects.create( - name='Road Test', - default_map_zoom=10, - default_map_center=Point(42, -71) + name='Road Test', default_map_zoom=10, default_map_center=Point(42, -71) ) call_command( From 48fe642555bbfe995f57151ccfe5a99ebdf6349e Mon Sep 17 00:00:00 2001 From: Anne Haley Date: Fri, 2 Aug 2024 15:12:40 +0000 Subject: [PATCH 07/10] fix: undo change to expected number of contexts --- uvdat/core/tests/test_populate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uvdat/core/tests/test_populate.py b/uvdat/core/tests/test_populate.py index 479628b8..f2bcb7c7 100644 --- a/uvdat/core/tests/test_populate.py +++ b/uvdat/core/tests/test_populate.py @@ -34,7 +34,7 @@ def test_populate(): ) assert Chart.objects.all().count() == 1 - assert Context.objects.all().count() == 2 + assert Context.objects.all().count() == 3 assert Dataset.objects.all().count() == 4 assert DerivedRegion.objects.all().count() == 0 assert FileItem.objects.all().count() == 7 From bfd69c2f7fd8e16cd8e39424d03f22f72c21bc9f Mon Sep 17 00:00:00 2001 From: Anne Haley Date: Fri, 2 Aug 2024 18:31:34 +0000 Subject: [PATCH 08/10] fix: remove old name reference --- sample_data/ingest_sample_data.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sample_data/ingest_sample_data.py b/sample_data/ingest_sample_data.py index fb6f11b6..064b4170 100644 --- a/sample_data/ingest_sample_data.py +++ b/sample_data/ingest_sample_data.py @@ -133,8 +133,8 @@ def ingest_datasets(include_large=False, dataset_indexes=None): print('\t', f'Converting data for {dataset_for_conversion.name}...') dataset_for_conversion.spawn_conversion_task( style_options=dataset.get('style_options'), - network_options=network_options, - region_options=region_options, + network_options=dataset.get('network_options'), + region_options=dataset.get('region_options'), asynchronous=False, ) else: From f50de697a5dc8abb7e98b00d17b233cf69fa82a1 Mon Sep 17 00:00:00 2001 From: Anne Haley Date: Tue, 20 Aug 2024 21:19:04 +0000 Subject: [PATCH 09/10] refactor: apply suggested changes --- uvdat/core/rest/chart.py | 3 +-- uvdat/core/rest/dataset.py | 14 +++++++------- uvdat/core/tasks/chart.py | 7 ++----- uvdat/core/tests/test_load_roads.py | 1 - 4 files changed, 10 insertions(+), 15 deletions(-) diff --git a/uvdat/core/rest/chart.py b/uvdat/core/rest/chart.py index a7ec8700..55371d6a 100644 --- a/uvdat/core/rest/chart.py +++ b/uvdat/core/rest/chart.py @@ -12,8 +12,7 @@ class ChartViewSet(GenericViewSet, mixins.ListModelMixin): serializer_class = ChartSerializer def get_queryset(self, **kwargs): - request = self.request - context_id = request.query_params.get('context') + context_id = self.request.query_params.get('context') if context_id: return Chart.objects.filter(context__id=context_id) return Chart.objects.all() diff --git a/uvdat/core/rest/dataset.py b/uvdat/core/rest/dataset.py index 590639e3..9eaeb6ce 100644 --- a/uvdat/core/rest/dataset.py +++ b/uvdat/core/rest/dataset.py @@ -46,21 +46,21 @@ def convert(self, request, **kwargs): @action(detail=True, methods=['get']) def network(self, request, **kwargs): dataset = self.get_object() - return HttpResponse( - json.dumps( + networks = [] + for network in dataset.networks.all(): + networks.append([ { 'nodes': [ uvdat_serializers.NetworkNodeSerializer(n).data - for n in NetworkNode.objects.filter(network__dataset=dataset) + for n in NetworkNode.objects.filter(network=network) ], 'edges': [ uvdat_serializers.NetworkEdgeSerializer(e).data - for e in NetworkEdge.objects.filter(network__dataset=dataset) + for e in NetworkEdge.objects.filter(network=network) ], } - ), - status=200, - ) + ]) + return HttpResponse(json.dumps(networks), status=200) @action(detail=True, methods=['get']) def gcc(self, request, **kwargs): diff --git a/uvdat/core/tasks/chart.py b/uvdat/core/tasks/chart.py index 536c6292..62a5f511 100644 --- a/uvdat/core/tasks/chart.py +++ b/uvdat/core/tasks/chart.py @@ -4,7 +4,7 @@ import pandas from webcolors import name_to_hex -from uvdat.core.models import Chart, Context +from uvdat.core.models import Chart, Context, NetworkNode @shared_task @@ -45,9 +45,6 @@ def convert_chart(chart_id, conversion_options): def get_gcc_chart(dataset, context_id): chart_name = f'{dataset.name} Greatest Connected Component Sizes' - max_nodes = 0 - for network in dataset.networks.all(): - max_nodes += network.nodes.count() try: return Chart.objects.get(name=chart_name) except Chart.DoesNotExist: @@ -66,7 +63,7 @@ def get_gcc_chart(dataset, context_id): 'chart_title': 'Size of Greatest Connected Component over Period', 'x_title': 'Step when Excluded Nodes Changed', 'y_title': 'Number of Nodes', - 'y_range': [0, max_nodes], + 'y_range': [0, NetworkNode.objects.filter(network__dataset=dataset).count()], }, ) print('\t', f'Chart {chart.name} created.') diff --git a/uvdat/core/tests/test_load_roads.py b/uvdat/core/tests/test_load_roads.py index 572de76e..2fbaa6b4 100644 --- a/uvdat/core/tests/test_load_roads.py +++ b/uvdat/core/tests/test_load_roads.py @@ -7,7 +7,6 @@ @pytest.mark.django_db def test_load_roads(): - context = Context.objects.create( name='Road Test', default_map_zoom=10, default_map_center=Point(42, -71) ) From 431537b5e495be578f2a30c588d3a893e7ac4413 Mon Sep 17 00:00:00 2001 From: Anne Haley Date: Tue, 20 Aug 2024 21:31:20 +0000 Subject: [PATCH 10/10] style: reformat with black --- uvdat/core/rest/dataset.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/uvdat/core/rest/dataset.py b/uvdat/core/rest/dataset.py index 9eaeb6ce..d650949a 100644 --- a/uvdat/core/rest/dataset.py +++ b/uvdat/core/rest/dataset.py @@ -48,18 +48,20 @@ def network(self, request, **kwargs): dataset = self.get_object() networks = [] for network in dataset.networks.all(): - networks.append([ - { - 'nodes': [ - uvdat_serializers.NetworkNodeSerializer(n).data - for n in NetworkNode.objects.filter(network=network) - ], - 'edges': [ - uvdat_serializers.NetworkEdgeSerializer(e).data - for e in NetworkEdge.objects.filter(network=network) - ], - } - ]) + networks.append( + [ + { + 'nodes': [ + uvdat_serializers.NetworkNodeSerializer(n).data + for n in NetworkNode.objects.filter(network=network) + ], + 'edges': [ + uvdat_serializers.NetworkEdgeSerializer(e).data + for e in NetworkEdge.objects.filter(network=network) + ], + } + ] + ) return HttpResponse(json.dumps(networks), status=200) @action(detail=True, methods=['get'])