From 80389150da5af005e9cc552427bb24957ecf1536 Mon Sep 17 00:00:00 2001 From: Vismayak Mohanarajan Date: Wed, 24 Apr 2024 12:26:25 -0500 Subject: [PATCH 1/4] Buyout Model Implementation --- pyincore/analyses/buyoutdecision/__init__.py | 8 + .../analyses/buyoutdecision/buyoutdecision.py | 175 ++++++++++++++++++ .../buyoutdecision/test_buyoutdecision.py | 46 +++++ 3 files changed, 229 insertions(+) create mode 100644 pyincore/analyses/buyoutdecision/__init__.py create mode 100644 pyincore/analyses/buyoutdecision/buyoutdecision.py create mode 100644 tests/pyincore/analyses/buyoutdecision/test_buyoutdecision.py diff --git a/pyincore/analyses/buyoutdecision/__init__.py b/pyincore/analyses/buyoutdecision/__init__.py new file mode 100644 index 00000000..ee1f996d --- /dev/null +++ b/pyincore/analyses/buyoutdecision/__init__.py @@ -0,0 +1,8 @@ +# Copyright (c) 2019 University of Illinois and others. All rights reserved. +# +# This program and the accompanying materials are made available under the +# terms of the Mozilla Public License v2.0 which accompanies this distribution, +# and is available at https://www.mozilla.org/en-US/MPL/2.0/ + + +from pyincore.analyses.buyoutdecision.buyoutdecision import BuyoutDecision \ No newline at end of file diff --git a/pyincore/analyses/buyoutdecision/buyoutdecision.py b/pyincore/analyses/buyoutdecision/buyoutdecision.py new file mode 100644 index 00000000..c7820321 --- /dev/null +++ b/pyincore/analyses/buyoutdecision/buyoutdecision.py @@ -0,0 +1,175 @@ +# This program and the accompanying materials are made available under the +# terms of the Mozilla Public License v2.0 which accompanies this distribution, +# and is available at https://www.mozilla.org/en-US/MPL/2.0/ + +import pandas as pd +from pyincore import BaseAnalysis +from pyincore.utils.dataprocessutil import DataProcessUtil + + +class BuyoutDecision(BaseAnalysis): + """A framework to select households for buyout based on past and future flood damaged. + + Args: + incore_client(IncoreClient): Service authentication. + + """ + + def __init__(self, incore_client): + super(BuyoutDecision, self).__init__(incore_client) + + def run(self): + # Get input parameters + fema_buyout_cap = self.get_parameter('fema_buyout_cap') + residential_archetypes = self.get_parameter('residential_archetypes') + + # Get input datasets + past_building_damage = self.get_input_dataset('past_building_damage').get_dataframe_from_csv(low_memory=False) + future_building_damage = self.get_input_dataset('future_building_damage').get_dataframe_from_csv( + low_memory=False) + + building_inventory = self.get_input_dataset('buildings').get_dataframe_from_shapefile() + + hua = (self.get_input_dataset('housing_unit_allocation').get_dataframe_from_csv(low_memory=False)) + pop_dislocation = self.get_input_dataset('population_dislocation').get_dataframe_from_csv(low_memory=False) + + buyout_decision_df = self.buyout_decision(past_building_damage, future_building_damage, building_inventory, hua, + pop_dislocation, fema_buyout_cap, residential_archetypes) + # Create the result dataset + self.set_result_csv_data("result", buyout_decision_df, self.get_parameter("result_name") + "_loss", + "dataframe") + + def buyout_decision(self, past_building_damage, future_building_damage, building_inventory, hua, pop_dislocation, + fema_buyout_cap, residential_archetpyes): + """Select households for buyout based on past and future flood damaged. + + Args: + past_building_damage (DataFrame): Past building damage. + future_building_damage (DataFrame): Future event building damage. + building_inventory (DataFrame): Building inventory. + hua (DataFrame): Housing unit allocation. + pop_dislocation (DataFrame): Population dislocation from past hazard event. + fema_buyout_cap (float): FEMA buyout cap. + residential_archetpyes (list): Residential archetypes. + + Returns: + buyout_decision_df (DataFrame): A dataframe with buyout decision for each household. + """ + + past_building_max_damage = DataProcessUtil.get_max_damage_state(past_building_damage) + future_building_max_damage = DataProcessUtil.get_max_damage_state(future_building_damage) + + # Criterion 1: Filter only residential buildings with damage state DS3 from past building damage + buyout_inventory = pd.merge(building_inventory, past_building_max_damage, on='guid', how='outer') + buyout_inventory = buyout_inventory[buyout_inventory['arch_wind'].isin(residential_archetpyes) + & (buyout_inventory['max_state'] == 'DS_3')] + buyout_inventory.rename(columns={'max_state': 'max_state_past_damage'}, inplace=True) + + # Criterion 2: Filter only residential buildings with damage state DS3 from predicted future building damage + buyout_inventory = pd.merge(buyout_inventory, future_building_max_damage, on='guid', how='inner') + buyout_inventory = buyout_inventory[buyout_inventory['max_state'] == 'DS_3'] + buyout_inventory.rename(columns={'max_state': 'max_state_future_damage'}, inplace=True) + + # Criterion 3: Fall within the FEMA buyout cap + buyout_inventory = buyout_inventory[buyout_inventory['appr_bldg'] <= fema_buyout_cap] + buyout_inventory = buyout_inventory[ + ["guid", "appr_bldg", "max_state_future_damage", "max_state_past_damage", "geometry"]] + + # Criterion 4: Use HUA to filter out buildings with 0 occupants + buyout_inventory = pd.merge(buyout_inventory, hua, on='guid', how='left') + buyout_inventory = buyout_inventory[(buyout_inventory['numprec'] != 0) & (~buyout_inventory['numprec'].isna())] + + # Removing any rows with NAN values in column "Race" + buyout_inventory = buyout_inventory.dropna(subset=['race']) + + # Merging with population dislocation + buyout_inventory = pd.merge(buyout_inventory, pop_dislocation[['huid', 'dislocated']], on='huid', how='left') + + # Create a new column showing the appraisal value of each building ('appr_bldg' divided by the number of times a guid is repeated) + # For the instances that a structure has more than one housing units. + buyout_inventory['count'] = buyout_inventory.groupby('guid')['guid'].transform('count') + buyout_inventory['housing_unit_appraisal_value'] = buyout_inventory['appr_bldg'] / buyout_inventory['count'] + + # Cleaning the dataframe + buyout_inventory.drop(['blockid', 'bgid', 'tractid', 'FIPScounty', + 'gqtype', 'BLOCKID10_str', 'placeNAME10', 'geometry_y'], axis=1, inplace=True) + buyout_inventory.rename(columns={'appr_bldg': 'building_appraisal_value', 'ownershp': 'ownership', + 'dislocated_combined_dmg': 'dislocated', 'count': 'number_of_housing_units', + 'geometry_x': 'geometry'}, + inplace=True) + buyout_inventory = buyout_inventory[ + ['guid', 'huid', 'building_appraisal_value', 'housing_unit_appraisal_value', 'geometry', + 'number_of_housing_units', 'numprec', 'ownership', 'race', 'hispan', 'family', 'vacancy', 'incomegroup', + 'hhinc','randincome','poverty', 'huestimate', 'dislocated', 'max_state_future_damage', + 'max_state_past_damage', 'x', 'y', ]] + + + return buyout_inventory + + def get_spec(self): + return { + "name": "buyout-decision", + "description": "Buyout decision framework", + "input_parameters": [ + { + 'id': 'fema_buyout_cap', + 'required': True, + 'description': 'FEMA buyout cap', + 'type': float, + }, + { + 'id': 'residential_archetypes', + 'required': True, + 'description': 'Residential archetypes', + 'type': list, + }, + { + 'id': 'result_name', + 'required': True, + 'description': 'Result name', + 'type': str, + } + ], + "input_datasets": [ + { + 'id': 'past_building_damage', + 'required': True, + 'description': 'Building Damage Results', + 'type': ['ergo:buildingDamageVer6'], + }, + { + 'id': 'future_building_damage', + 'required': True, + 'description': 'Building Damage Results', + 'type': ['ergo:buildingDamageVer6'], + }, + { + 'id': 'buildings', + 'required': True, + 'description': 'Building Inventory', + 'type': ['ergo:buildingInventoryVer4', 'ergo:buildingInventoryVer5', + 'ergo:buildingInventoryVer6', 'ergo:buildingInventoryVer7'], + }, + { + 'id': 'housing_unit_allocation', + 'required': True, + 'description': 'A csv file with the merged dataset of the inputs, aka Probabilistic' + 'House Unit Allocation', + 'type': ['incore:housingUnitAllocation'] + }, + { + 'id': 'population_dislocation', + 'required': True, + 'description': 'Population Dislocation from past hazard event', + 'type': ['incore:popDislocation'] + } + ], + "output_datasets": [ + { + 'id': 'result', + 'label': 'Buyout Decision Results', + 'description': 'Buyout Decision Results', + 'type': ['incore:buyoutDecision'] + } + ] + } diff --git a/tests/pyincore/analyses/buyoutdecision/test_buyoutdecision.py b/tests/pyincore/analyses/buyoutdecision/test_buyoutdecision.py new file mode 100644 index 00000000..3f5d62d8 --- /dev/null +++ b/tests/pyincore/analyses/buyoutdecision/test_buyoutdecision.py @@ -0,0 +1,46 @@ +# This program and the accompanying materials are made available under the +# terms of the Mozilla Public License v2.0 which accompanies this distribution, +# and is available at https://www.mozilla.org/en-US/MPL/2.0/ + + +from pyincore.analyses.buyoutdecision import BuyoutDecision +from pyincore import IncoreClient, Dataset +import pyincore.globals as pyglobals +import pandas as pd + + +def BuyoutDecisionTest(): + client = IncoreClient(pyglobals.INCORE_API_DEV_URL) + + past_building_damage_id = "6632d2605da5fd22b268511f" + future_building_damage_id = "6632d45b5da5fd22b2685136" + past_pop_dislocation_id = "6632d5205da5fd22b26878bb" + + hua_id = "64227016b18d026e7c80d2bc" + + buildings_id = "63ff69a96d3b2a308baaca12" + + fema_buyout_cap = 321291.600 + residential_archetypes = [1, 2, 3, 4, 5, 17] + + buyout_decision = BuyoutDecision(client) + buyout_decision.set_parameter("fema_buyout_cap", fema_buyout_cap) + buyout_decision.set_parameter("residential_archetypes", residential_archetypes) + buyout_decision.set_parameter("result_name", "galveston_buyout") + + buyout_decision.load_remote_input_dataset("buildings", buildings_id) + buyout_decision.load_remote_input_dataset("housing_unit_allocation", hua_id) + buyout_decision.load_remote_input_dataset("past_building_damage", past_building_damage_id) + buyout_decision.load_remote_input_dataset("future_building_damage", future_building_damage_id) + buyout_decision.load_remote_input_dataset("population_dislocation", past_pop_dislocation_id) + + buyout_decision.run_analysis() + + result = buyout_decision.get_output_dataset("result") + result_df = result.get_dataframe_from_csv() + print(result_df.head()) + print(len(result_df)) + + +if __name__ == "__main__": + BuyoutDecisionTest() From d9a0b7c4503ed356547986753fe259c7a2ec35d1 Mon Sep 17 00:00:00 2001 From: Vismayak Mohanarajan Date: Thu, 2 May 2024 10:14:18 -0500 Subject: [PATCH 2/4] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c1ed2536..c75d941b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ### Added - Gas Facility Damage Analysis [#568](https://github.com/IN-CORE/pyincore/issues/568) - Copyrights to transportation recovery analysis [#579](https://github.com/IN-CORE/pyincore/issues/579) +- Buyout Model Analyses [#539](https://github.com/IN-CORE/pyincore/issues/539) ### Fixed - Permission error in clearing cache process [#563](https://github.com/IN-CORE/pyincore/issues/563) From 471dcd992d8a321e813ba590af3d2139a03fdfbc Mon Sep 17 00:00:00 2001 From: Vismayak Mohanarajan Date: Mon, 3 Jun 2024 10:00:11 -0500 Subject: [PATCH 3/4] Suggested fixes --- pyincore/analyses/buyoutdecision/__init__.py | 3 +-- pyincore/analyses/buyoutdecision/buyoutdecision.py | 2 ++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pyincore/analyses/buyoutdecision/__init__.py b/pyincore/analyses/buyoutdecision/__init__.py index ee1f996d..211952b6 100644 --- a/pyincore/analyses/buyoutdecision/__init__.py +++ b/pyincore/analyses/buyoutdecision/__init__.py @@ -1,8 +1,7 @@ -# Copyright (c) 2019 University of Illinois and others. All rights reserved. +# Copyright (c) 2024 University of Illinois and others. All rights reserved. # # This program and the accompanying materials are made available under the # terms of the Mozilla Public License v2.0 which accompanies this distribution, # and is available at https://www.mozilla.org/en-US/MPL/2.0/ - from pyincore.analyses.buyoutdecision.buyoutdecision import BuyoutDecision \ No newline at end of file diff --git a/pyincore/analyses/buyoutdecision/buyoutdecision.py b/pyincore/analyses/buyoutdecision/buyoutdecision.py index c7820321..adb027a5 100644 --- a/pyincore/analyses/buyoutdecision/buyoutdecision.py +++ b/pyincore/analyses/buyoutdecision/buyoutdecision.py @@ -1,3 +1,5 @@ +# Copyright (c) 2024 University of Illinois and others. All rights reserved. + # This program and the accompanying materials are made available under the # terms of the Mozilla Public License v2.0 which accompanies this distribution, # and is available at https://www.mozilla.org/en-US/MPL/2.0/ From 2793f5fbddc6f126ce168b7f11ae58b53b195722 Mon Sep 17 00:00:00 2001 From: Chen Wang Date: Tue, 4 Jun 2024 15:39:28 -0500 Subject: [PATCH 4/4] add to sphinx doc and pep8 reformat --- docs/source/modules.rst | 5 +++++ pyincore/analyses/buyoutdecision/__init__.py | 2 +- pyincore/analyses/buyoutdecision/buyoutdecision.py | 6 +++--- .../pyincore/analyses/buyoutdecision/test_buyoutdecision.py | 2 +- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/docs/source/modules.rst b/docs/source/modules.rst index afe10f54..e8b9bf80 100644 --- a/docs/source/modules.rst +++ b/docs/source/modules.rst @@ -35,6 +35,11 @@ analyses/buildingportfolio .. autoclass:: pyincore.analyses.buildingportfolio.recovery.BuildingData :members: +analyses/buyoutdecision +======================= +.. autoclass:: buyoutdecision.buyoutdecision.BuyoutDecision + :members: + analyses/capitalshocks ====================== diff --git a/pyincore/analyses/buyoutdecision/__init__.py b/pyincore/analyses/buyoutdecision/__init__.py index 211952b6..390600ff 100644 --- a/pyincore/analyses/buyoutdecision/__init__.py +++ b/pyincore/analyses/buyoutdecision/__init__.py @@ -4,4 +4,4 @@ # terms of the Mozilla Public License v2.0 which accompanies this distribution, # and is available at https://www.mozilla.org/en-US/MPL/2.0/ -from pyincore.analyses.buyoutdecision.buyoutdecision import BuyoutDecision \ No newline at end of file +from pyincore.analyses.buyoutdecision.buyoutdecision import BuyoutDecision diff --git a/pyincore/analyses/buyoutdecision/buyoutdecision.py b/pyincore/analyses/buyoutdecision/buyoutdecision.py index adb027a5..0fa352f7 100644 --- a/pyincore/analyses/buyoutdecision/buyoutdecision.py +++ b/pyincore/analyses/buyoutdecision/buyoutdecision.py @@ -87,7 +87,8 @@ def buyout_decision(self, past_building_damage, future_building_damage, building # Merging with population dislocation buyout_inventory = pd.merge(buyout_inventory, pop_dislocation[['huid', 'dislocated']], on='huid', how='left') - # Create a new column showing the appraisal value of each building ('appr_bldg' divided by the number of times a guid is repeated) + # Create a new column showing the appraisal value of each building ('appr_bldg' divided by the number of times + # a guid is repeated) # For the instances that a structure has more than one housing units. buyout_inventory['count'] = buyout_inventory.groupby('guid')['guid'].transform('count') buyout_inventory['housing_unit_appraisal_value'] = buyout_inventory['appr_bldg'] / buyout_inventory['count'] @@ -102,10 +103,9 @@ def buyout_decision(self, past_building_damage, future_building_damage, building buyout_inventory = buyout_inventory[ ['guid', 'huid', 'building_appraisal_value', 'housing_unit_appraisal_value', 'geometry', 'number_of_housing_units', 'numprec', 'ownership', 'race', 'hispan', 'family', 'vacancy', 'incomegroup', - 'hhinc','randincome','poverty', 'huestimate', 'dislocated', 'max_state_future_damage', + 'hhinc', 'randincome', 'poverty', 'huestimate', 'dislocated', 'max_state_future_damage', 'max_state_past_damage', 'x', 'y', ]] - return buyout_inventory def get_spec(self): diff --git a/tests/pyincore/analyses/buyoutdecision/test_buyoutdecision.py b/tests/pyincore/analyses/buyoutdecision/test_buyoutdecision.py index 3f5d62d8..843f65ca 100644 --- a/tests/pyincore/analyses/buyoutdecision/test_buyoutdecision.py +++ b/tests/pyincore/analyses/buyoutdecision/test_buyoutdecision.py @@ -12,7 +12,7 @@ def BuyoutDecisionTest(): client = IncoreClient(pyglobals.INCORE_API_DEV_URL) - past_building_damage_id = "6632d2605da5fd22b268511f" + past_building_damage_id = "6632d2605da5fd22b268511f" future_building_damage_id = "6632d45b5da5fd22b2685136" past_pop_dislocation_id = "6632d5205da5fd22b26878bb"