Api (#125)

* Added functions to load existing StreamCat ftp csv tables to the StreamCat API database * fix to populate tables function * Added function to list published / unpublished metrics in API * Updated delete table function and control table * updates to API * updates for StreamCat_API.py and ControlTable * updated control table * adjusting metric metadata info * channel width additions
USEPA · Mar 24, 2023 · 63b5b63 · 63b5b63
1 parent 8c7a3ab
commit 63b5b63
Show file tree

Hide file tree

Showing 2 changed files with 135 additions and 4 deletions.
diff --git a/ControlTable_StreamCat.csv b/ControlTable_StreamCat.csv
@@ -144,8 +144,8 @@ MTBS_2015,Continuous,MTBS_2015,none,MTBS_2015.tif,,MTBS,Mean,100,0,0,0,,Yes,,Yes
 MTBS_2016,Continuous,MTBS_2016,none,MTBS_2016.tif,,MTBS,Mean,100,0,0,0,,Yes,,Yes
 MTBS_2017,Continuous,MTBS_2017,none,MTBS_2017.tif,,MTBS,Mean,100,0,0,0,,Yes,,Yes
 MTBS_2018,Continuous,MTBS_2018,none,MTBS_2018.tif,,MTBS,Mean,100,0,0,0,,Yes,,Yes
-MTBS_Severity_1984,Categorical,lookup/MTBS_severity1984_lookup.csv,none,MTBS_Severity_1984.tif,,MTBS_Severity_1984,Percent,1,0,0,0,,Yes,,Yes
-MTBS_Severity_1985,Categorical,lookup/MTBS_severity1985_lookup.csv,none,MTBS_Severity_1985.tif,,MTBS_Severity_1985,Percent,1,0,0,0,,Yes,,Yes
+MTBS_Severity_1984,Categorical,lookup/MTBS_severity1984_lookup.csv,none,MTBS_Severity_1984.tif,,MTBS_Severity_1984,Percent,1,0,0,0,,Yes,,
+MTBS_Severity_1985,Categorical,lookup/MTBS_severity1985_lookup.csv,none,MTBS_Severity_1985.tif,,MTBS_Severity_1985,Percent,1,0,0,0,,Yes,,
 MTBS_Severity_1986,Categorical,lookup/MTBS_severity1986_lookup.csv,none,MTBS_Severity_1986.tif,,MTBS_Severity_1986,Percent,1,0,0,0,,Yes,,
 MTBS_Severity_1987,Categorical,lookup/MTBS_severity1987_lookup.csv,none,MTBS_Severity_1987.tif,,MTBS_Severity_1987,Percent,1,0,0,0,,Yes,,
 MTBS_Severity_1988,Categorical,lookup/MTBS_severity1988_lookup.csv,none,MTBS_Severity_1988.tif,,MTBS_Severity_1988,Percent,1,0,0,0,,Yes,,
@@ -293,7 +293,7 @@ RdDens_RipBuf100,Continuous,RdDens,Rp100,roadden.tif,,RoadDensityRipBuf100,Mean,
 RdCrs,Continuous,RdCrs,none,rdstcrs.tif,SlpWtd,RoadStreamCrossings,Density,0.03,0,0,0,,Yes,,Yes
 RockN,Continuous,RockN,none,RockN_USA_USGSproj_1km_kgkm2.tif,,RockN,Mean,1,0,0,0,,Yes,,Yes
 Runoff,Continuous,Runoff,none,runoff.tif,,Runoff,Mean,1,0,0,0,,Yes,,Yes
-Septic,Continuous,Septic,none,septic1990.tif,,Septic,Mean,0.001,0,0,0,,Yes,,Yes
+Septic,Continuous,Septic,none,septic1990.tif,,Septic,Mean,1,0,0,0,,Yes,,Yes
 Clay,Continuous,Clay,none,clay.tif,,STATSGO_Set1,Mean,0.01,0,0,0,,Yes,,Yes
 Sand,Continuous,Sand,none,sand.tif,,STATSGO_Set1,Mean,0.01,0,0,0,,Yes,,Yes
 Om,Continuous,Om,none,om.tif,,STATSGO_Set2,Mean,0.01,0,0,0,,Yes,,Yes

diff --git a/StreamCat_API.py b/StreamCat_API.py
@@ -15,6 +15,7 @@
 from datetime import datetime as dt
 import csv
 import urllib3
+from io import StringIO
 from bs4 import BeautifulSoup
 
 def DBtablesAsDF(config_file):
@@ -39,6 +40,7 @@ def DBtablesAsDF(config_file):
     df=pd.DataFrame.from_records(json_object)
     return df
 
+
 def ViewDBtable(config_file, table):
     """
     __author__ =  "Marc Weber <weber.marc@epa.gov>"
@@ -59,8 +61,34 @@ def ViewDBtable(config_file, table):
     headers=config.defaults(),
     verify=False
     )
+    #json_object = json.loads(r.text)
+    #df=pd.DataFrame.from_records(json_object)
+    #return df
     pprint(json.loads(r.text))
 
+def ViewMetadatatable(config_file, table):
+    """
+    __author__ =  "Marc Weber <weber.marc@epa.gov>"
+                  "Rick Debbout <debbout.rick@epa.gov>"
+    List table info for a specific StreamCat API database
+    table, using a config file that contains db url,username, 
+    password, and server
+    
+    Arguments
+    ---------
+    config_file            : configuration file with db configuration parameters
+    table                  : database table name
+    """
+    config = configparser.ConfigParser()
+    config.read(config_file)
+    r = requests.get(
+    f"{config['server']['URL']}/StreamCat/admin/manage/tables/{table}/variable_info",
+    headers=config.defaults(),
+    verify=False
+    )
+    df = pd.read_csv(StringIO(r.content.decode("utf8" , errors="ignore")))
+    return(df)
+
 
 def DeleteDBtable(config_file, table, just_data=False):
     """
@@ -145,6 +173,7 @@ def PopulateDBtable(config_file, table, file_loc, temp_file):
 
     counter=0
     for file in files:
+        print(file)
         infile = file_loc + '/' + file
         df = pd.read_csv(infile)
         counter+=len(df)
@@ -224,15 +253,45 @@ def MissingAPImetrics(config_file):
     return(published, not_published)
 
 
+def UpdateMetricMetadata(config_file, table, infile, temp_file):
+    """
+    __author__ =  "Marc Weber <weber.marc@epa.gov>"
+    Modify metric information in the StreamCat API database, 
+    using a config file that contains db url,username, password, 
+    and server.  You must provide a fresh list of all variable 
+    info entries for this resource, as it clears out the 
+    existing list and substitutes a new one.
+    
+    Arguments
+    ---------
+    config                 : configuration file with db configuration parameters
+    table                 : name of table metadata to load
+    infile                 : pandas data frame from ViewMetadatatable function     
+    """
+
+    config = configparser.ConfigParser()
+    config.read(config_file)
+    requests.urllib3.disable_warnings()
+    infile.to_csv(f'{temp_file}', index=False)
+    filedata = open(f'{temp_file}', "rb")
+    response = requests.put(f"{config['server']['URL']}/StreamCat/admin/manage/tables/{table}/variable_info", 
+                            headers=config.defaults(), verify=False, data=filedata)
+    return(response)
+
+###############
+
 # Define config file        
 config_file='E:/GitProjects/NARS/NARS/api/api_config.ini'
+# config_file='E:/GitProjects/NARS/NARS/api/api_config_postgres.ini'
+
 # List tables
 test = DBtablesAsDF(config_file)
 test.head()
 test.tail()
 test['DSNAME'][0:20]
 test['DSNAME'][21:40]
 test['DSNAME'][41:60]
+test['DSNAME'][61:70]
 test['DSNAME'][61:80]
 test['DSNAME'][81:120]
 # View a particular table
@@ -244,6 +303,7 @@ def MissingAPImetrics(config_file):
 # Delete a tables
 DeleteDBtable(config_file, table, just_data =True)
 # DeleteDBtable(config_file, table, just_data =False)
+
 # Create a table
 test = CreateDBtable(config_file, table_params)
 print(test)
@@ -255,10 +315,17 @@ def MissingAPImetrics(config_file):
 # table='GeoChemPhys2'
 table='ImperviousSurfacesRipBuf100'
 table='RoadDensityRipBuf100'
+table='Dams'
+table='predicted_channel_widths_depths'
+# file_loc='O:/PRIV/CPHEA/PESD/COR/CORFILES/Geospatial_Library_Projects/StreamCat/FTP_Staging/HydroRegions'
+file_loc='O:/PRIV/CPHEA/PESD/COR/CORFILES/Geospatial_Library_Projects/StreamCat/predicted-values/widths-depths_v2'
+temp_file='E:/WorkingData/junk.csv'
+
 table='MTBS'
 table='WWTP'
 file_loc='O:/PRIV/CPHEA/PESD/COR/CORFILES/Geospatial_Library_Projects/StreamCat/FTP_Staging/HydroRegions'
 temp_file='E:/WorkingData/junk2.csv'
+
 LoadTime = dt.now()
 PopulateDBtable(config_file, table, file_loc, temp_file)
 print("Table load complete in : " + str(dt.now() - LoadTime))
@@ -277,6 +344,51 @@ def MissingAPImetrics(config_file):
 published, unpublished = MissingAPImetrics(config_file)
 
 
+# View metadata for a table
+table = 'WWTP'
+df = ViewMetadatatable(config_file, table)
+df.loc[df.METRIC_NAME == 'WWTPAllDens[AOI]', 'METRIC_UNITS'] = "number/ km2"
+
+# Read in .csv file of metadata
+table='predicted_channel_widths_depths'
+met = pd.read_csv('O:/PRIV/CPHEA/PESD/COR/CORFILES/Geospatial_Library_Projects/StreamCat/MetaData/StreamCatMetrics.csv', encoding='cp1252')
+met = met.loc[met['final_table'] == table]
+met.columns = met.columns.str.upper()
+met = met[df.columns]
+temp_file='E:/WorkingData/junk.csv'
+UpdateMetricMetadata(config_file, table, met, temp_file)
+
+# Update metadata for a table
+# make any adjustments to metrics in table and update
+df['SOURCE_URL'].values[0]
+df['SOURCE_URL'] = 'https://nadp.slh.wisc.edu/maps-data/ntn-gradient-maps/'
+df['SOURCE_URL'].values[0]
+
+# View metadata for a table
+table = 'RefStreamTempPred'
+df = ViewMetadatatable(config_file, table)
+
+# Update metadata for a table
+# make any adjustments to metrics in table and update
+df['SOURCE_URL'].values[0]
+df['SOURCE_URL'] = 'https://enviroatlas.epa.gov/enviroatlas/DataFactSheets/pdf/Supplemental/PotentialWetlandArea.pdf'
+df['SOURCE_URL'].values[0]
+temp_file='E:/WorkingData/junk.csv'
+UpdateMetricMetadata(config_file, table, df, temp_file)
+
+
+
+
+table_params = {"name": "predicted_channel_widths_depths",
+            "metrics":[{"name": "wetted_width_m", "display_name": "Predicted wetted width"},
+                       {"name": "thalweg_depth_cm", "display_name": "Predicted Thalweg Depth"},
+                       {"name": "bankfull_width_m", "display_name": "Predicted Bankfull Widthy"},
+                       {"name": "bankfull_depth_m", "display_name": "Predicted Bankfull Depth"}],
+            "columns": [{"name": "CatPctFull", "type": "number"},{"name": "WsPctFull", "type": "number"},
+                        {"name": "wetted_width_m", "type": "number"},{"name": "thalweg_depth_cm","type": "number"},
+                        {"name": "bankfull_width_m", "type": "number"},{"name": "bankfull_depth_m","type": "number"},]}
+
+
 table_params = {"name": "AgMidHiSlopes2011",
             "metrics":[{"name": "PctAg2011Slp10Cat", "display_name": "Percent of Agriculture on 10% Slope"},
                        {"name": "PctAg2011Slp20Cat", "display_name": "Percent of Agriculture on 20% Slope"}],
@@ -314,6 +426,24 @@ def MissingAPImetrics(config_file):
                         {"name": "WWTPAllDensCat", "type": "number"},{"name": "WWTPAllDensWs","type": "number"},]}
 
 
+
+table_params = {"name": "MTBS_Severity_1984",
+            "metrics":[{"name": "pctnofire1984", "display_name": "Percent No Fire Burn Class For Year"},
+                       {"name": "pctundsev1984", "display_name": "Percent Underburned to Low Burn Severity Class For Year"},
+                       {"name": "pctlowsev1984", "display_name": "Percent Low Burn Severity Class For Year"},
+                       {"name": "pctmodsev1984", "display_name": "Percent Moderate Burn Severity Class For Year"},
+                       {"name": "pcthighsev1984", "display_name": "Percent High Burn Severity Class For Year"},
+                       {"name": "pctincvegresp1984", "display_name": "Percent Increased Greenness and Veg Response Class For Year"},
+                       {"name": "pctnonprocmask1984", "display_name": "Percent Non Processing Mask Class For Year"}],
+            "columns": [{"name": "CatPctFull", "type": "number"},{"name": "WsPctFull", "type": "number"},
+                        {"name": "PctNoFireCat1984Cat", "type": "number"},{"name": "PctNoFire1984Ws","type": "number"},
+                        {"name": "PctUndSev1984Cat", "type": "number"},{"name": "PctUndSev1984Ws","type": "number"},
+                        {"name": "PctLowSev1984Cat", "type": "number"},{"name": "PctLowSev1984Ws","type": "number"},
+                        {"name": "PctModSev1984Cat", "type": "number"},{"name": "PctModSev1984Ws","type": "number"},
+                        {"name": "PctHighSev1984Cat", "type": "number"},{"name": "PctHighSev1984Ws","type": "number"},
+                        {"name": "PctIncVegResp1984Cat", "type": "number"},{"name": "PctIncVegResp1984Ws","type": "number"},
+                        {"name": "PctNonProcMask1984Cat", "type": "number"},{"name": "PctNonProcMask1984Ws","type": "number"}]}
+
 table_params = {"name": "MTBS_Severity_2018",
             "metrics":[{"name": "pctnofire2018", "display_name": "Percent No Fire Burn Class For Year"},
                        {"name": "pctundsev2018", "display_name": "Percent Underburned to Low Burn Severity Class For Year"},
@@ -474,4 +604,5 @@ def MissingAPImetrics(config_file):
                         {"name": "PctUrbLo2019CatRp100", "type": "number"},{"name": "PctUrbLo2019WsRp100","type": "number"},
                         {"name": "PctUrbMd2019CatRp100", "type": "number"},{"name": "PctUrbMd2019WsRp100","type": "number"},
                         {"name": "PctUrbOp2019CatRp100", "type": "number"},{"name": "PctUrbOp2019WsRp100","type": "number"},
-                        {"name": "PctWdWet2019CatRp100", "type": "number"},{"name": "PctWdWet2019WsRp100","type": "number"}]}
+                        {"name": "PctWdWet2019CatRp100", "type": "number"},{"name": "PctWdWet2019WsRp100","type": "number"}]}
+