Skip to content

Commit

Permalink
0.3.3
Browse files Browse the repository at this point in the history
  • Loading branch information
@fenke
fenke committed Sep 10, 2024
1 parent 9d8625c commit 217b787
Show file tree
Hide file tree
Showing 10 changed files with 11,676 additions and 261 deletions.
2 changes: 2 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -157,3 +157,5 @@ nbs/saves/rscript/stderr-*
nbs/saves/rscript/stdout-*
nbs/saves/rscript/Data/*.RData
nbs/saves/rscript/*.RData
nbs/saves/rscript/*.csv
nbs/saves/rscript/*.R
2 changes: 1 addition & 1 deletion corebridge/__init__.py
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
__version__ = "0.2.28"
__version__ = "0.3.3"
32 changes: 30 additions & 2 deletions corebridge/_modidx.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,9 @@
'corebridge/aicorebridge.py'),
'corebridge.aicorebridge.build_historic_args': ( 'aicorebridge.html#build_historic_args',
'corebridge/aicorebridge.py')},
'corebridge.core': { 'corebridge.core.init_console_logging': ('core.html#init_console_logging', 'corebridge/core.py'),
'corebridge.core': { 'corebridge.core.AICoreModuleBase': ('core.html#aicoremodulebase', 'corebridge/core.py'),
'corebridge.core.AICoreModuleBase.__init__': ('core.html#aicoremodulebase.__init__', 'corebridge/core.py'),
'corebridge.core.init_console_logging': ('core.html#init_console_logging', 'corebridge/core.py'),
'corebridge.core.pop_nan_values': ('core.html#pop_nan_values', 'corebridge/core.py'),
'corebridge.core.set_time_index_zone': ('core.html#set_time_index_zone', 'corebridge/core.py'),
'corebridge.core.timeseries_dataframe': ('core.html#timeseries_dataframe', 'corebridge/core.py'),
Expand All @@ -33,7 +35,27 @@
'corebridge/core.py'),
'corebridge.core.timeseries_dataframe_to_datadict': ( 'core.html#timeseries_dataframe_to_datadict',
'corebridge/core.py')},
'corebridge.rscript': { 'corebridge.rscript.calc_hash_from_data_files': ( 'rscriptbridge.html#calc_hash_from_data_files',
'corebridge.rscript': { 'corebridge.rscript.AICoreRScriptModule': ( 'rscriptbridge.html#aicorerscriptmodule',
'corebridge/rscript.py'),
'corebridge.rscript.AICoreRScriptModule.__init__': ( 'rscriptbridge.html#aicorerscriptmodule.__init__',
'corebridge/rscript.py'),
'corebridge.rscript.AICoreRScriptModule.get_flow_status': ( 'rscriptbridge.html#aicorerscriptmodule.get_flow_status',
'corebridge/rscript.py'),
'corebridge.rscript.AICoreRScriptModule.get_rscript_libpath': ( 'rscriptbridge.html#aicorerscriptmodule.get_rscript_libpath',
'corebridge/rscript.py'),
'corebridge.rscript.AICoreRScriptModule.get_rscript_workdir': ( 'rscriptbridge.html#aicorerscriptmodule.get_rscript_workdir',
'corebridge/rscript.py'),
'corebridge.rscript.AICoreRScriptModule.get_save_path': ( 'rscriptbridge.html#aicorerscriptmodule.get_save_path',
'corebridge/rscript.py'),
'corebridge.rscript.AICoreRScriptModule.infer': ( 'rscriptbridge.html#aicorerscriptmodule.infer',
'corebridge/rscript.py'),
'corebridge.rscript.AICoreRScriptModule.read_data': ( 'rscriptbridge.html#aicorerscriptmodule.read_data',
'corebridge/rscript.py'),
'corebridge.rscript.AICoreRScriptModule.update_flow': ( 'rscriptbridge.html#aicorerscriptmodule.update_flow',
'corebridge/rscript.py'),
'corebridge.rscript.AICoreRScriptModule.write_uploaded_data': ( 'rscriptbridge.html#aicorerscriptmodule.write_uploaded_data',
'corebridge/rscript.py'),
'corebridge.rscript.calc_hash_from_data_files': ( 'rscriptbridge.html#calc_hash_from_data_files',
'corebridge/rscript.py'),
'corebridge.rscript.calc_hash_from_files': ( 'rscriptbridge.html#calc_hash_from_files',
'corebridge/rscript.py'),
Expand All @@ -58,7 +80,13 @@
'corebridge.rscript.get_save_path': ('rscriptbridge.html#get_save_path', 'corebridge/rscript.py'),
'corebridge.rscript.install_R_package_wait': ( 'rscriptbridge.html#install_r_package_wait',
'corebridge/rscript.py'),
'corebridge.rscript.recursive_flatten_nested_data': ( 'rscriptbridge.html#recursive_flatten_nested_data',
'corebridge/rscript.py'),
'corebridge.rscript.release_script_lock': ( 'rscriptbridge.html#release_script_lock',
'corebridge/rscript.py'),
'corebridge.rscript.run_rscript_nowait': ( 'rscriptbridge.html#run_rscript_nowait',
'corebridge/rscript.py'),
'corebridge.rscript.run_rscript_wait': ('rscriptbridge.html#run_rscript_wait', 'corebridge/rscript.py'),
'corebridge.rscript.snake_case_to_camel_case': ( 'rscriptbridge.html#snake_case_to_camel_case',
'corebridge/rscript.py'),
'corebridge.rscript.unpack_assets': ('rscriptbridge.html#unpack_assets', 'corebridge/rscript.py')}}}
5 changes: 3 additions & 2 deletions corebridge/aicorebridge.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ def build_historic_args(data:pd.DataFrame, history:dict|list) -> dict:


# %% ../nbs/01_aicorebridge.ipynb 12
class AICoreModule(): pass
class AICoreModule(AICoreModuleBase): pass

# %% ../nbs/01_aicorebridge.ipynb 13
@patch
Expand Down Expand Up @@ -199,6 +199,7 @@ def infer(self:AICoreModule, data:dict, *_, **kwargs):
]
}


# %% ../nbs/01_aicorebridge.ipynb 20
@patch
def init_annotated_param(self:AICoreModule, param_name, value):
Expand Down Expand Up @@ -252,7 +253,7 @@ def get_callargs(self:AICoreModule, kwargs, history):
}


# %% ../nbs/01_aicorebridge.ipynb 25
# %% ../nbs/01_aicorebridge.ipynb 24
@patch
def get_call_data(
self:AICoreModule,
Expand Down
42 changes: 28 additions & 14 deletions corebridge/rscript.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
'calc_hash_from_flowobject', 'calc_hash_from_files', 'calc_hash_from_input_files',
'calc_hash_from_data_files', 'check_script_inputs', 'check_script_output', 'generate_checksum_file',
'run_rscript_wait', 'run_rscript_nowait', 'release_script_lock', 'AICoreRScriptModule',
'recursive_flatten_nested_data']
'snake_case_to_camel_case', 'recursive_flatten_nested_data']

# %% ../nbs/02_rscriptbridge.ipynb 4
import os, logging, json, hashlib
Expand Down Expand Up @@ -500,9 +500,14 @@ def update_flow(self:AICoreRScriptModule):


# %% ../nbs/02_rscriptbridge.ipynb 125
def snake_case_to_camel_case(snake_case:str) -> str:
splittext = snake_case.split('_')
return ''.join([x.capitalize() if n > 0 else x for x,n in zip(splittext, range(len(splittext)))])

def recursive_flatten_nested_data(
data:dict,
column_prefix:str='') -> dict:
column_prefix:str='',
camel_case=False) -> dict:

if isinstance(data, np.ndarray):
return {column_prefix:data}
Expand All @@ -511,7 +516,7 @@ def recursive_flatten_nested_data(
return reduce(
lambda R, X: dict(**R, **X) if R else X,
[
recursive_flatten_nested_data(value, f"{column_prefix}_{i+1}_")
recursive_flatten_nested_data(value, f"{column_prefix}_{i+1}_", camel_case)
for i, value in enumerate(data)

],
Expand All @@ -529,7 +534,9 @@ def recursive_flatten_nested_data(
[
recursive_flatten_nested_data(
value,
f"{column_prefix}{str(key).capitalize()}")
snake_case_to_camel_case(column_prefix+'_'+str(key)) if camel_case else column_prefix+'_'+str(key),
camel_case
)
for key, value in data.items()
],
{}
Expand All @@ -538,9 +545,12 @@ def recursive_flatten_nested_data(
else:
key = list(data.keys())[0]
value = data[key]
column_name = f"{column_prefix}{str(key).capitalize()}" if column_prefix else str(key)
if column_prefix:
column_name = snake_case_to_camel_case(column_prefix+'_'+str(key)) if camel_case else column_prefix+'_'+str(key)
else:
column_name = snake_case_to_camel_case(str(key)) if camel_case else str(key)
return recursive_flatten_nested_data(
value, column_name,
value, column_name, camel_case
)


Expand All @@ -558,15 +568,19 @@ def write_uploaded_data(
df.reset_index().to_csv(csv_filename, index=False, date_format='%Y-%m-%d %H:%M:%S')

@patch
def read_data(self:AICoreRScriptModule, tag:str=None, **kwargs):
def read_data(self:AICoreRScriptModule, tag:str=None, camel_case=False, **kwargs):

rdata_filename = self.get_save_path(self.data_files_map.get(tag, tag))
converted = rdata.read_rda(rdata_filename)

flattened = recursive_flatten_nested_data(converted)
df = pd.DataFrame(flattened)
df.set_index( pd.DatetimeIndex(df['ensemble_predTime']), inplace=True)

time_column = [k for k,v in df.dtypes.to_dict().items() if 'float' not in str(v)][0]
df.set_index( pd.DatetimeIndex(df[time_column]), inplace=True)
df.index.name = 'time'
df.drop('ensemble_predTime', axis=1, inplace=True)
df.drop(time_column, axis=1, inplace=True)

return df


Expand All @@ -576,10 +590,6 @@ def infer(
self:AICoreRScriptModule,
data:dict,
*_,
writeTag:str=None,
readTag:str=None,
timezone:str='UTC',
recordformat:str='records',
**kwargs):

"""
Expand All @@ -595,6 +605,10 @@ def infer(

msg += self.update_flow()
# Pickup params, pop those that are not intended for the processor
writeTag = kwargs.pop('writeTag', None)
readTag = kwargs.pop('readTag', None)
camelCase = bool(kwargs.pop('camelCase', False))

lastSeen = kwargs.pop('lastSeen', False)
recordformat = kwargs.pop('format', "records").lower()
timezone = kwargs.get('timezone', 'UTC')
Expand All @@ -613,7 +627,7 @@ def infer(
self.write_uploaded_data(df, writeTag)

if readTag:
result = self.read_data(readTag)
result = self.read_data(readTag, camel_case=camelCase)

if reversed:
result = result[::-1]
Expand Down
95 changes: 90 additions & 5 deletions nbs/00_core.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,11 @@
"source": [
"#| export\n",
"import typing, logging\n",
"import os\n",
"import os, datetime\n",
"import numpy as np\n",
"import pandas as pd"
"import pandas as pd\n",
"\n",
"from fastcore.basics import patch_to, patch\n"
]
},
{
Expand Down Expand Up @@ -80,6 +82,20 @@
" logging.getLogger(name).info(f'There already is a logger installed for {name}.')\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
Expand Down Expand Up @@ -350,7 +366,7 @@
" df = pd.DataFrame.from_records(data)\n",
" time_columns_in_df = [C for C in df.columns if C in timecolumns]\n",
" if not time_columns_in_df:\n",
" syslog.error(f\"No column in records {df.columns} matches specification in time columns {timecolumns}, assuming first column is time\")\n",
" #syslog.error(f\"No column in records {df.columns} matches specification in time columns {timecolumns}, assuming first column is time\")\n",
" time_column = df.columns[0]\n",
" else:\n",
" time_column = time_columns_in_df[0]\n",
Expand Down Expand Up @@ -928,6 +944,75 @@
"timeseries_dataframe_resample(df, \"80min\", 'mean;count')"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# AICoreModuleBase"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#| exports\n",
"class AICoreModuleBase:\n",
" pass\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#| exports\n",
"@patch\n",
"def __init__(self:AICoreModuleBase, \n",
" save_dir:str, # path where the module can keep files \n",
" assets_dir:str, # path to support files (scripts, metadata, etc)\n",
" *args, **kwargs):\n",
" \n",
" self.init_time = datetime.datetime.now(datetime.UTC)\n",
"\n",
" self.init_args = args\n",
" self.init_kwargs = dict(\n",
" **kwargs,\n",
" assets_dir=assets_dir,\n",
" save_dir=save_dir\n",
" )\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
Expand All @@ -941,9 +1026,9 @@
],
"metadata": {
"kernelspec": {
"display_name": "corebridge.venv",
"display_name": "python3",
"language": "python",
"name": "corebridge.venv"
"name": "python3"
}
},
"nbformat": 4,
Expand Down
18 changes: 6 additions & 12 deletions nbs/01_aicorebridge.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -209,7 +209,7 @@
"outputs": [],
"source": [
"#| export\n",
"class AICoreModule(): pass"
"class AICoreModule(AICoreModuleBase): pass"
]
},
{
Expand Down Expand Up @@ -370,14 +370,14 @@
"metadata": {},
"outputs": [],
"source": [
"#| export\n",
"#| exports\n",
"# Specialized types for initializing annotated parameters\n",
"# Add types by adding a tuple with the type name and a builder function\n",
"annotated_arg_builders = {\n",
" str(B[0]):B[1] for B in [\n",
" (numpy.ndarray, lambda X: numpy.array(X, dtype=X.dtype))\n",
" ]\n",
"}"
"}\n"
]
},
{
Expand All @@ -386,7 +386,8 @@
"metadata": {},
"outputs": [],
"source": [
"#| export\n",
"\n",
"#| exports\n",
"\n",
"@patch\n",
"def init_annotated_param(self:AICoreModule, param_name, value):\n",
Expand Down Expand Up @@ -421,7 +422,7 @@
"metadata": {},
"outputs": [],
"source": [
"#| export\n",
"#| exports\n",
"@patch\n",
"def get_callargs(self:AICoreModule, kwargs, history):\n",
" \"Get arguments for the processor call\"\n",
Expand All @@ -447,13 +448,6 @@
" }\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
Expand Down
Loading

0 comments on commit 217b787

Please sign in to comment.