Skip to content

Commit

Permalink
Port over cdp_viewer.OutputViewer and remove cdp dependency (#773)
Browse files Browse the repository at this point in the history
  • Loading branch information
tomvothecoder authored Jan 3, 2024
1 parent c94715c commit c8ffba9
Show file tree
Hide file tree
Showing 18 changed files with 151 additions and 20 deletions.
2 changes: 1 addition & 1 deletion conda-env/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ dependencies:
- beautifulsoup4
- cartopy >=0.17.0
- cartopy_offlinedata
- cdp 1.7.0
- cdms2 3.1.5
- cdutil 8.2.1
- dask
Expand All @@ -23,6 +22,7 @@ dependencies:
- mache >=0.15.0
- matplotlib-base
- netcdf4
- output_viewer >=1.3.0
- numpy >=1.23.0
- shapely >=2.0.0,<3.0.0
- xarray >=2023.02.0
Expand Down
2 changes: 1 addition & 1 deletion conda-env/dev-nompi.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ dependencies:
- beautifulsoup4
- cartopy >=0.17.0
- cartopy_offlinedata
- cdp 1.7.0
- cdms2 3.1.5
- cdutil 8.2.1
- dask
Expand All @@ -26,6 +25,7 @@ dependencies:
- mache >=0.15.0
- matplotlib-base
- netcdf4
- output_viewer >=1.3.0
- numpy >=1.23.0
- shapely >=2.0.0,<3.0.0
- xarray >=2023.02.0
Expand Down
2 changes: 1 addition & 1 deletion conda-env/dev.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ dependencies:
- beautifulsoup4
- cartopy >=0.17.0
- cartopy_offlinedata
- cdp 1.7.0
- cdms2 3.1.5
- cdutil 8.2.1
- dask
Expand All @@ -21,6 +20,7 @@ dependencies:
- mache >=0.15.0
- matplotlib-base
- netcdf4
- output_viewer >=1.3.0
- numpy >=1.23.0
- shapely >=2.0.0,<3.0.0
- xarray >=2023.02.0
Expand Down
2 changes: 1 addition & 1 deletion docs/source/dev_guide/adding-new-diags-sets.rst
Original file line number Diff line number Diff line change
Expand Up @@ -616,7 +616,7 @@ create a file ``diff_diags_viewer.py`` paste in the below code.
import os
from .utils import add_header, h1_to_h3
from .default_viewer import create_metadata
from cdp.cdp_viewer import OutputViewer
from e3sm_diags.viewer.core_viewer import OutputViewer
def create_viewer(root_dir, parameters):
Expand Down
2 changes: 1 addition & 1 deletion docs/source/dev_guide/using-cdp-output-viewer.rst
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ The code below was used to create the figures above.

.. code:: python
from cdp.cdp_viewer import OutputViewer
from e3sm_diags.viewer.core_viewer import OutputViewer
viewer = OutputViewer(index_name='My Cool Results')
viewer.add_page("My Results", ['Description', 'Generated File'])
Expand Down
3 changes: 2 additions & 1 deletion e3sm_diags/e3sm_diags_driver.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
from typing import Dict, List, Tuple

import dask
import dask.bag as db

import e3sm_diags
from e3sm_diags.logger import custom_logger
Expand Down Expand Up @@ -300,7 +301,7 @@ def _run_with_dask(parameters: List[CoreParameter]) -> List[CoreParameter]:
https://docs.dask.org/en/stable/generated/dask.bag.map.html
https://docs.dask.org/en/stable/generated/dask.dataframe.DataFrame.compute.html
"""
bag = dask.bag.from_sequence(parameters)
bag = db.from_sequence(parameters)
config = {"scheduler": "processes", "multiprocessing.context": "fork"}

num_workers = getattr(parameters[0], "num_workers", None)
Expand Down
2 changes: 1 addition & 1 deletion e3sm_diags/viewer/aerosol_budget_viewer.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import os
import shutil

from cdp.cdp_viewer import OutputViewer
from e3sm_diags.viewer.core_viewer import OutputViewer

from .default_viewer import seasons_used
from .lat_lon_viewer import _cvs_to_html
Expand Down
2 changes: 1 addition & 1 deletion e3sm_diags/viewer/annual_cycle_zonal_mean_viewer.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import os

from cdp.cdp_viewer import OutputViewer
from e3sm_diags.viewer.core_viewer import OutputViewer

from .default_viewer import create_metadata
from .utils import add_header, h1_to_h3
Expand Down
2 changes: 1 addition & 1 deletion e3sm_diags/viewer/area_mean_time_series_viewer.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import os

from cdp.cdp_viewer import OutputViewer
from e3sm_diags.viewer.core_viewer import OutputViewer

from .default_viewer import create_metadata
from .utils import add_header, h1_to_h3
Expand Down
2 changes: 1 addition & 1 deletion e3sm_diags/viewer/arm_diags_viewer.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import os

from cdp.cdp_viewer import OutputViewer
from e3sm_diags.viewer.core_viewer import OutputViewer

from .utils import add_header, h1_to_h3

Expand Down
133 changes: 133 additions & 0 deletions e3sm_diags/viewer/core_viewer.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
import os
import stat

from output_viewer.build import build_page, build_viewer
from output_viewer.index import (
OutputFile,
OutputGroup,
OutputIndex,
OutputPage,
OutputRow,
)
from output_viewer.utils import rechmod


class OutputViewer(object):
def __init__(self, path=".", index_name="Results"):
self.path = os.path.abspath(path)
self.index = OutputIndex(index_name)
self.cache = {} # dict of { OutputPage: { OutputGroup: [OutputRow] } }
self.page = None
self.group = None
self.row = None

def add_page(self, page_title, *args, **kwargs):
"""Add a page to the viewer's index"""
self.page = OutputPage(page_title, *args, **kwargs)
self.cache[self.page] = {}
self.index.addPage(self.page)

def set_page(self, page_title):
"""Sets the page with the title name as the current page"""
for output_page in self.cache:
if page_title == output_page.title:
self.page = output_page
return
raise RuntimeError("There is no page titled: %s" % page_title)

def add_group(self, group_name):
"""Add a group to the current page"""
if self.page is None:
raise RuntimeError("You must first insert a page with add_page()")
self.group = OutputGroup(group_name)
if self.group not in self.cache[self.page]:
self.cache[self.page][self.group] = [] # group doesn't have any rows yet
self.page.addGroup(self.group)

def set_group(self, group_name):
"""Sets the group with the title name as the current group"""
for output_group in self.cache[self.page]:
if group_name == output_group.title:
self.group = output_group
return
raise RuntimeError("There is no group titled: %s" % group_name)

def add_row(self, row_name):
"""Add a row with the title name to the current group"""
if self.group is None:
raise RuntimeError("You must first insert a group with add_group()")
self.row = OutputRow(row_name, [])
if self.row not in self.cache[self.page][self.group]:
self.cache[self.page][self.group].append(self.row)
self.page.addRow(self.row, len(self.page.groups) - 1) # type: ignore

def set_row(self, row_name):
"""Sets the row with the title name as the current row"""
for output_row in self.cache[self.page][self.group]:
if row_name == output_row.title:
self.row = output_row
return
raise RuntimeError("There is no row titled: %s" % row_name)

def add_cols(self, cols):
"""Add multiple string cols to the current row"""
self.row.columns.append(cols) # type: ignore

def add_col(self, col, is_file=False, **kwargs):
"""Add a single col to the current row. Set is_file to True if the col is a file path."""
if is_file:
self.row.columns.append(OutputFile(col, **kwargs)) # type: ignore
else:
self.row.columns.append(col) # type: ignore

def generate_page(self):
"""
Generate and return the location of the current HTML page.
"""
self.index.toJSON(os.path.join(self.path, "index.json"))

default_mask = stat.S_IMODE(os.stat(self.path).st_mode)
rechmod(self.path, default_mask)

if os.access(self.path, os.W_OK):
default_mask = stat.S_IMODE(
os.stat(self.path).st_mode
) # mode of files to be included
url = build_page(
self.page,
os.path.join(self.path, "index.json"),
default_mask=default_mask,
)
return url

raise RuntimeError("Error geneating the page.")

def generate_viewer(self, prompt_user=True):
"""Generate the webpage and ask the user if they want to see it."""
self.index.toJSON(os.path.join(self.path, "index.json"))

default_mask = stat.S_IMODE(os.stat(self.path).st_mode)
rechmod(self.path, default_mask)

if os.access(self.path, os.W_OK):
default_mask = stat.S_IMODE(
os.stat(self.path).st_mode
) # mode of files to be included
build_viewer(
os.path.join(self.path, "index.json"),
diag_name="My Diagnostics",
default_mask=default_mask,
)

if os.path.exists(os.path.join(self.path, "index.html")):
if prompt_user:
user_prompt = "Would you like to open in a browser? y/[n]: "
should_open = input(user_prompt)
if should_open and should_open.lower()[0] == "y":
import webbrowser

index_path = os.path.join(self.path, "index.html")
url = "file://{path}".format(path=index_path)
webbrowser.open(url)
else:
raise RuntimeError("Failed to generate the viewer.")
2 changes: 1 addition & 1 deletion e3sm_diags/viewer/default_viewer.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,10 @@
from typing import Dict

import numpy
from cdp.cdp_viewer import OutputViewer

from e3sm_diags.logger import custom_logger
from e3sm_diags.parser import SET_TO_PARSER
from e3sm_diags.viewer.core_viewer import OutputViewer

from . import lat_lon_viewer, utils

Expand Down
3 changes: 1 addition & 2 deletions e3sm_diags/viewer/enso_diags_viewer.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,8 @@
import os
from typing import Dict, List

from cdp.cdp_viewer import OutputViewer

from e3sm_diags.logger import custom_logger
from e3sm_diags.viewer.core_viewer import OutputViewer

from .default_viewer import create_metadata
from .utils import add_header, h1_to_h3
Expand Down
2 changes: 1 addition & 1 deletion e3sm_diags/viewer/mean_2d_viewer.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import os

from cdp.cdp_viewer import OutputViewer
from e3sm_diags.viewer.core_viewer import OutputViewer

from .default_viewer import SEASONS, create_metadata, seasons_used
from .utils import add_header, h1_to_h3
Expand Down
3 changes: 1 addition & 2 deletions e3sm_diags/viewer/mp_partition_viewer.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
import os

from cdp.cdp_viewer import OutputViewer

from e3sm_diags.logger import custom_logger
from e3sm_diags.viewer.core_viewer import OutputViewer

from .default_viewer import create_metadata
from .utils import add_header, h1_to_h3
Expand Down
3 changes: 1 addition & 2 deletions e3sm_diags/viewer/qbo_viewer.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
import os

from cdp.cdp_viewer import OutputViewer

from e3sm_diags.logger import custom_logger
from e3sm_diags.viewer.core_viewer import OutputViewer

from .default_viewer import create_metadata
from .utils import add_header, h1_to_h3
Expand Down
2 changes: 1 addition & 1 deletion e3sm_diags/viewer/streamflow_viewer.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import os

from cdp.cdp_viewer import OutputViewer
from e3sm_diags.viewer.core_viewer import OutputViewer

from .default_viewer import create_metadata
from .utils import add_header, h1_to_h3
Expand Down
2 changes: 1 addition & 1 deletion e3sm_diags/viewer/tc_analysis_viewer.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import os

from cdp.cdp_viewer import OutputViewer
from e3sm_diags.viewer.core_viewer import OutputViewer

from .utils import add_header, h1_to_h3

Expand Down

0 comments on commit c8ffba9

Please sign in to comment.