From 77e924ebedea217d568599d35bf6204cd0f32864 Mon Sep 17 00:00:00 2001 From: Nathan <95725385+treefern@users.noreply.github.com> Date: Tue, 7 Jan 2025 08:40:30 +0000 Subject: [PATCH 1/3] NPI-3676 added exception handling and unit test for (currently unsupported) SP3 velocity data output --- gnssanalysis/gn_io/sp3.py | 31 ++++++++++++++++++++----------- tests/test_sp3.py | 14 ++++++++++++++ 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/gnssanalysis/gn_io/sp3.py b/gnssanalysis/gn_io/sp3.py index 6c2ff50..8eedb71 100644 --- a/gnssanalysis/gn_io/sp3.py +++ b/gnssanalysis/gn_io/sp3.py @@ -642,6 +642,7 @@ def gen_sp3_content( sp3_df: _pd.DataFrame, sort_outputs: bool = False, buf: Union[None, _io.TextIOBase] = None, + continue_on_unhandled_velocity_data: bool = True, ) -> str: """ Organises, formats (including nodata values), then writes out SP3 content to a buffer if provided, or returns @@ -651,26 +652,34 @@ def gen_sp3_content( :param pandas.DataFrame sp3_df: The DataFrame containing the SP3 data. :param bool sort_outputs: Whether to sort the outputs. Defaults to False. :param io.TextIOBase buf: The buffer to write the SP3 content to. Defaults to None. + :param bool continue_on_unhandled_velocity_data: If (currently unsupported) velocity data exists in the DataFrame, + log a warning and skip velocity data, but write out position data. Set to false to raise an exception instead. :return str or None: The SP3 content if `buf` is None, otherwise None. """ - # TODO ensure we correctly handle outputting Velocity data! I.e. does this need to be interlaced back in, - # not printed as additional columns?! - # E.g. do we need: - # PG01... X Y Z CLK ... - # VG01... VX VY VZ ... - # - # Rather than: - # PG01... X Y Z CLK ... VX VY VZ ... - # ? - out_buf = buf if buf is not None else _io.StringIO() if sort_outputs: # If we need to do particular sorting/ordering of satellites and constellations we can use some of the # options that .sort_index() provides sp3_df = sp3_df.sort_index(ascending=True) out_df = sp3_df["EST"] - flags_df = sp3_df["FLAGS"] # Prediction, maneuver, etc. + + # Check for velocity columns (named by read_sp3() with a V prefix) + if any([col.startswith("V") for col in out_df.columns.values]): + if not continue_on_unhandled_velocity_data: + raise NotImplementedError("Output of SP3 velocity data not currently supported") + + # Drop any of the defined velocity columns that are present, so it doesn't mess up the output. + logger.warning("SP3 velocity output not currently supported! Dropping velocity columns before writing out.") + # Remove any defined velocity column we have, don't raise exceptions for defined vel columns we may not have: + out_df = out_df.drop(columns=SP3_VELOCITY_COLUMNS[1], errors="ignore") + + # NOTE: correctly writing velocity records would involve interlacing records, i.e.: + # PG01... X Y Z CLK ... + # VG01... VX VY VZ ... + + # Extract flags for Prediction, maneuver, etc. + flags_df = sp3_df["FLAGS"] # Valid values for the respective flags are 'E' 'P' 'M' 'P' (or blank), as per page 11-12 of the SP3d standard: # https://files.igs.org/pub/data/format/sp3d.pdf diff --git a/tests/test_sp3.py b/tests/test_sp3.py index 6db417c..2e7a468 100644 --- a/tests/test_sp3.py +++ b/tests/test_sp3.py @@ -148,6 +148,20 @@ def test_read_sp3_header_svs_detailed(self): # TODO Add test(s) for correctly reading header fundamentals (ACC, ORB_TYPE, etc.) # TODO add tests for correctly reading the actual content of the SP3 in addition to the header. # TODO add tests for correctly generating sp3 output content with gen_sp3_content() and gen_sp3_header() + + @patch("builtins.open", new_callable=mock_open, read_data=input_data) + def test_gen_sp3_content_velocity_exception_handling(self, mock_file): + """ + gen_sp3_content() velocity output should raise exception (currently unsupported).\ + If asked to continue with warning, it should remove velocity columns before output. + """ + sp3_df = sp3.read_sp3("mock_path", pOnly=False) + with self.assertRaises(NotImplementedError): + generated_sp3_content = sp3.gen_sp3_content(sp3_df, continue_on_unhandled_velocity_data=False) + + generated_sp3_content = sp3.gen_sp3_content(sp3_df, continue_on_unhandled_velocity_data=True) + self.assertTrue("VX" not in generated_sp3_content, "Velocity data should be removed before outputting SP3") + def test_sp3_clock_nodata_to_nan(self): sp3_df = pd.DataFrame( {("EST", "CLK"): [999999.999999, 123456.789, 999999.999999, 987654.321]} From a1358e4309ce089e883d3837547c9520158d8603 Mon Sep 17 00:00:00 2001 From: Nathan <95725385+treefern@users.noreply.github.com> Date: Tue, 7 Jan 2025 08:41:40 +0000 Subject: [PATCH 2/3] NPI-3676 improve robustness of test_sp3_merge() unit test by cleaning up pyfakefs, which can persist data between runs --- tests/test_sp3.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/test_sp3.py b/tests/test_sp3.py index 2e7a468..38ebf30 100644 --- a/tests/test_sp3.py +++ b/tests/test_sp3.py @@ -230,10 +230,22 @@ class TestMergeSP3(TestCase): def setUp(self): self.setUpPyfakefs() + # Not sure if this is helpful + def tearDown(self): + self.tearDownPyfakefs() + def test_sp3merge(self): + # Surprisingly, this reset step must be done explicitly. The fake filesystem is backed by the real one, and + # the temp directory used may retain files from a previous run! + self.fs.reset() + # Create some fake files file_paths = ["/fake/dir/file1.sp3", "/fake/dir/file2.sp3"] - self.fs.create_file(file_paths[0], contents=input_data) + # Note this fails if the fake file has previously been created in the fakefs (which does actually exist somewhere on the real filesystem) + self.fs.create_file( + file_paths[0], + contents=input_data, + ) self.fs.create_file(file_paths[1], contents=input_data2) # Call the function to test From ee7507a06e612fc2fa193e5b2d442abc759d76f0 Mon Sep 17 00:00:00 2001 From: Nathan <95725385+treefern@users.noreply.github.com> Date: Wed, 8 Jan 2025 02:25:08 +0000 Subject: [PATCH 3/3] NPI-3676 update docstrings in sp3.py for consistency, in response to PR comments --- gnssanalysis/gn_io/sp3.py | 49 +++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/gnssanalysis/gn_io/sp3.py b/gnssanalysis/gn_io/sp3.py index 8eedb71..7909471 100644 --- a/gnssanalysis/gn_io/sp3.py +++ b/gnssanalysis/gn_io/sp3.py @@ -290,8 +290,7 @@ def read_sp3( ) -> _pd.DataFrame: """Reads an SP3 file and returns the data as a pandas DataFrame. - - :param str sp3_path: The path to the SP3 file. + :param Union[str, Path, bytes] sp3_path_or_bytes: SP3 file path (as str or Path) or SP3 data as bytes object. :param bool pOnly: If True, only P* values (positions) are included in the DataFrame. Defaults to True. :param bool nodata_to_nan: If True, converts 0.000000 (indicating nodata) to NaN in the SP3 POS column and converts 999999* (indicating nodata) to NaN in the SP3 CLK column. Defaults to True. @@ -300,9 +299,9 @@ def read_sp3( :param bool continue_on_ep_ev_encountered: If True, logs a warning and continues if EV or EP rows are found in the input SP3. These are currently unsupported by this function and will be ignored. Set to false to raise a NotImplementedError instead. - :return pandas.DataFrame: The SP3 data as a DataFrame. - :raise FileNotFoundError: If the SP3 file specified by sp3_path_or_bytes does not exist. - :raise Exception: For other errors reading SP3 file/bytes + :return _pd.DataFrame: The SP3 data as a DataFrame. + :raises FileNotFoundError: If the SP3 file specified by sp3_path_or_bytes does not exist. + :raises Exception: For other errors reading SP3 file/bytes :note: The SP3 file format is a standard format used for representing precise satellite ephemeris and clock data. This function reads the SP3 file, parses the header information, and extracts the data into a DataFrame. @@ -396,7 +395,7 @@ def _reformat_df(sp3_df: _pd.DataFrame) -> _pd.DataFrame: """ Reformat the SP3 DataFrame for internal use - :param pandas.DataFrame sp3_df: The DataFrame containing the SP3 data. + :param _pd.DataFrame sp3_df: The DataFrame containing the SP3 data. :return _pd.DataFrame: reformated SP3 data as a DataFrame. """ name_float = [ @@ -437,7 +436,7 @@ def parse_sp3_header(header: bytes, warn_on_negative_sv_acc_values: bool = True) Parse the header of an SP3 file and extract relevant information. :param bytes header: The header of the SP3 file (as a byte string). - :return pandas.Series: A Series containing the parsed information from the SP3 header. + :return _pd.Series: A pandas Series containing the parsed information from the SP3 header. """ try: sp3_heading = _pd.Series( @@ -516,8 +515,8 @@ def parse_sp3_header(header: bytes, warn_on_negative_sv_acc_values: bool = True) def getVelSpline(sp3Df: _pd.DataFrame) -> _pd.DataFrame: """Returns the velocity spline of the input dataframe. - :param DataFrame sp3Df: The input dataframe containing position data. - :return DataFrame: The dataframe containing the velocity spline. + :param _pd.DataFrame sp3Df: The input pandas DataFrame containing SP3 position data. + :return _pd.DataFrame: The dataframe containing the velocity spline. :caution :This function cannot handle *any* NaN / nodata / non-finite position values. By contrast, getVelPoly() is more forgiving, but accuracy of results, particulary in the presence of NaNs, has not been assessed. @@ -538,9 +537,9 @@ def getVelPoly(sp3Df: _pd.DataFrame, deg: int = 35) -> _pd.DataFrame: """ Interpolates the positions for -1s and +1s in the sp3_df DataFrame and outputs velocities. - :param DataFrame sp3Df: A pandas DataFrame containing the sp3 data. + :param _pd.DataFrame sp3Df: A pandas DataFrame containing the sp3 data. :param int deg: Degree of the polynomial fit. Default is 35. - :return DataFrame: A pandas DataFrame with the interpolated velocities added as a new column. + :return _pd.DataFrame: A pandas DataFrame with the interpolated velocities added as a new column. """ est = sp3Df.unstack(1).EST[["X", "Y", "Z"]] @@ -579,7 +578,7 @@ def gen_sp3_header(sp3_df: _pd.DataFrame) -> str: """ Generate the header for an SP3 file based on the given DataFrame. - :param pandas.DataFrame sp3_df: The DataFrame containing the SP3 data. + :param _pd.DataFrame sp3_df: The DataFrame containing the SP3 data. :return str: The generated SP3 header as a string. """ sp3_j2000 = sp3_df.index.levels[0].values @@ -649,9 +648,9 @@ def gen_sp3_content( it otherwise. Args: - :param pandas.DataFrame sp3_df: The DataFrame containing the SP3 data. + :param _pd.DataFrame sp3_df: The DataFrame containing the SP3 data. :param bool sort_outputs: Whether to sort the outputs. Defaults to False. - :param io.TextIOBase buf: The buffer to write the SP3 content to. Defaults to None. + :param _io.TextIOBase buf: The buffer to write the SP3 content to. Defaults to None. :param bool continue_on_unhandled_velocity_data: If (currently unsupported) velocity data exists in the DataFrame, log a warning and skip velocity data, but write out position data. Set to false to raise an exception instead. :return str or None: The SP3 content if `buf` is None, otherwise None. @@ -849,9 +848,9 @@ def clk_std_formatter(x): def write_sp3(sp3_df: _pd.DataFrame, path: str) -> None: - """sp3 writer, dataframe to sp3 file + """Takes a DataFrame representation of SP3 data, formats and writes it out as an SP3 file at the given path. - :param pandas.DataFrame sp3_df: The DataFrame containing the SP3 data. + :param _pd.DataFrame sp3_df: The DataFrame containing the SP3 data. :param str path: The path to write the SP3 file to. """ content = gen_sp3_header(sp3_df) + gen_sp3_content(sp3_df) + "EOF" @@ -863,7 +862,7 @@ def merge_attrs(df_list: List[_pd.DataFrame]) -> _pd.Series: """Merges attributes of a list of sp3 dataframes into a single set of attributes. :param List[pd.DataFrame] df_list: The list of sp3 dataframes. - :return pd.Series: The merged attributes. + :return _pd.Series: The merged attributes. """ df = _pd.concat(list(map(lambda obj: obj.attrs["HEADER"], df_list)), axis=1) mask_mixed = ~_gn_aux.unique_cols(df.loc["HEAD"]) @@ -901,7 +900,7 @@ def sp3merge( :param Union[List[str], None] clkpaths: The list of paths to the clk files, or None if no clk files are provided. :param bool nodata_to_nan: Flag indicating whether to convert nodata values to NaN. - :return pd.DataFrame: The merged sp3 DataFrame. + :return _pd.DataFrame: The merged SP3 DataFrame. """ sp3_dfs = [read_sp3(sp3_file, nodata_to_nan=nodata_to_nan) for sp3_file in sp3paths] # Create a new attrs dictionary to be used for the output DataFrame @@ -921,10 +920,10 @@ def sp3_hlm_trans(a: _pd.DataFrame, b: _pd.DataFrame) -> tuple[_pd.DataFrame, li """ Rotates sp3_b into sp3_a. - :param DataFrame a: The sp3_a DataFrame. - :param DataFrame b : The sp3_b DataFrame. + :param _pd.DataFrame a: The sp3_a DataFrame. + :param _pd.DataFrame b: The sp3_b DataFrame. - :returntuple[pandas.DataFrame, list]: A tuple containing the updated sp3_b DataFrame and the HLM array with applied computed parameters and residuals. + :return tuple[_pd.DataFrame, list]: A tuple containing the updated sp3_b DataFrame and the HLM array with applied computed parameters and residuals. """ hlm = _gn_transform.get_helmert7(pt1=a.EST[["X", "Y", "Z"]].values, pt2=b.EST[["X", "Y", "Z"]].values) b.iloc[:, :3] = _gn_transform.transform7(xyz_in=b.EST[["X", "Y", "Z"]].values, hlm_params=hlm[0]) @@ -942,16 +941,16 @@ def diff_sp3_rac( """ Computes the difference between the two sp3 files in the radial, along-track and cross-track coordinates. - :param DataFrame sp3_baseline: The baseline sp3 DataFrame. - :param DataFrame sp3_test: The test sp3 DataFrame. - :param string hlm_mode: The mode for HLM transformation. Can be None, "ECF", or "ECI". + :param _pd.DataFrame sp3_baseline: The baseline sp3 DataFrame. + :param _pd.DataFrame sp3_test: The test sp3 DataFrame. + :param str hlm_mode: The mode for HLM transformation. Can be None, "ECF", or "ECI". :param bool use_cubic_spline: Flag indicating whether to use cubic spline for velocity computation. Caution: cubic spline interpolation does not tolerate NaN / nodata values. Consider enabling use_offline_sat_removal if using cubic spline, or alternatively use poly interpolation by setting use_cubic_spline to False. :param bool use_offline_sat_removal: Flag indicating whether to remove satellites which are offline / have some nodata position values. Caution: ensure you turn this on if using cubic spline interpolation with data which may have holes in it (nodata). - :return: The DataFrame containing the difference in RAC coordinates. + :return _pd.DataFrame: The DataFrame containing the difference in RAC coordinates. """ hlm_modes = [None, "ECF", "ECI"] if hlm_mode not in hlm_modes: