From 32f300f0d351358fc3b96a8a449011e42afd3184 Mon Sep 17 00:00:00 2001 From: Ronald Maj Date: Wed, 8 Jan 2025 11:50:43 +0000 Subject: [PATCH 1/7] fix: Errors in parse_igs_log() function discovered from quick-tests --- gnssanalysis/gn_io/igslog.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/gnssanalysis/gn_io/igslog.py b/gnssanalysis/gn_io/igslog.py index e769d1a..b0f41aa 100644 --- a/gnssanalysis/gn_io/igslog.py +++ b/gnssanalysis/gn_io/igslog.py @@ -173,11 +173,15 @@ def extract_id_block(data: bytes, file_path: str, file_code: str, version: str = if version == None: version = determine_log_version(data) + print(f"Version is {version}") + if version == "v1.0": _REGEX_ID = _REGEX_ID_V1 + print("Reached here") elif version == "v2.0": _REGEX_ID = _REGEX_ID_V2 else: + print("but also reached here") raise LogVersionError("Incorrect version string passed to the extract_id_block() function") id_block = _REGEX_ID.search(data) @@ -260,14 +264,15 @@ def parse_igs_log(filename_array: _np.ndarray) -> Union[_np.ndarray, None]: try: version = determine_log_version(data) + print(f"File found to be version: {version}") except LogVersionError as e: logger.warning(f"Error: {e}, skipping parsing the log file") return - blk_id = extract_id_block(data, version, file_path, file_code) - blk_loc = extract_location_block(data, version, file_path) - blk_rec = extract_receiver_block(data, file_path) - blk_ant = extract_antenna_block(data, file_path) + blk_id = extract_id_block(data = data, file_path=file_path, file_code=file_code, version=version) + blk_loc = extract_location_block(data=data, file_path=file_path, version=version,) + blk_rec = extract_receiver_block(data=data, file_path=file_path) + blk_ant = extract_antenna_block(data=data, file_path=file_path) blk_loc = [group.decode(encoding="utf8", errors="ignore") for group in blk_loc.groups()] blk_rec = _np.asarray(blk_rec, dtype=str) @@ -277,8 +282,8 @@ def parse_igs_log(filename_array: _np.ndarray) -> Union[_np.ndarray, None]: len_ants = blk_ant.shape[0] blk_id_loc = _np.asarray([0] + blk_id + blk_loc, dtype=object)[_np.newaxis] - - code = [code] + + code = [blk_id[0]] blk_rec = _np.concatenate( [ _np.asarray([1] * len_recs, dtype=object)[:, _np.newaxis], From a6667102b3cd0872b8c981e79e01c920df8c5a34 Mon Sep 17 00:00:00 2001 From: Ronald Maj Date: Wed, 8 Jan 2025 11:54:13 +0000 Subject: [PATCH 2/7] remove: Print statements used for debugging --- gnssanalysis/gn_io/igslog.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/gnssanalysis/gn_io/igslog.py b/gnssanalysis/gn_io/igslog.py index b0f41aa..e1502b8 100644 --- a/gnssanalysis/gn_io/igslog.py +++ b/gnssanalysis/gn_io/igslog.py @@ -173,15 +173,11 @@ def extract_id_block(data: bytes, file_path: str, file_code: str, version: str = if version == None: version = determine_log_version(data) - print(f"Version is {version}") - if version == "v1.0": _REGEX_ID = _REGEX_ID_V1 - print("Reached here") elif version == "v2.0": _REGEX_ID = _REGEX_ID_V2 else: - print("but also reached here") raise LogVersionError("Incorrect version string passed to the extract_id_block() function") id_block = _REGEX_ID.search(data) @@ -264,13 +260,16 @@ def parse_igs_log(filename_array: _np.ndarray) -> Union[_np.ndarray, None]: try: version = determine_log_version(data) - print(f"File found to be version: {version}") except LogVersionError as e: logger.warning(f"Error: {e}, skipping parsing the log file") return - blk_id = extract_id_block(data = data, file_path=file_path, file_code=file_code, version=version) - blk_loc = extract_location_block(data=data, file_path=file_path, version=version,) + blk_id = extract_id_block(data=data, file_path=file_path, file_code=file_code, version=version) + blk_loc = extract_location_block( + data=data, + file_path=file_path, + version=version, + ) blk_rec = extract_receiver_block(data=data, file_path=file_path) blk_ant = extract_antenna_block(data=data, file_path=file_path) @@ -282,7 +281,7 @@ def parse_igs_log(filename_array: _np.ndarray) -> Union[_np.ndarray, None]: len_ants = blk_ant.shape[0] blk_id_loc = _np.asarray([0] + blk_id + blk_loc, dtype=object)[_np.newaxis] - + code = [blk_id[0]] blk_rec = _np.concatenate( [ From 43d695d68915aaaed22f5c3a1dd3b72ab5bffc89 Mon Sep 17 00:00:00 2001 From: Ronald Maj Date: Thu, 9 Jan 2025 01:26:53 +0000 Subject: [PATCH 3/7] refactor: The parse_igs_log function to split into reading and parsing. add: Unit-tests to test parsing and gathering of files --- gnssanalysis/gn_io/aux_dicts.py | 3 +- gnssanalysis/gn_io/igslog.py | 67 ++++++++++++++++++++------------- tests/test_igslog.py | 62 ++++++++++++++++++++++++++++-- 3 files changed, 101 insertions(+), 31 deletions(-) diff --git a/gnssanalysis/gn_io/aux_dicts.py b/gnssanalysis/gn_io/aux_dicts.py index b3a232b..eafde57 100644 --- a/gnssanalysis/gn_io/aux_dicts.py +++ b/gnssanalysis/gn_io/aux_dicts.py @@ -56,7 +56,8 @@ "VIRGIN ISLANDS": "VIRGIN ISL", "US VIRGIN ISLANDS": "VIRGIN ISL", "WALLIS AND FUTUNA": "WALLIS", - "WEST ANTARCTICA": "W AFRICA", + "WEST ANTARCTICA": "ANTARCTICA", + "W ANTARCTICA": "ANTARCTICA", } translation_rec = { diff --git a/gnssanalysis/gn_io/igslog.py b/gnssanalysis/gn_io/igslog.py index e1502b8..e80da1e 100644 --- a/gnssanalysis/gn_io/igslog.py +++ b/gnssanalysis/gn_io/igslog.py @@ -16,6 +16,9 @@ logger = logging.getLogger(__name__) +_REGEX_LOG_VERSION_1 = _re.compile(rb"""(site log\))""") +_REGEX_LOG_VERSION_2 = _re.compile(rb"""(site log v2.0)""") + _REGEX_ID_V1 = _re.compile( rb""" (?:Four\sCharacter\sID|Site\sID)\s+\:\s*(\w{4}).*\W+ @@ -138,10 +141,6 @@ def find_recent_logs(logs_glob_path: str, rnx_glob_path: str = None) -> _pd.Data return recent_logs_df -_REGEX_VERSION_1 = _re.compile(rb"""(site log\))""") -_REGEX_VERSION_2 = _re.compile(rb"""(site log v2)""") - - def determine_log_version(data: bytes) -> str: """Given the byes object that results from reading an IGS log file, determine the version ("v1.0" or "v2.0") @@ -149,11 +148,11 @@ def determine_log_version(data: bytes) -> str: :return str: Return the version number: "v1.0" or "v2.0" (or "Unknown" if file does not conform to standard) """ - result_v1 = _REGEX_VERSION_1.search(data) + result_v1 = _REGEX_LOG_VERSION_1.search(data) if result_v1: return "v1.0" - result_v2 = _REGEX_VERSION_2.search(data) + result_v2 = _REGEX_LOG_VERSION_2.search(data) if result_v2: return "v2.0" @@ -247,42 +246,37 @@ def extract_antenna_block(data: bytes, file_path: str) -> Union[List[Tuple[bytes return antenna_block -def parse_igs_log(filename_array: _np.ndarray) -> Union[_np.ndarray, None]: - """Parses igs log and outputs ndarray with parsed data +def parse_igs_log_data(data: bytes, file_path: str, file_code: str) -> Union[_np.ndarray, None]: + """Given the bytes object returned opening a IGS log file, parse to produce an ndarray with relevant data - :param _np.ndarray filename_array: Metadata on input log file. Expects ndarray of the form [CODE DATE PATH] - :return _np.ndarray: Returns array with data from the IGS log file parsed + :param bytes data: The bytes object returned from an open() call on a IGS site log in "rb" mode + :param str file_path: The path to the file from which the "data" bytes object was obtained + :param str file_code: Code from the filename_array passed to the parse_igs_log() function + :return Union[_np.ndarray, None]: Returns array with relevant data from the IGS log file bytes object """ - file_code, _, file_path = filename_array - - with open(file_path, "rb") as file: - data = file.read() - + # Determine the version of the IGS log based on the data, Warn if unrecognised try: version = determine_log_version(data) except LogVersionError as e: logger.warning(f"Error: {e}, skipping parsing the log file") return + # Extract information from ID block blk_id = extract_id_block(data=data, file_path=file_path, file_code=file_code, version=version) + code = [blk_id[0]] # Site code + # Extract information from Location block blk_loc = extract_location_block( data=data, file_path=file_path, version=version, ) - blk_rec = extract_receiver_block(data=data, file_path=file_path) - blk_ant = extract_antenna_block(data=data, file_path=file_path) - blk_loc = [group.decode(encoding="utf8", errors="ignore") for group in blk_loc.groups()] + # Combine ID and Location information: + blk_id_loc = _np.asarray([0] + blk_id + blk_loc, dtype=object)[_np.newaxis] + # Extract and re-format information from receiver block: + blk_rec = extract_receiver_block(data=data, file_path=file_path) blk_rec = _np.asarray(blk_rec, dtype=str) - blk_ant = _np.asarray(blk_ant, dtype=str) - len_recs = blk_rec.shape[0] - len_ants = blk_ant.shape[0] - - blk_id_loc = _np.asarray([0] + blk_id + blk_loc, dtype=object)[_np.newaxis] - - code = [blk_id[0]] blk_rec = _np.concatenate( [ _np.asarray([1] * len_recs, dtype=object)[:, _np.newaxis], @@ -291,6 +285,10 @@ def parse_igs_log(filename_array: _np.ndarray) -> Union[_np.ndarray, None]: ], axis=1, ) + # Extract and re-format information from antenna block: + blk_ant = extract_antenna_block(data=data, file_path=file_path) + blk_ant = _np.asarray(blk_ant, dtype=str) + len_ants = blk_ant.shape[0] blk_ant = _np.concatenate( [ _np.asarray([2] * len_ants, dtype=object)[:, _np.newaxis], @@ -299,11 +297,26 @@ def parse_igs_log(filename_array: _np.ndarray) -> Union[_np.ndarray, None]: ], axis=1, ) + # Create unified information block: blk_uni = _np.concatenate([blk_id_loc, blk_rec, blk_ant], axis=0) file_path_arr = _np.asarray([file_path] * (1 + len_ants + len_recs))[:, _np.newaxis] return _np.concatenate([blk_uni, file_path_arr], axis=1) +def parse_igs_log_file(filename_array: _np.ndarray) -> Union[_np.ndarray, None]: + """Reads igs log file and outputs ndarray with parsed data + + :param _np.ndarray filename_array: Metadata on input log file. Expects ndarray of the form [CODE DATE PATH] + :return _np.ndarray: Returns array with data from the parsed IGS log file + """ + file_code, _, file_path = filename_array + + with open(file_path, "rb") as file: + data = file.read() + + return parse_igs_log_data(data=data, file_path=file_path, file_code=file_code) + + def igslogdate2datetime64(stacked_rec_ant_dt: _np.ndarray) -> _np.datetime64: """Function to convert datetimes for IGS log files to np.datetime64 objects, e.g. 2010-01-01T00:00 @@ -382,10 +395,10 @@ def gather_metadata( if num_threads == 1: gather = [] for file in parsed_filenames: - gather.append(parse_igs_log(file)) + gather.append(parse_igs_log_file(file)) else: with _Pool(processes=num_threads) as pool: - gather = list(pool.imap_unordered(parse_igs_log, parsed_filenames)) + gather = list(pool.imap_unordered(parse_igs_log_file, parsed_filenames)) gather_raw = _np.concatenate(gather) diff --git a/tests/test_igslog.py b/tests/test_igslog.py index 05c0487..4429075 100644 --- a/tests/test_igslog.py +++ b/tests/test_igslog.py @@ -1,12 +1,15 @@ import unittest -import numpy as _np -import pandas as _pd +from pyfakefs.fake_filesystem_unittest import TestCase from gnssanalysis.gn_io import igslog from test_datasets.sitelog_test_data import abmf_site_log_v1 as v1_data, abmf_site_log_v2 as v2_data -class Testregex(unittest.TestCase): +class TestRegex(unittest.TestCase): + """ + Test the various regex expressions used in the parsing of IGS log files + """ + def test_determine_log_version(self): # Ensure version 1 and 2 strings are produced as expected self.assertEqual(igslog.determine_log_version(v1_data), "v1.0") @@ -93,3 +96,56 @@ def test_extract_antenna_block(self): self.assertEqual(v2_antenna_block[0][8], b"2009-10-15T20:00Z") # Check end date of second entry # Last antenna should not have an end date assigned (i.e. current): self.assertEqual(v2_antenna_block[-1][-1], b"") + + +class TestDataParsing(unittest.TestCase): + """ + Test the integrated functions that gather and parse information from IGS log files + """ + + def test_parse_igs_log_data(self): + # Parse version 1 log file: + v1_data_parsed = igslog.parse_igs_log_data(data=v1_data, file_path="/example/path1", file_code="ABMF") + # Check country name: + self.assertEqual(v1_data_parsed[0][4], "Guadeloupe") + # Check last antenna type: + self.assertEqual(v1_data_parsed[-1][2], "TRM57971.00") + + # Parse version 2 log file: + v2_data_parsed = igslog.parse_igs_log_data(data=v2_data, file_path="/example/path2", file_code="ABMF") + # Check country name: + self.assertEqual(v2_data_parsed[0][4], "GLP") + # Check last antenna type: + self.assertEqual(v2_data_parsed[-1][2], "TRM57971.00") + + +class TestFileParsing(TestCase): + """ + Test gather_metadata() + """ + + def setUp(self): + self.setUpPyfakefs() + + def test_gather_metadata(self): + # Create some fake files + file_paths = ["/fake/dir/v1/abmf-v1.log", "/fake/dir/v2/abmf-v2.log"] + self.fs.create_file(file_paths[0], contents=v1_data) + self.fs.create_file(file_paths[1], contents=v2_data) + + # Call gather_metadata to grab log files - Version 2 file will be ignored - same station + result = igslog.gather_metadata(logs_glob_path="/fake/dir/*/*") + + # Test that various data has been read correctly: + # CODE + self.assertEqual(result[0].CODE[0], "ABMF") + # Country - V1 + self.assertEqual(result[0].COUNTRY[0], "GUADELOUPE") + # Last Receiver + self.assertEqual(result[1].RECEIVER[2], "SEPT POLARX5") + # Last Receiver start time: + self.assertEqual(result[1].BEGIN_RAW[2], "2019-10-01T16:00Z") + # Last Receiver end time: + self.assertEqual(result[1].END_RAW[2], "") + # First Antenna serial number: + self.assertEqual(result[2]["S/N"][0], "5546") From 7444622661342a94fbd9474a0de880c9857f5937 Mon Sep 17 00:00:00 2001 From: Ronald Maj Date: Thu, 9 Jan 2025 02:01:01 +0000 Subject: [PATCH 4/7] fix: Unit tests for gather_metadata function --- tests/test_datasets/sitelog_test_data.py | 163 +++++++++++++++++++++++ tests/test_igslog.py | 44 +++--- 2 files changed, 189 insertions(+), 18 deletions(-) diff --git a/tests/test_datasets/sitelog_test_data.py b/tests/test_datasets/sitelog_test_data.py index 42545ee..c42e72b 100644 --- a/tests/test_datasets/sitelog_test_data.py +++ b/tests/test_datasets/sitelog_test_data.py @@ -327,3 +327,166 @@ """, "utf-8", ) + +aggo_site_log_v2 = bytes( + """ + AGGO00ARG Site Information Form (site log v2.0) + International GNSS Service + See Instructions at: + https://files.igs.org/pub/station/general/sitelog_instr_v2.0.txt + +0. Form + + Prepared by (full name) : Thomas Fischer + Date Prepared : 2023-06-08 + Report Type : UPDATE + If Update: + Previous Site Log : (ssssmrccc_ccyymmdd.log) + Modified/Added Sections : (n.n,n.n,...) + + +1. Site Identification of the GNSS Monument + + Site Name : AGGO / Argentina + Nine Character ID : AGGO00ARG + Monument Inscription : Pillar + IERS DOMES Number : 41596M001 + CDP Number : AGGO + Monument Description : CONCRETE PILLAR + Height of the Monument : 4.0 m + Monument Foundation : CONCRETE BLOCK + Foundation Depth : 3.5 m + Marker Description : Pillar plate 14A + Date Installed : 2016-11-11T00:00Z + Geologic Characteristic : sedimentary basin + Bedrock Type : METAMORPHIC PRECAMBRIAN BASEMENT + Bedrock Condition : SEDIMENTS + Fracture Spacing : none + Fault zones nearby : No + Distance/activity : + Additional Information : Argentinean German Geodetic Observatory (AGGO) + : The pillar is insulated by an outer cylinder of + : concrete Pillar plate 14A - standard version + : (Goecke Schwelm) and semipherical vertical + : reference marker next to pillar plate + : Metamorphic Precambrian basement, lower + : Cretaceous and upper Jurassic rocks (the maximum + : sedimentary thickness is 6500m to 7000m) + + +2. Site Location Information + + City or Town : La Plata + State or Province : Province of Buenos Aires + Country or Region : ARG + Tectonic Plate : SOUTH AMERICAN + Approximate Position (ITRF) + X coordinate (m) : 2765120.9 + Y coordinate (m) : -4449250.25 + Z coordinate (m) : -3626405.6 + Latitude (N is +) : -345225.35 + Longitude (E is +) : -0580823.50 + Elevation (m,ellips.) : 42.1 + Additional Information : + + +3. GNSS Receiver Information + +3.1 Receiver Type : SEPT POLARX4TR + Satellite System : GPS+GLO+GAL+BDS+SBAS + Serial Number : 3002049 + Firmware Version : 2.9.6 + Elevation Cutoff Setting : 0 deg + Date Installed : 2016-11-11T10:45Z + Date Removed : 2018-12-06T20:35Z + Temperature Stabiliz. : 5.0 + Additional Information : + +3.2 Receiver Type : SEPT POLARX5TR + Satellite System : GPS+GLO+GAL+BDS+SBAS + Serial Number : 3228290 + Firmware Version : 5.4.0 + Elevation Cutoff Setting : 0 deg + Date Installed : 2018-12-06T20:40Z + Date Removed : (CCYY-MM-DDThh:mmZ) + Temperature Stabiliz. : none + Additional Information : Elimination of the IRNSS system due to lack of + : visibility on 09-JAN-2022 + +3.x Receiver Type : (A20, from rcvr_ant.tab; see instructions) + Satellite System : (GPS+GLO+GAL+BDS+QZSS+SBAS) + Serial Number : (A20, but note the first A5 is used in SINEX) + Firmware Version : (A11) + Elevation Cutoff Setting : (deg) + Date Installed : (CCYY-MM-DDThh:mmZ) + Date Removed : (CCYY-MM-DDThh:mmZ) + Temperature Stabiliz. : (none or tolerance in degrees C) + Additional Information : (multiple lines) + + +4. GNSS Antenna Information + +4.1 Antenna Type : LEIAR25.R4 LEIT + Serial Number : 726722 + Antenna Reference Point : BPA + Marker->ARP Up Ecc. (m) : 000.1550 + Marker->ARP North Ecc(m) : 000.0000 + Marker->ARP East Ecc(m) : 000.0000 + Alignment from True N : 0 deg + Antenna Radome Type : LEIT + Radome Serial Number : + Antenna Cable Type : Nokia Cable M17/75-RG214 + Antenna Cable Length : 60.0 m + Date Installed : 2016-11-11T10:30Z + Date Removed : 2021-06-11T18:30Z + Additional Information : Antenna and radome calibrated by Geo+++ GmbH, + : 2013-11-22. antenna height refering to vertical + : reference marker at pillar + +4.2 Antenna Type : LEIAR25.R4 LEIT + Serial Number : 726722 + Antenna Reference Point : BPA + Marker->ARP Up Ecc. (m) : 000.1550 + Marker->ARP North Ecc(m) : 000.0000 + Marker->ARP East Ecc(m) : 000.0000 + Alignment from True N : 0 deg + Antenna Radome Type : LEIT + Radome Serial Number : + Antenna Cable Type : EcoFlex 10 Cable 50 ohms + Antenna Cable Length : 50.0 m + Date Installed : 2021-06-11T18:30Z + Date Removed : 2022-10-11T13:30Z + Additional Information : Antenna cable replaced + +4.3 Antenna Type : LEIAR25.R4 LEIT + Serial Number : 726722 + Antenna Reference Point : BPA + Marker->ARP Up Ecc. (m) : 000.4100 + Marker->ARP North Ecc(m) : 000.0000 + Marker->ARP East Ecc(m) : 000.0000 + Alignment from True N : 0 deg + Antenna Radome Type : LEIT + Radome Serial Number : + Antenna Cable Type : EcoFlex 10 Cable 50 ohms + Antenna Cable Length : 50.0 m + Date Installed : 2022-10-11T13:30Z + Date Removed : (CCYY-MM-DDThh:mmZ) + Additional Information : Antenna height corrected from 0.4400 m to 0.4100 + +4.x Antenna Type : (A20, from rcvr_ant.tab; see instructions) + Serial Number : (A*, but note the first A5 is used in SINEX) + Antenna Reference Point : (BPA/BCR/XXX from "antenna.gra"; see instr.) + Marker->ARP Up Ecc. (m) : (F8.4) + Marker->ARP North Ecc(m) : (F8.4) + Marker->ARP East Ecc(m) : (F8.4) + Alignment from True N : (deg; + is clockwise/east) + Antenna Radome Type : (A4 from rcvr_ant.tab; see instructions) + Radome Serial Number : + Antenna Cable Type : (vendor & type number) + Antenna Cable Length : (m) + Date Installed : (CCYY-MM-DDThh:mmZ) + Date Removed : (CCYY-MM-DDThh:mmZ) + Additional Information : (multiple lines) + """, + "utf-8", +) diff --git a/tests/test_igslog.py b/tests/test_igslog.py index 4429075..788c705 100644 --- a/tests/test_igslog.py +++ b/tests/test_igslog.py @@ -2,7 +2,11 @@ from pyfakefs.fake_filesystem_unittest import TestCase from gnssanalysis.gn_io import igslog -from test_datasets.sitelog_test_data import abmf_site_log_v1 as v1_data, abmf_site_log_v2 as v2_data +from test_datasets.sitelog_test_data import ( + abmf_site_log_v1 as v1_data, + abmf_site_log_v2 as v2_data, + aggo_site_log_v2 as aggo_v2_data, +) class TestRegex(unittest.TestCase): @@ -129,23 +133,27 @@ def setUp(self): def test_gather_metadata(self): # Create some fake files - file_paths = ["/fake/dir/v1/abmf-v1.log", "/fake/dir/v2/abmf-v2.log"] - self.fs.create_file(file_paths[0], contents=v1_data) - self.fs.create_file(file_paths[1], contents=v2_data) + file_paths = ["/fake/dir/abmf.log", "/fake/dir/aggo.log"] + self.fs.create_file(file_paths[0], contents=v2_data) + self.fs.create_file(file_paths[1], contents=aggo_v2_data) - # Call gather_metadata to grab log files - Version 2 file will be ignored - same station - result = igslog.gather_metadata(logs_glob_path="/fake/dir/*/*") + # Call gather_metadata to grab log files for two stations + result = igslog.gather_metadata(logs_glob_path="/fake/dir/*") # Test that various data has been read correctly: - # CODE - self.assertEqual(result[0].CODE[0], "ABMF") - # Country - V1 - self.assertEqual(result[0].COUNTRY[0], "GUADELOUPE") - # Last Receiver - self.assertEqual(result[1].RECEIVER[2], "SEPT POLARX5") - # Last Receiver start time: - self.assertEqual(result[1].BEGIN_RAW[2], "2019-10-01T16:00Z") - # Last Receiver end time: - self.assertEqual(result[1].END_RAW[2], "") - # First Antenna serial number: - self.assertEqual(result[2]["S/N"][0], "5546") + # ID/Location Info: test CODE and Country / region + id_loc_results = result[0] + self.assertEqual(id_loc_results.CODE[0], "ABMF") + self.assertEqual(id_loc_results.COUNTRY[0], "GLP") + self.assertEqual(id_loc_results.CODE[1], "AGGO") + self.assertEqual(id_loc_results.COUNTRY[1], "ARG") + # Receiver info: test a couple receivers + receiver_results = result[1] + record_0 = receiver_results.loc[0] + record_3 = receiver_results.loc[3] + self.assertEqual(record_0.RECEIVER, "LEICA GR25") + self.assertEqual(record_0.END_RAW, "2019-04-15T12:00Z") + self.assertEqual(record_3.RECEIVER, "SEPT POLARX4TR") + self.assertEqual(record_3.CODE, "AGGO") + # Antenna info: test for antenna serial number + self.assertEqual(result[2]["S/N"][4], "726722") From 00619a9e0bae8ea52e0ac328acd9d20d9b702b54 Mon Sep 17 00:00:00 2001 From: Ronald Maj Date: Thu, 9 Jan 2025 02:04:12 +0000 Subject: [PATCH 5/7] add: Minor comments to log test datasets --- tests/test_datasets/sitelog_test_data.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_datasets/sitelog_test_data.py b/tests/test_datasets/sitelog_test_data.py index c42e72b..2dbef93 100644 --- a/tests/test_datasets/sitelog_test_data.py +++ b/tests/test_datasets/sitelog_test_data.py @@ -1,6 +1,6 @@ # Central record of IGS site log test data sets to be shared across unit tests -# first dataset is a truncated version of file abmf_20240710.log +# Dataset below is a truncated version of file abmf_20240710.log abmf_site_log_v1 = bytes( """ @@ -165,6 +165,8 @@ "utf-8", ) +# Dataset below is a truncated version of file abmf00glp_20240710.log + abmf_site_log_v2 = bytes( """ ABMF00GLP Site Information Form (site log v2.0) @@ -328,6 +330,8 @@ "utf-8", ) +# Dataset below is a truncated version of file aggo00arg_20230608.log + aggo_site_log_v2 = bytes( """ AGGO00ARG Site Information Form (site log v2.0) From 3272712786a90e3bc1ce0bb7bc82e5a67a1a0107 Mon Sep 17 00:00:00 2001 From: Ronald Maj Date: Thu, 9 Jan 2025 02:08:04 +0000 Subject: [PATCH 6/7] add: Additional comments on supported versions of IGS logs files, including reference links --- gnssanalysis/gn_io/igslog.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/gnssanalysis/gn_io/igslog.py b/gnssanalysis/gn_io/igslog.py index e80da1e..c3ed733 100644 --- a/gnssanalysis/gn_io/igslog.py +++ b/gnssanalysis/gn_io/igslog.py @@ -16,6 +16,11 @@ logger = logging.getLogger(__name__) +# Defines what IGS Site Log format versions we currently support. +# Example logs for the first two versions can be found at: +# Version 1: https://files.igs.org/pub/station/general/blank.log +# Version 2: https://files.igs.org/pub/station/general/blank_v2.0.log + _REGEX_LOG_VERSION_1 = _re.compile(rb"""(site log\))""") _REGEX_LOG_VERSION_2 = _re.compile(rb"""(site log v2.0)""") From 651437f441ef87615c9a87e1c72a8e210ebca543 Mon Sep 17 00:00:00 2001 From: Ronald Maj <36747983+ronaldmaj@users.noreply.github.com> Date: Thu, 9 Jan 2025 14:07:59 +1100 Subject: [PATCH 7/7] Apply suggestions from code review Fixing various type hints and adding helpful comments Co-authored-by: Nathan <95725385+treefern@users.noreply.github.com> --- gnssanalysis/gn_io/igslog.py | 9 ++++++--- tests/test_igslog.py | 1 + 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/gnssanalysis/gn_io/igslog.py b/gnssanalysis/gn_io/igslog.py index c3ed733..adea66a 100644 --- a/gnssanalysis/gn_io/igslog.py +++ b/gnssanalysis/gn_io/igslog.py @@ -257,14 +257,15 @@ def parse_igs_log_data(data: bytes, file_path: str, file_code: str) -> Union[_np :param bytes data: The bytes object returned from an open() call on a IGS site log in "rb" mode :param str file_path: The path to the file from which the "data" bytes object was obtained :param str file_code: Code from the filename_array passed to the parse_igs_log() function - :return Union[_np.ndarray, None]: Returns array with relevant data from the IGS log file bytes object + :return Union[_np.ndarray, None]: Returns array with relevant data from the IGS log file bytes object, + or `None` for unsupported version of the IGS Site log format. """ # Determine the version of the IGS log based on the data, Warn if unrecognised try: version = determine_log_version(data) except LogVersionError as e: logger.warning(f"Error: {e}, skipping parsing the log file") - return + return None # Extract information from ID block blk_id = extract_id_block(data=data, file_path=file_path, file_code=file_code, version=version) @@ -312,8 +313,10 @@ def parse_igs_log_file(filename_array: _np.ndarray) -> Union[_np.ndarray, None]: """Reads igs log file and outputs ndarray with parsed data :param _np.ndarray filename_array: Metadata on input log file. Expects ndarray of the form [CODE DATE PATH] - :return _np.ndarray: Returns array with data from the parsed IGS log file + :return Union[_np.ndarray, None]: Returns array with data from the parsed IGS log file, or `None` for unsupported + version of the IGS Site log format. """ + # Split filename_array out into its three components (CODE, DATE, PATH), discarding the second element (DATE): file_code, _, file_path = filename_array with open(file_path, "rb") as file: diff --git a/tests/test_igslog.py b/tests/test_igslog.py index 788c705..5b8e5da 100644 --- a/tests/test_igslog.py +++ b/tests/test_igslog.py @@ -132,6 +132,7 @@ def setUp(self): self.setUpPyfakefs() def test_gather_metadata(self): + self.fs.reset() # Ensure fake filesystem is cleared from any previous tests, as it is backed by real filesystem. # Create some fake files file_paths = ["/fake/dir/abmf.log", "/fake/dir/aggo.log"] self.fs.create_file(file_paths[0], contents=v2_data)