diff --git a/gnssanalysis/gn_io/aux_dicts.py b/gnssanalysis/gn_io/aux_dicts.py index b3a232b..eafde57 100644 --- a/gnssanalysis/gn_io/aux_dicts.py +++ b/gnssanalysis/gn_io/aux_dicts.py @@ -56,7 +56,8 @@ "VIRGIN ISLANDS": "VIRGIN ISL", "US VIRGIN ISLANDS": "VIRGIN ISL", "WALLIS AND FUTUNA": "WALLIS", - "WEST ANTARCTICA": "W AFRICA", + "WEST ANTARCTICA": "ANTARCTICA", + "W ANTARCTICA": "ANTARCTICA", } translation_rec = { diff --git a/gnssanalysis/gn_io/igslog.py b/gnssanalysis/gn_io/igslog.py index e769d1a..adea66a 100644 --- a/gnssanalysis/gn_io/igslog.py +++ b/gnssanalysis/gn_io/igslog.py @@ -16,6 +16,14 @@ logger = logging.getLogger(__name__) +# Defines what IGS Site Log format versions we currently support. +# Example logs for the first two versions can be found at: +# Version 1: https://files.igs.org/pub/station/general/blank.log +# Version 2: https://files.igs.org/pub/station/general/blank_v2.0.log + +_REGEX_LOG_VERSION_1 = _re.compile(rb"""(site log\))""") +_REGEX_LOG_VERSION_2 = _re.compile(rb"""(site log v2.0)""") + _REGEX_ID_V1 = _re.compile( rb""" (?:Four\sCharacter\sID|Site\sID)\s+\:\s*(\w{4}).*\W+ @@ -138,10 +146,6 @@ def find_recent_logs(logs_glob_path: str, rnx_glob_path: str = None) -> _pd.Data return recent_logs_df -_REGEX_VERSION_1 = _re.compile(rb"""(site log\))""") -_REGEX_VERSION_2 = _re.compile(rb"""(site log v2)""") - - def determine_log_version(data: bytes) -> str: """Given the byes object that results from reading an IGS log file, determine the version ("v1.0" or "v2.0") @@ -149,11 +153,11 @@ def determine_log_version(data: bytes) -> str: :return str: Return the version number: "v1.0" or "v2.0" (or "Unknown" if file does not conform to standard) """ - result_v1 = _REGEX_VERSION_1.search(data) + result_v1 = _REGEX_LOG_VERSION_1.search(data) if result_v1: return "v1.0" - result_v2 = _REGEX_VERSION_2.search(data) + result_v2 = _REGEX_LOG_VERSION_2.search(data) if result_v2: return "v2.0" @@ -247,38 +251,38 @@ def extract_antenna_block(data: bytes, file_path: str) -> Union[List[Tuple[bytes return antenna_block -def parse_igs_log(filename_array: _np.ndarray) -> Union[_np.ndarray, None]: - """Parses igs log and outputs ndarray with parsed data +def parse_igs_log_data(data: bytes, file_path: str, file_code: str) -> Union[_np.ndarray, None]: + """Given the bytes object returned opening a IGS log file, parse to produce an ndarray with relevant data - :param _np.ndarray filename_array: Metadata on input log file. Expects ndarray of the form [CODE DATE PATH] - :return _np.ndarray: Returns array with data from the IGS log file parsed + :param bytes data: The bytes object returned from an open() call on a IGS site log in "rb" mode + :param str file_path: The path to the file from which the "data" bytes object was obtained + :param str file_code: Code from the filename_array passed to the parse_igs_log() function + :return Union[_np.ndarray, None]: Returns array with relevant data from the IGS log file bytes object, + or `None` for unsupported version of the IGS Site log format. """ - file_code, _, file_path = filename_array - - with open(file_path, "rb") as file: - data = file.read() - + # Determine the version of the IGS log based on the data, Warn if unrecognised try: version = determine_log_version(data) except LogVersionError as e: logger.warning(f"Error: {e}, skipping parsing the log file") - return - - blk_id = extract_id_block(data, version, file_path, file_code) - blk_loc = extract_location_block(data, version, file_path) - blk_rec = extract_receiver_block(data, file_path) - blk_ant = extract_antenna_block(data, file_path) - + return None + + # Extract information from ID block + blk_id = extract_id_block(data=data, file_path=file_path, file_code=file_code, version=version) + code = [blk_id[0]] # Site code + # Extract information from Location block + blk_loc = extract_location_block( + data=data, + file_path=file_path, + version=version, + ) blk_loc = [group.decode(encoding="utf8", errors="ignore") for group in blk_loc.groups()] + # Combine ID and Location information: + blk_id_loc = _np.asarray([0] + blk_id + blk_loc, dtype=object)[_np.newaxis] + # Extract and re-format information from receiver block: + blk_rec = extract_receiver_block(data=data, file_path=file_path) blk_rec = _np.asarray(blk_rec, dtype=str) - blk_ant = _np.asarray(blk_ant, dtype=str) - len_recs = blk_rec.shape[0] - len_ants = blk_ant.shape[0] - - blk_id_loc = _np.asarray([0] + blk_id + blk_loc, dtype=object)[_np.newaxis] - - code = [code] blk_rec = _np.concatenate( [ _np.asarray([1] * len_recs, dtype=object)[:, _np.newaxis], @@ -287,6 +291,10 @@ def parse_igs_log(filename_array: _np.ndarray) -> Union[_np.ndarray, None]: ], axis=1, ) + # Extract and re-format information from antenna block: + blk_ant = extract_antenna_block(data=data, file_path=file_path) + blk_ant = _np.asarray(blk_ant, dtype=str) + len_ants = blk_ant.shape[0] blk_ant = _np.concatenate( [ _np.asarray([2] * len_ants, dtype=object)[:, _np.newaxis], @@ -295,11 +303,28 @@ def parse_igs_log(filename_array: _np.ndarray) -> Union[_np.ndarray, None]: ], axis=1, ) + # Create unified information block: blk_uni = _np.concatenate([blk_id_loc, blk_rec, blk_ant], axis=0) file_path_arr = _np.asarray([file_path] * (1 + len_ants + len_recs))[:, _np.newaxis] return _np.concatenate([blk_uni, file_path_arr], axis=1) +def parse_igs_log_file(filename_array: _np.ndarray) -> Union[_np.ndarray, None]: + """Reads igs log file and outputs ndarray with parsed data + + :param _np.ndarray filename_array: Metadata on input log file. Expects ndarray of the form [CODE DATE PATH] + :return Union[_np.ndarray, None]: Returns array with data from the parsed IGS log file, or `None` for unsupported + version of the IGS Site log format. + """ + # Split filename_array out into its three components (CODE, DATE, PATH), discarding the second element (DATE): + file_code, _, file_path = filename_array + + with open(file_path, "rb") as file: + data = file.read() + + return parse_igs_log_data(data=data, file_path=file_path, file_code=file_code) + + def igslogdate2datetime64(stacked_rec_ant_dt: _np.ndarray) -> _np.datetime64: """Function to convert datetimes for IGS log files to np.datetime64 objects, e.g. 2010-01-01T00:00 @@ -378,10 +403,10 @@ def gather_metadata( if num_threads == 1: gather = [] for file in parsed_filenames: - gather.append(parse_igs_log(file)) + gather.append(parse_igs_log_file(file)) else: with _Pool(processes=num_threads) as pool: - gather = list(pool.imap_unordered(parse_igs_log, parsed_filenames)) + gather = list(pool.imap_unordered(parse_igs_log_file, parsed_filenames)) gather_raw = _np.concatenate(gather) diff --git a/tests/test_datasets/sitelog_test_data.py b/tests/test_datasets/sitelog_test_data.py index 42545ee..2dbef93 100644 --- a/tests/test_datasets/sitelog_test_data.py +++ b/tests/test_datasets/sitelog_test_data.py @@ -1,6 +1,6 @@ # Central record of IGS site log test data sets to be shared across unit tests -# first dataset is a truncated version of file abmf_20240710.log +# Dataset below is a truncated version of file abmf_20240710.log abmf_site_log_v1 = bytes( """ @@ -165,6 +165,8 @@ "utf-8", ) +# Dataset below is a truncated version of file abmf00glp_20240710.log + abmf_site_log_v2 = bytes( """ ABMF00GLP Site Information Form (site log v2.0) @@ -327,3 +329,168 @@ """, "utf-8", ) + +# Dataset below is a truncated version of file aggo00arg_20230608.log + +aggo_site_log_v2 = bytes( + """ + AGGO00ARG Site Information Form (site log v2.0) + International GNSS Service + See Instructions at: + https://files.igs.org/pub/station/general/sitelog_instr_v2.0.txt + +0. Form + + Prepared by (full name) : Thomas Fischer + Date Prepared : 2023-06-08 + Report Type : UPDATE + If Update: + Previous Site Log : (ssssmrccc_ccyymmdd.log) + Modified/Added Sections : (n.n,n.n,...) + + +1. Site Identification of the GNSS Monument + + Site Name : AGGO / Argentina + Nine Character ID : AGGO00ARG + Monument Inscription : Pillar + IERS DOMES Number : 41596M001 + CDP Number : AGGO + Monument Description : CONCRETE PILLAR + Height of the Monument : 4.0 m + Monument Foundation : CONCRETE BLOCK + Foundation Depth : 3.5 m + Marker Description : Pillar plate 14A + Date Installed : 2016-11-11T00:00Z + Geologic Characteristic : sedimentary basin + Bedrock Type : METAMORPHIC PRECAMBRIAN BASEMENT + Bedrock Condition : SEDIMENTS + Fracture Spacing : none + Fault zones nearby : No + Distance/activity : + Additional Information : Argentinean German Geodetic Observatory (AGGO) + : The pillar is insulated by an outer cylinder of + : concrete Pillar plate 14A - standard version + : (Goecke Schwelm) and semipherical vertical + : reference marker next to pillar plate + : Metamorphic Precambrian basement, lower + : Cretaceous and upper Jurassic rocks (the maximum + : sedimentary thickness is 6500m to 7000m) + + +2. Site Location Information + + City or Town : La Plata + State or Province : Province of Buenos Aires + Country or Region : ARG + Tectonic Plate : SOUTH AMERICAN + Approximate Position (ITRF) + X coordinate (m) : 2765120.9 + Y coordinate (m) : -4449250.25 + Z coordinate (m) : -3626405.6 + Latitude (N is +) : -345225.35 + Longitude (E is +) : -0580823.50 + Elevation (m,ellips.) : 42.1 + Additional Information : + + +3. GNSS Receiver Information + +3.1 Receiver Type : SEPT POLARX4TR + Satellite System : GPS+GLO+GAL+BDS+SBAS + Serial Number : 3002049 + Firmware Version : 2.9.6 + Elevation Cutoff Setting : 0 deg + Date Installed : 2016-11-11T10:45Z + Date Removed : 2018-12-06T20:35Z + Temperature Stabiliz. : 5.0 + Additional Information : + +3.2 Receiver Type : SEPT POLARX5TR + Satellite System : GPS+GLO+GAL+BDS+SBAS + Serial Number : 3228290 + Firmware Version : 5.4.0 + Elevation Cutoff Setting : 0 deg + Date Installed : 2018-12-06T20:40Z + Date Removed : (CCYY-MM-DDThh:mmZ) + Temperature Stabiliz. : none + Additional Information : Elimination of the IRNSS system due to lack of + : visibility on 09-JAN-2022 + +3.x Receiver Type : (A20, from rcvr_ant.tab; see instructions) + Satellite System : (GPS+GLO+GAL+BDS+QZSS+SBAS) + Serial Number : (A20, but note the first A5 is used in SINEX) + Firmware Version : (A11) + Elevation Cutoff Setting : (deg) + Date Installed : (CCYY-MM-DDThh:mmZ) + Date Removed : (CCYY-MM-DDThh:mmZ) + Temperature Stabiliz. : (none or tolerance in degrees C) + Additional Information : (multiple lines) + + +4. GNSS Antenna Information + +4.1 Antenna Type : LEIAR25.R4 LEIT + Serial Number : 726722 + Antenna Reference Point : BPA + Marker->ARP Up Ecc. (m) : 000.1550 + Marker->ARP North Ecc(m) : 000.0000 + Marker->ARP East Ecc(m) : 000.0000 + Alignment from True N : 0 deg + Antenna Radome Type : LEIT + Radome Serial Number : + Antenna Cable Type : Nokia Cable M17/75-RG214 + Antenna Cable Length : 60.0 m + Date Installed : 2016-11-11T10:30Z + Date Removed : 2021-06-11T18:30Z + Additional Information : Antenna and radome calibrated by Geo+++ GmbH, + : 2013-11-22. antenna height refering to vertical + : reference marker at pillar + +4.2 Antenna Type : LEIAR25.R4 LEIT + Serial Number : 726722 + Antenna Reference Point : BPA + Marker->ARP Up Ecc. (m) : 000.1550 + Marker->ARP North Ecc(m) : 000.0000 + Marker->ARP East Ecc(m) : 000.0000 + Alignment from True N : 0 deg + Antenna Radome Type : LEIT + Radome Serial Number : + Antenna Cable Type : EcoFlex 10 Cable 50 ohms + Antenna Cable Length : 50.0 m + Date Installed : 2021-06-11T18:30Z + Date Removed : 2022-10-11T13:30Z + Additional Information : Antenna cable replaced + +4.3 Antenna Type : LEIAR25.R4 LEIT + Serial Number : 726722 + Antenna Reference Point : BPA + Marker->ARP Up Ecc. (m) : 000.4100 + Marker->ARP North Ecc(m) : 000.0000 + Marker->ARP East Ecc(m) : 000.0000 + Alignment from True N : 0 deg + Antenna Radome Type : LEIT + Radome Serial Number : + Antenna Cable Type : EcoFlex 10 Cable 50 ohms + Antenna Cable Length : 50.0 m + Date Installed : 2022-10-11T13:30Z + Date Removed : (CCYY-MM-DDThh:mmZ) + Additional Information : Antenna height corrected from 0.4400 m to 0.4100 + +4.x Antenna Type : (A20, from rcvr_ant.tab; see instructions) + Serial Number : (A*, but note the first A5 is used in SINEX) + Antenna Reference Point : (BPA/BCR/XXX from "antenna.gra"; see instr.) + Marker->ARP Up Ecc. (m) : (F8.4) + Marker->ARP North Ecc(m) : (F8.4) + Marker->ARP East Ecc(m) : (F8.4) + Alignment from True N : (deg; + is clockwise/east) + Antenna Radome Type : (A4 from rcvr_ant.tab; see instructions) + Radome Serial Number : + Antenna Cable Type : (vendor & type number) + Antenna Cable Length : (m) + Date Installed : (CCYY-MM-DDThh:mmZ) + Date Removed : (CCYY-MM-DDThh:mmZ) + Additional Information : (multiple lines) + """, + "utf-8", +) diff --git a/tests/test_igslog.py b/tests/test_igslog.py index 05c0487..5b8e5da 100644 --- a/tests/test_igslog.py +++ b/tests/test_igslog.py @@ -1,12 +1,19 @@ import unittest -import numpy as _np -import pandas as _pd +from pyfakefs.fake_filesystem_unittest import TestCase from gnssanalysis.gn_io import igslog -from test_datasets.sitelog_test_data import abmf_site_log_v1 as v1_data, abmf_site_log_v2 as v2_data +from test_datasets.sitelog_test_data import ( + abmf_site_log_v1 as v1_data, + abmf_site_log_v2 as v2_data, + aggo_site_log_v2 as aggo_v2_data, +) -class Testregex(unittest.TestCase): +class TestRegex(unittest.TestCase): + """ + Test the various regex expressions used in the parsing of IGS log files + """ + def test_determine_log_version(self): # Ensure version 1 and 2 strings are produced as expected self.assertEqual(igslog.determine_log_version(v1_data), "v1.0") @@ -93,3 +100,61 @@ def test_extract_antenna_block(self): self.assertEqual(v2_antenna_block[0][8], b"2009-10-15T20:00Z") # Check end date of second entry # Last antenna should not have an end date assigned (i.e. current): self.assertEqual(v2_antenna_block[-1][-1], b"") + + +class TestDataParsing(unittest.TestCase): + """ + Test the integrated functions that gather and parse information from IGS log files + """ + + def test_parse_igs_log_data(self): + # Parse version 1 log file: + v1_data_parsed = igslog.parse_igs_log_data(data=v1_data, file_path="/example/path1", file_code="ABMF") + # Check country name: + self.assertEqual(v1_data_parsed[0][4], "Guadeloupe") + # Check last antenna type: + self.assertEqual(v1_data_parsed[-1][2], "TRM57971.00") + + # Parse version 2 log file: + v2_data_parsed = igslog.parse_igs_log_data(data=v2_data, file_path="/example/path2", file_code="ABMF") + # Check country name: + self.assertEqual(v2_data_parsed[0][4], "GLP") + # Check last antenna type: + self.assertEqual(v2_data_parsed[-1][2], "TRM57971.00") + + +class TestFileParsing(TestCase): + """ + Test gather_metadata() + """ + + def setUp(self): + self.setUpPyfakefs() + + def test_gather_metadata(self): + self.fs.reset() # Ensure fake filesystem is cleared from any previous tests, as it is backed by real filesystem. + # Create some fake files + file_paths = ["/fake/dir/abmf.log", "/fake/dir/aggo.log"] + self.fs.create_file(file_paths[0], contents=v2_data) + self.fs.create_file(file_paths[1], contents=aggo_v2_data) + + # Call gather_metadata to grab log files for two stations + result = igslog.gather_metadata(logs_glob_path="/fake/dir/*") + + # Test that various data has been read correctly: + # ID/Location Info: test CODE and Country / region + id_loc_results = result[0] + self.assertEqual(id_loc_results.CODE[0], "ABMF") + self.assertEqual(id_loc_results.COUNTRY[0], "GLP") + self.assertEqual(id_loc_results.CODE[1], "AGGO") + self.assertEqual(id_loc_results.COUNTRY[1], "ARG") + # Receiver info: test a couple receivers + receiver_results = result[1] + record_0 = receiver_results.loc[0] + record_3 = receiver_results.loc[3] + self.assertEqual(record_0.RECEIVER, "LEICA GR25") + self.assertEqual(record_0.END_RAW, "2019-04-15T12:00Z") + self.assertEqual(record_3.RECEIVER, "SEPT POLARX4TR") + self.assertEqual(record_3.CODE, "AGGO") + # Antenna info: test for antenna serial number + self.assertEqual(result[2]["S/N"][4], "726722")