Skip to content

Commit

Permalink
Merge pull request #68 from GeoscienceAustralia/NPI-3653-igslog-updates
Browse files Browse the repository at this point in the history 
NPI-3653 - Fix errors in parsing IGS log updates
  • Loading branch information
@ronaldmaj
ronaldmaj authored Jan 9, 2025
2 parents 36c0374 + 651437f commit 3305782
Show file tree
Hide file tree
Showing 4 changed files with 295 additions and 37 deletions.
3 changes: 2 additions & 1 deletion gnssanalysis/gn_io/aux_dicts.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,8 @@
"VIRGIN ISLANDS": "VIRGIN ISL",
"US VIRGIN ISLANDS": "VIRGIN ISL",
"WALLIS AND FUTUNA": "WALLIS",
"WEST ANTARCTICA": "W AFRICA",
"WEST ANTARCTICA": "ANTARCTICA",
"W ANTARCTICA": "ANTARCTICA",
}

translation_rec = {
Expand Down
87 changes: 56 additions & 31 deletions gnssanalysis/gn_io/igslog.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,14 @@

logger = logging.getLogger(__name__)

# Defines what IGS Site Log format versions we currently support.
# Example logs for the first two versions can be found at:
# Version 1: https://files.igs.org/pub/station/general/blank.log
# Version 2: https://files.igs.org/pub/station/general/blank_v2.0.log

_REGEX_LOG_VERSION_1 = _re.compile(rb"""(site log\))""")
_REGEX_LOG_VERSION_2 = _re.compile(rb"""(site log v2.0)""")

_REGEX_ID_V1 = _re.compile(
rb"""
(?:Four\sCharacter\sID|Site\sID)\s+\:\s*(\w{4}).*\W+
Expand Down Expand Up @@ -138,22 +146,18 @@ def find_recent_logs(logs_glob_path: str, rnx_glob_path: str = None) -> _pd.Data
return recent_logs_df


_REGEX_VERSION_1 = _re.compile(rb"""(site log\))""")
_REGEX_VERSION_2 = _re.compile(rb"""(site log v2)""")


def determine_log_version(data: bytes) -> str:
"""Given the byes object that results from reading an IGS log file, determine the version ("v1.0" or "v2.0")
:param bytes data: IGS log file bytes object to determine the version of
:return str: Return the version number: "v1.0" or "v2.0" (or "Unknown" if file does not conform to standard)
"""

result_v1 = _REGEX_VERSION_1.search(data)
result_v1 = _REGEX_LOG_VERSION_1.search(data)
if result_v1:
return "v1.0"

result_v2 = _REGEX_VERSION_2.search(data)
result_v2 = _REGEX_LOG_VERSION_2.search(data)
if result_v2:
return "v2.0"

Expand Down Expand Up @@ -247,38 +251,38 @@ def extract_antenna_block(data: bytes, file_path: str) -> Union[List[Tuple[bytes
return antenna_block


def parse_igs_log(filename_array: _np.ndarray) -> Union[_np.ndarray, None]:
"""Parses igs log and outputs ndarray with parsed data
def parse_igs_log_data(data: bytes, file_path: str, file_code: str) -> Union[_np.ndarray, None]:
"""Given the bytes object returned opening a IGS log file, parse to produce an ndarray with relevant data
:param _np.ndarray filename_array: Metadata on input log file. Expects ndarray of the form [CODE DATE PATH]
:return _np.ndarray: Returns array with data from the IGS log file parsed
:param bytes data: The bytes object returned from an open() call on a IGS site log in "rb" mode
:param str file_path: The path to the file from which the "data" bytes object was obtained
:param str file_code: Code from the filename_array passed to the parse_igs_log() function
:return Union[_np.ndarray, None]: Returns array with relevant data from the IGS log file bytes object,
or `None` for unsupported version of the IGS Site log format.
"""
file_code, _, file_path = filename_array

with open(file_path, "rb") as file:
data = file.read()

# Determine the version of the IGS log based on the data, Warn if unrecognised
try:
version = determine_log_version(data)
except LogVersionError as e:
logger.warning(f"Error: {e}, skipping parsing the log file")
return

blk_id = extract_id_block(data, version, file_path, file_code)
blk_loc = extract_location_block(data, version, file_path)
blk_rec = extract_receiver_block(data, file_path)
blk_ant = extract_antenna_block(data, file_path)

return None

# Extract information from ID block
blk_id = extract_id_block(data=data, file_path=file_path, file_code=file_code, version=version)
code = [blk_id[0]] # Site code
# Extract information from Location block
blk_loc = extract_location_block(
data=data,
file_path=file_path,
version=version,
)
blk_loc = [group.decode(encoding="utf8", errors="ignore") for group in blk_loc.groups()]
# Combine ID and Location information:
blk_id_loc = _np.asarray([0] + blk_id + blk_loc, dtype=object)[_np.newaxis]
# Extract and re-format information from receiver block:
blk_rec = extract_receiver_block(data=data, file_path=file_path)
blk_rec = _np.asarray(blk_rec, dtype=str)
blk_ant = _np.asarray(blk_ant, dtype=str)

len_recs = blk_rec.shape[0]
len_ants = blk_ant.shape[0]

blk_id_loc = _np.asarray([0] + blk_id + blk_loc, dtype=object)[_np.newaxis]

code = [code]
blk_rec = _np.concatenate(
[
_np.asarray([1] * len_recs, dtype=object)[:, _np.newaxis],
Expand All @@ -287,6 +291,10 @@ def parse_igs_log(filename_array: _np.ndarray) -> Union[_np.ndarray, None]:
],
axis=1,
)
# Extract and re-format information from antenna block:
blk_ant = extract_antenna_block(data=data, file_path=file_path)
blk_ant = _np.asarray(blk_ant, dtype=str)
len_ants = blk_ant.shape[0]
blk_ant = _np.concatenate(
[
_np.asarray([2] * len_ants, dtype=object)[:, _np.newaxis],
Expand All @@ -295,11 +303,28 @@ def parse_igs_log(filename_array: _np.ndarray) -> Union[_np.ndarray, None]:
],
axis=1,
)
# Create unified information block:
blk_uni = _np.concatenate([blk_id_loc, blk_rec, blk_ant], axis=0)
file_path_arr = _np.asarray([file_path] * (1 + len_ants + len_recs))[:, _np.newaxis]
return _np.concatenate([blk_uni, file_path_arr], axis=1)


def parse_igs_log_file(filename_array: _np.ndarray) -> Union[_np.ndarray, None]:
"""Reads igs log file and outputs ndarray with parsed data
:param _np.ndarray filename_array: Metadata on input log file. Expects ndarray of the form [CODE DATE PATH]
:return Union[_np.ndarray, None]: Returns array with data from the parsed IGS log file, or `None` for unsupported
version of the IGS Site log format.
"""
# Split filename_array out into its three components (CODE, DATE, PATH), discarding the second element (DATE):
file_code, _, file_path = filename_array

with open(file_path, "rb") as file:
data = file.read()

return parse_igs_log_data(data=data, file_path=file_path, file_code=file_code)


def igslogdate2datetime64(stacked_rec_ant_dt: _np.ndarray) -> _np.datetime64:
"""Function to convert datetimes for IGS log files to np.datetime64 objects, e.g. 2010-01-01T00:00
Expand Down Expand Up @@ -378,10 +403,10 @@ def gather_metadata(
if num_threads == 1:
gather = []
for file in parsed_filenames:
gather.append(parse_igs_log(file))
gather.append(parse_igs_log_file(file))
else:
with _Pool(processes=num_threads) as pool:
gather = list(pool.imap_unordered(parse_igs_log, parsed_filenames))
gather = list(pool.imap_unordered(parse_igs_log_file, parsed_filenames))

gather_raw = _np.concatenate(gather)

Expand Down
169 changes: 168 additions & 1 deletion tests/test_datasets/sitelog_test_data.py
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Central record of IGS site log test data sets to be shared across unit tests

# first dataset is a truncated version of file abmf_20240710.log
# Dataset below is a truncated version of file abmf_20240710.log

abmf_site_log_v1 = bytes(
"""
Expand Down Expand Up @@ -165,6 +165,8 @@
"utf-8",
)

# Dataset below is a truncated version of file abmf00glp_20240710.log

abmf_site_log_v2 = bytes(
"""
ABMF00GLP Site Information Form (site log v2.0)
Expand Down Expand Up @@ -327,3 +329,168 @@
""",
"utf-8",
)

# Dataset below is a truncated version of file aggo00arg_20230608.log

aggo_site_log_v2 = bytes(
"""
AGGO00ARG Site Information Form (site log v2.0)
International GNSS Service
See Instructions at:
https://files.igs.org/pub/station/general/sitelog_instr_v2.0.txt
0. Form
Prepared by (full name) : Thomas Fischer
Date Prepared : 2023-06-08
Report Type : UPDATE
If Update:
Previous Site Log : (ssssmrccc_ccyymmdd.log)
Modified/Added Sections : (n.n,n.n,...)
1. Site Identification of the GNSS Monument
Site Name : AGGO / Argentina
Nine Character ID : AGGO00ARG
Monument Inscription : Pillar
IERS DOMES Number : 41596M001
CDP Number : AGGO
Monument Description : CONCRETE PILLAR
Height of the Monument : 4.0 m
Monument Foundation : CONCRETE BLOCK
Foundation Depth : 3.5 m
Marker Description : Pillar plate 14A
Date Installed : 2016-11-11T00:00Z
Geologic Characteristic : sedimentary basin
Bedrock Type : METAMORPHIC PRECAMBRIAN BASEMENT
Bedrock Condition : SEDIMENTS
Fracture Spacing : none
Fault zones nearby : No
Distance/activity :
Additional Information : Argentinean German Geodetic Observatory (AGGO)
: The pillar is insulated by an outer cylinder of
: concrete Pillar plate 14A - standard version
: (Goecke Schwelm) and semipherical vertical
: reference marker next to pillar plate
: Metamorphic Precambrian basement, lower
: Cretaceous and upper Jurassic rocks (the maximum
: sedimentary thickness is 6500m to 7000m)
2. Site Location Information
City or Town : La Plata
State or Province : Province of Buenos Aires
Country or Region : ARG
Tectonic Plate : SOUTH AMERICAN
Approximate Position (ITRF)
X coordinate (m) : 2765120.9
Y coordinate (m) : -4449250.25
Z coordinate (m) : -3626405.6
Latitude (N is +) : -345225.35
Longitude (E is +) : -0580823.50
Elevation (m,ellips.) : 42.1
Additional Information :
3. GNSS Receiver Information
3.1 Receiver Type : SEPT POLARX4TR
Satellite System : GPS+GLO+GAL+BDS+SBAS
Serial Number : 3002049
Firmware Version : 2.9.6
Elevation Cutoff Setting : 0 deg
Date Installed : 2016-11-11T10:45Z
Date Removed : 2018-12-06T20:35Z
Temperature Stabiliz. : 5.0
Additional Information :
3.2 Receiver Type : SEPT POLARX5TR
Satellite System : GPS+GLO+GAL+BDS+SBAS
Serial Number : 3228290
Firmware Version : 5.4.0
Elevation Cutoff Setting : 0 deg
Date Installed : 2018-12-06T20:40Z
Date Removed : (CCYY-MM-DDThh:mmZ)
Temperature Stabiliz. : none
Additional Information : Elimination of the IRNSS system due to lack of
: visibility on 09-JAN-2022
3.x Receiver Type : (A20, from rcvr_ant.tab; see instructions)
Satellite System : (GPS+GLO+GAL+BDS+QZSS+SBAS)
Serial Number : (A20, but note the first A5 is used in SINEX)
Firmware Version : (A11)
Elevation Cutoff Setting : (deg)
Date Installed : (CCYY-MM-DDThh:mmZ)
Date Removed : (CCYY-MM-DDThh:mmZ)
Temperature Stabiliz. : (none or tolerance in degrees C)
Additional Information : (multiple lines)
4. GNSS Antenna Information
4.1 Antenna Type : LEIAR25.R4 LEIT
Serial Number : 726722
Antenna Reference Point : BPA
Marker->ARP Up Ecc. (m) : 000.1550
Marker->ARP North Ecc(m) : 000.0000
Marker->ARP East Ecc(m) : 000.0000
Alignment from True N : 0 deg
Antenna Radome Type : LEIT
Radome Serial Number :
Antenna Cable Type : Nokia Cable M17/75-RG214
Antenna Cable Length : 60.0 m
Date Installed : 2016-11-11T10:30Z
Date Removed : 2021-06-11T18:30Z
Additional Information : Antenna and radome calibrated by Geo+++ GmbH,
: 2013-11-22. antenna height refering to vertical
: reference marker at pillar
4.2 Antenna Type : LEIAR25.R4 LEIT
Serial Number : 726722
Antenna Reference Point : BPA
Marker->ARP Up Ecc. (m) : 000.1550
Marker->ARP North Ecc(m) : 000.0000
Marker->ARP East Ecc(m) : 000.0000
Alignment from True N : 0 deg
Antenna Radome Type : LEIT
Radome Serial Number :
Antenna Cable Type : EcoFlex 10 Cable 50 ohms
Antenna Cable Length : 50.0 m
Date Installed : 2021-06-11T18:30Z
Date Removed : 2022-10-11T13:30Z
Additional Information : Antenna cable replaced
4.3 Antenna Type : LEIAR25.R4 LEIT
Serial Number : 726722
Antenna Reference Point : BPA
Marker->ARP Up Ecc. (m) : 000.4100
Marker->ARP North Ecc(m) : 000.0000
Marker->ARP East Ecc(m) : 000.0000
Alignment from True N : 0 deg
Antenna Radome Type : LEIT
Radome Serial Number :
Antenna Cable Type : EcoFlex 10 Cable 50 ohms
Antenna Cable Length : 50.0 m
Date Installed : 2022-10-11T13:30Z
Date Removed : (CCYY-MM-DDThh:mmZ)
Additional Information : Antenna height corrected from 0.4400 m to 0.4100
4.x Antenna Type : (A20, from rcvr_ant.tab; see instructions)
Serial Number : (A*, but note the first A5 is used in SINEX)
Antenna Reference Point : (BPA/BCR/XXX from "antenna.gra"; see instr.)
Marker->ARP Up Ecc. (m) : (F8.4)
Marker->ARP North Ecc(m) : (F8.4)
Marker->ARP East Ecc(m) : (F8.4)
Alignment from True N : (deg; + is clockwise/east)
Antenna Radome Type : (A4 from rcvr_ant.tab; see instructions)
Radome Serial Number :
Antenna Cable Type : (vendor & type number)
Antenna Cable Length : (m)
Date Installed : (CCYY-MM-DDThh:mmZ)
Date Removed : (CCYY-MM-DDThh:mmZ)
Additional Information : (multiple lines)
""",
"utf-8",
)
Loading

0 comments on commit 3305782

Please sign in to comment.