From d4c9f06c131ff3c4bad50731ccef1bf6de55af8d Mon Sep 17 00:00:00 2001 From: akrherz Date: Mon, 4 Dec 2023 08:17:49 -0600 Subject: [PATCH 1/3] chore: bump ruff --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 62ee7803..e682d41f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ ci: autoupdate_schedule: quarterly repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: "v0.1.5" + rev: "v0.1.7" hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] From 66f086f2be00ed19651a3f0a076f7cbd1592657f Mon Sep 17 00:00:00 2001 From: akrherz Date: Tue, 5 Dec 2023 08:43:53 -0600 Subject: [PATCH 2/3] pin back openssl --- environment.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index f2a139eb..55613fd8 100644 --- a/environment.yml +++ b/environment.yml @@ -22,6 +22,8 @@ dependencies: - ncurses - netCDF4 - numpy + # Temporary until my breakage clears + - openssl <3.2 - pandas # geoplot usage - pillow @@ -48,4 +50,4 @@ dependencies: - treq - twisted - xarray - \ No newline at end of file + From d9e761d6387abfd5633d652abc2c02b703f74ad4 Mon Sep 17 00:00:00 2001 From: akrherz Date: Mon, 4 Dec 2023 14:09:20 -0600 Subject: [PATCH 3/3] feat: inital swing at BUFR surface ingest refs akrherz/iem#603 --- .gitignore | 1 + environment.yml | 2 + examples/BUFR/ISAB02_CWAO.bufr | Bin 0 -> 741 bytes examples/BUFR/ISIA14_CWAO.bufr | Bin 0 -> 834 bytes examples/BUFR/ISMA01_FNLU.bufr | Bin 0 -> 20732 bytes examples/BUFR/ISMA01_FNLU_badid.bufr | Bin 0 -> 21528 bytes examples/BUFR/ISMI60_SBBR.bufr | Bin 0 -> 267 bytes examples/BUFR/ISND01_LEMM.bufr | Bin 0 -> 1374 bytes examples/BUFR/ISXD03_EDZW.bufr | Bin 0 -> 453 bytes examples/BUFR/ISXT14_EGRR.bufr | Bin 0 -> 15190 bytes examples/BUFR/IS_2023120401.bufr | Bin 0 -> 1644 bytes parsers/bufr_surface.py | 6 + parsers/pywwa/workflows/bufr_surface.py | 486 ++++++++++++++++++++++++ pqact.d/pqact_iemingest.conf | 6 + tests/test_workflows.py | 1 + tests/workflows/test_bufr_surface.py | 112 ++++++ 16 files changed, 614 insertions(+) create mode 100644 examples/BUFR/ISAB02_CWAO.bufr create mode 100644 examples/BUFR/ISIA14_CWAO.bufr create mode 100644 examples/BUFR/ISMA01_FNLU.bufr create mode 100644 examples/BUFR/ISMA01_FNLU_badid.bufr create mode 100644 examples/BUFR/ISMI60_SBBR.bufr create mode 100644 examples/BUFR/ISND01_LEMM.bufr create mode 100644 examples/BUFR/ISXD03_EDZW.bufr create mode 100644 examples/BUFR/ISXT14_EGRR.bufr create mode 100644 examples/BUFR/IS_2023120401.bufr create mode 100644 parsers/bufr_surface.py create mode 100644 parsers/pywwa/workflows/bufr_surface.py create mode 100644 tests/workflows/test_bufr_surface.py diff --git a/.gitignore b/.gitignore index b3aaecdf..f9c474c8 100644 --- a/.gitignore +++ b/.gitignore @@ -14,5 +14,6 @@ util/rabbitmq.json **/ridge_processor.pid /goes/*.nc .coverage +coverage.xml .eggs/ parsers/pywwa.egg-info/ diff --git a/environment.yml b/environment.yml index 55613fd8..d24ce295 100644 --- a/environment.yml +++ b/environment.yml @@ -31,6 +31,8 @@ dependencies: - psycopg # Transient via pyIEM - pyarrow + # BUFR processing + - pybufrkit # Models - pydantic # gini reader diff --git a/examples/BUFR/ISAB02_CWAO.bufr b/examples/BUFR/ISAB02_CWAO.bufr new file mode 100644 index 0000000000000000000000000000000000000000..5493644c3d6905c2238517efd38d191b6949a909 GIT binary patch literal 741 zcmZ8fZ%9*76hH62xu;WmFHIpe-%}x`25w97Pug=DK{JYsD3!=IMzTJ{G-xbodn2eU zbULsyiU@+}OQ95EQt!(!8RVxzqTuD1{vh!RgDu8xXB!rDINbB+cg{V(!@W#lp{cRO zDxo=W)ZS&WN!`cohb2ps&0?`gy}fn>x{mGd1+riOC*ym+Q-7wG zc>Ir@RiwzY{BJY}X8w&4%(zNLv~_H8eEMXe1W>&4sb#}FZN0GmG~ufqs(yZVZEoEE zt^e5|zz*9>-~4NFcyOxV&Mx_i<60Ql?+1(Qk)e6V*CRII6)ixMJge&MJnpV$G%ltL zH=~Y`qS5QY;^?`?(uR+D$SF{4#0!zEzpdDpBI(=1F#r3Bme*a literal 0 HcmV?d00001 diff --git a/examples/BUFR/ISIA14_CWAO.bufr b/examples/BUFR/ISIA14_CWAO.bufr new file mode 100644 index 0000000000000000000000000000000000000000..b068ed3bd0249258d4ee0c29aa8fa71d9f1d02e2 GIT binary patch literal 834 zcmZWoO=uHQ5T0aH6B}b9Y85NyS*@*T+C*ElB28Yln`U*BT{pYYpx{A8q=zDS=t0aT zicnglD0t8wilr(Ah2|uPbn&1!@edJuF!Z8SkU~X*P4k@HT0J-{Z|0jf^L@jx&*Agc zckGW4bYe!Nkq*Kir!on5?u&%On2hTQ6}UzhKm$N0EE@s$mo;@8EwCEEXTDUxIS)4+ z50mhrU*FTX1ugexG$E9bbEH*oBNEM02~|Yo z?rD)xo2E0%Wwx#&tJ)uEJLFSjo@%0?{v|!83vq#OKZ3195;C+`AfhT}t&&iW9+G5T zv3vht0b>sEE)-#_3z~|sr3eAUQLTB`+Y1vz0NZz*)7v8t4u)@O?`9sEpAHS2-;`(7 ziD#bV$-T+atC6wq!*fqUO+{%T|265nw%!D9xSuiiY~#JOcbxxdY;KFq7~q0I&+F1O z>KB4B{ah)&7{0ppv@zFfCLG{rCfs%bfuSEs z`l~x*1R3Kh!9qDdALZtw)(;cqYVLC0tokv{YHo%Xtm4UAJ=fdgpxI`HFu17|2p9RL6T literal 0 HcmV?d00001 diff --git a/examples/BUFR/ISMA01_FNLU.bufr b/examples/BUFR/ISMA01_FNLU.bufr new file mode 100644 index 0000000000000000000000000000000000000000..d05a1bac10ead73b9a77d88dcc89b7613464c626 GIT binary patch literal 20732 zcmeHP30M=?+CD>&eG?1_vP2L;v??m%3Nfxkv{c2dXb?q_YZam*;F1yBT9@m-A_i); zSlV6{?eDE!fZzgdpjO*jynz4-K@@O-VM*pcXC{F}hPGb4wdvF0A#>(C2Qu@%-*&da ztgS8l2My-HQ^@owV+Q(jCQJfQ~_fT!E>=%A^ar zumk|Z^A22z1lR&#ECYbGaG}FORHHe}#v4UtSqMGNv;z9vO89d}D^AU zafKx1K^{~QeKcLqOd(Z`ycpxYpnzZmL~p*hBj%1TX| zq^&?~j68MZQ_hKTAt}8e5Ab+_4I(k~5UFb7#h5iBHe4H>9@aU(G#hnCUI!a7aVGj~ zob^0Ou~9CT-P2`5nGD&G<-?~Dy=gX}nIPL3>|mz_9RRbC3oaJNvSRK1qIAuJVxwo( zaYa5(W{7@-ci(}HCYjG3iVapBVk30TxL-~?zcd?9;Xh6%$yRuJ1>1uB59p#0~=9X7`DTrBMQFS;tk26u78@skr4m~u89gm>SO zjsN)UB2^8N7*;D{W77H2--J28G#jtSQ$^wC`fQx(UO=PjhcTUPqhs`jp$Fp z#$X3rhmDXYyex}Y`y`AfOwKkpisGl*C!0?~Y>cQp>OGTP>2+*3qiqnLJ_n&~NMbB_ zhz=sAs_ zt?Q=%Kx_n_jPst}`;&8d_mO3Hunpq$5*WpyZJ2dKYz$@fb_;2M@Hx!#88`_*|`phoH%xwH2 zIhC2dZJcrLN3l^It2EMOqdFFOsGC&)Iue(XMSpz}+| zjk+VRBd1zwXRXi1+3J0?Z6ufLWw>uFr|Q%;erMdke=BZyEkPMBj2l6S4M(T0mUS=B z#_Ne2OYj#nT>d~hZb)SFb#3E~#f?tcaQJ;W75-Z}l{E7J<*DeLDx@)}<&Al&sNxPe zRSCt0GN|gZE*ozwr|QVY%k?+#k)8E7a?ZmD2qR%F>Kl_Jk^T$(6s;i1?|oneylC&2 z2GOGg;V6b-{gOZUqC$6QG|j76~{2Hv(oLF+whNvXqsX;zsdH<(ygX7qf;oD|U$&6a}bV zp9Tjrrd+~z+rk9b-s_fktt8}l`K4{1QVNtuKZTx^9O!yO`H;O`U zH$o5hSj%4?A=x(sPipxE=c=r8hpog;YS9ZQNftP^f>{u~;XhYI`atxGn=*Har(E+U zU9$!3g6lxor(#-VL-hb%G}_99kDU-*bpQDK;5K2$f?=UIp)b_E6-tEO`U|%F=weAs z7CyXw2IxgrXSjufsy6f@HhVzy!XbKGb2$g0m)q=fNF0CtpJYjalwB|b2rVldtG3ph z>WB%H5dLfmz2UqiBE|-^(6G>(g3ycq7@@aLb|G{dH|1mo9-uDfx~emAmkFR4p=Sbq zVy%hzO5o(6z7Nq0EUe5xmV2)$bH8}bjkieGy8`yrFM#mlN?uh|XXrUNO+oi33h2AmBa?KHX2`gBWdoDJ1$pi1zF9f_+DM@a|5;eN#yT)~r)UlU-(SOMammdVSfDr^~HA%L9(RakOr80$4c!$L0wp%=FRp_iC9 zkKb!SisA_FRzD?oD3OWx7zGkRB!=67L{?l$P`YCU7|V(@8jw_u&|^Jr$=oiEzR>_& zr2_Wl8W4Z9e1yDNIh_$5VtDudsKVczZcgN>E7Dp#~}sfdG-fzx0a7`IW1nfJp#c;NcT+O{K{%qrH5k; zm;upS4*ed|y-_Voc8llSaR+YxQg+c0urgoqVYRHb2O~O!XV;Tdg-kou!Pv~O&^tuw z-p`cZTm3=Eo$t7X1^96FLg1-BgL`ZTKeg%JJZphVxq3E4FKLdPlL^t|v?_Lp7u>0A zeh?&J-*E-2zEdVw^Ez>B1YmuE@N(%{7R-nnEi^3j`CWYwrNs^~;hEc;vm6<9|DtJzo>=dxCoCERSlqbmD zAEh&bLzvm{P{ms*5y6ZNXrW=Dx34ek=8#PYy*0z`PFcLlsQEA$ueQz|q+VR=GYXUf zj@8Czro|pJBUYt5wmkn4qSx>1iu6XPdkv_s@y^wN*0d~`?-_&mf5`612h|7|!6Do| zDXkoQOgr?(*vzodTa0vXbv@<$eyskZ6c!P|gcgx|fE9-ZUy|j6vun z&bz^1bSVFA3*cPO2liwNZrcrJEL1<&mV6d5)&pkC#ZdP=x5!b+;dGnS>nQ*-!-KX0`~RoK-f~*pnOtu zgaH(&d%fMyp$hk&`x)ysLc>DuA5_f!4C!9-)ntD8rA2E9pzfh;`Am6~{6I}0BRYgh*WRe2&#`eL#s;*| zu+aMiS#Eq+gx;!iLnhlq>T*VK2w=%a6@8=L&0%an3k?fBCDOeFYLp=UAJzOb zi#AQVjR(}v0iGm$$ShDmp~w2kb`8u}2D9oObch-Su&1WUSbi?gR{A|gQ#e}Q@fw%YVp8gEOQSc)gzM~7vRUB7s+iva zy~6K->k*i-&@oqO^mr%4MqO?C0zy#txG#v9E)5Gk0YWcsII`Sz6NZJ9e!zWz1MKNm zVBd5i!^Rg3)LL#}L>csZPoeG^B@`;Co|HWX}#gSe|d<1v8e(Tw7osX**e)v6z9O5jgaF zR8OsMp*->mjJeG!pTpw#t17bO6rC*>m5VF))Z{Y)Ljc=4RN>^=jj>)MG%WP;DfFmO z0^$DCA=l!@ts4aVTLoMfRZ)gZII>&}_r(5fQg@`qW5h!;uL~Db9hR2DQ3B)lTQf!C z^@VcaDi*LM+dh|CK%B;MO&ij73Km+b=RnKl6+S}fvA%6qoD-K_JF2Go8qa`mbVYKN_~9l- zaEKnZ7O27}uQ8ah0WCBv^umzt#Zei{+8;whiY-9N5!{)$j$5f0McTZD#`owEssuL{ z^n2=`kl))HeA3iiU2FWRtTZrYh(n67}ae8su;(c%~-Dy8Wwuf zk?yUd(VOu>$h`n?L!|q3ga$p9o>aHR>n$4=dN&Yy@l?zeHXNS3%zo^);kcW+2kxle zRqDPH9MZRh&cKJNg3+uHH5+aMhM6V0ek^rz zP?dJb_pW)@N1T&>d94=^7FO<&H&#Zcv9+WA3lEy$vqHvp2v+YDpBzSbX`x~1UK&C#f$D1{ zY)cHOAB;`#1>?#6Tu(AKgA=axd)-0fQJ%-AE%J_IEO#%}&zmoezk3m^j<4QR<5}ay z2o3>g^lTT*{p+B&i9(Oc_Y&Qv@uU8gKf?%jBlEa6@YmX+eKtHWEmGjr4Rjo1xv~Se zCQ8|ttU$cCj4!vKR-man^w)?k*8f4Ux*hWpp>rpPVdpj*ttBw_Rlzk>(Hh-@ ztALR1Rq`JBFsc>d<4SEx^pi{rW^6zU4NLbXBHdf@Gt#~FYgh9{Uu_!ghaUiJE}zWG z2ncUq{WvsY5}c7FVZLY7>$55j8kx6i&P*5YzY-d00a4msm9ZL@5!;+-x(wAh( zTsR~748$qRZddG}vSoD@Twtts5GM54@-U;n5PBCWzvoY7Ec51sTvCsFFC2ul`r*CR z>{72&fEp!m$A;OjcIBkQIkh_-`Wly~0@p=Swqy~Ai!YUx_omnH0X^v81<})SACh%l zus;}`J2?zX_oh+0SBB7Ay9oYFZOncKzqvXaV2wuJ3$VKUll!izj)PUxVNct zFF4>Kssn;&z^NHntAObS9HyLY`^AB6ad+)1Va$zMv#H%LzGJLHdyg--U%b;D16~es zXaPTcm`%9qV&RgNny3}fXSnYVAHBNDJG`ECqDcGwou1KdnxON8e+hGD*xS_hS9n`V zTY$Uw5ku*(yhAyL19#J8;9O!Cm$RyfunRuxxs2Msp{Q1L+%K*O)McZK3Sb*>AZlmB z=v48#0h)Wn22dp}8|mX`X>a~j+0g7z_VS8shu&SN^T zOR+H}qW5rL*3nbi@7F&Q7ZmI82hT~hDN^mmfLT5x>YV>@HqbrEM2kzin5&N9{fK|& zIzl!qgR^hD!HU|C4zno@OjOW7-M_dQw3PYIWyP>zDR_clgvAmUyrAX4l^-V9l$xn zmPj_V4U3%Yjr6+|6!#P#>g_Z`2BHvXJrxGy#+NG4cLSX5YWGwyZcIkzgAOo8&o)xq z!``df2GG9QLYIvYiHn7BnuXqbEvWtC_{j0!_96~%Jh+p-uV48@Y|L^B+g9w%z((8O z{SV_tG=Mvl_j9?#)(kpsWc$->R5w)1^w{u(Y*gz-aTqtCr^58vaIe+e_6uz6YF`}n zn%MAKsLRHrK`#2Q7@-UV*~TcF&%FI-E$TxlM;oS01;mDXVD}zxjE%@J))rL??m}$j zT0`3~$@vO|QBl17vJBbAj-(vTJ9sWYHe{2vwt;f0j!_((DjvB)%fYvpN$)(WkaZI8`QfL^x1$JF0>8X;C4@CTw5G|O7p1L z&7G*EW#cF|OH=X6QC#Dx9LDO}#yFwSZ-m7NUkwfP$r!N_^xcIM!S^p(@l1TEJAiRR ztrIthJ((H!yyx(ZNNI=-w}bXQ20FhXHloAqk*A7rsjk6n0jM>}Eu~z&2 zot_aJjyWV*=lrJFsI%LmIt=C$+wlRA4X5DjjmIh5klib5MxJV8FzuLIP4sU(Ihxh;D zxQz7v=xy!ycX~!_I0u^f?Q(uYY{X{3-TR;C1`+#9y^yDZaibgUsd|?U(zOkR4YZBY z@;2LmY>ZX`pAC2~ohWWRpx9HZg4j~(@wII1z1XNl;Wfn#hlRTDZX55nHH#n17xMJz zK(;YD4?q6w!bUrMvnh24FJ!oyyOJ!0X&_`HWLD6-VN7iN5gBe2ZnZ_V70e_)$e?ZG zbbt7Er{%27I+4>;D1w Co82q` literal 0 HcmV?d00001 diff --git a/examples/BUFR/ISMA01_FNLU_badid.bufr b/examples/BUFR/ISMA01_FNLU_badid.bufr new file mode 100644 index 0000000000000000000000000000000000000000..32c4047a1c1650a08981ec2cb10423164642e9dd GIT binary patch literal 21528 zcmd6v30xD`zQ>0(i%JmOLaidJrd1HER8UC+F663Ee2TT&N>BoMZBbeUwS|aP1Y4nQ z+@clqf!MtJ3WB}1-nLNet+kt}eYf=4-ilZage7QwW2^Qd|?;2c;Qo_Q+?+yTCl`7bo$h=&?n%eV#(Y@%xRY^hK<6o zF&Nsy!p33(!%+7j=ncF$X+Pb6W)08))E43X9(iz?D6Cwdg=LR)h@!)sYr9VjeS9wQ%dHYt&v6(4PgX< zEhcVb)%0I#U(2PAwDgI-U3Wau{dg|1rHr^ODI*MoGkxh!^zEXa=)S)r>P>fu=WoY% z!YT7j)_L>IJA&_smfZB3AtFmXaR)CWh*!=LM15WB2&cCQ(~i!_4y98uI(^5f*y4$G zM#4+$PWUf!3F4xeF!vG`OEi&F1ZVERzxsKEnIKHx5hi%U+RUx2H`L*+Ti)z*5wQJR zG*VatFP!77(uw&nFhpZknDb{oi#<BXkeH44$FgZ||HL1_4IP3XfRoNrX=WSJr-g-Tbt zaXxba(TJ)(aMZUlHZCs!IKalEfv>MxMp%ApOcQ>MXe>*Tq=riGpT^=`JJ3*t4lS^- zwQ*v{&m@iZFrz!0Mtd00XgLkN$#pahFj~k`g?oE4gN|XsxKlpjBRY+jOvScUB#jBJ z6^7GVqCB?=THlXG&n0ycVYva8svFT*X;wT_BfW1LzrlZ8Mi94QHyBH`Y6OplYA;3O zyzyHOjq^aGO%0!x>u4IZ-w5~CvT0mb=)@YYjZTWpO|>Ftm9o@plW0Do5qagEz@;Ov z_#b)8CJlT-RT^RGQKX5-A{voqkBOdA2hmu~_Z!E3gd`0^EfoZ*e%L_}l8+~)$?Ppp|Mfwt2d+l%8Ps#JvKX6_xa6497_ za)`QEdfzmDuxEfZ!Earqw`T%-8U^8QX8+1sN5A%u`+K8 z<;Bs6vNrrh_0sG(vm#AsL^PIW4d1_3YC{{=TWr#>kKah~1seP0X?S7zv{%mIW9sYL zUZwoTIIb5bX|((a+dn?H8+-l+Ho6ycvtTYcT1hlvjc^4;dQAr1Y*Fbsbi_oshq|kn z{@h=LI}{DNFN?T6>-oWWadqfO zt~P4Vk0ohz$vU>Pwb3O5Z8##|xGRm{7vY9u_lj^|R>u*RZisNtB5kZRdn~bSsRqve zR@x)qP-%IVszw${(&+GL7qe+}cpxwCfC#rgjr)plfyO-|+>61Wjo%=`y@qHckDl4u zegE39kEPOJUR-UQDEx?`;n%L?(C`Bqj)-vk)3~n)HypcHggZWXDa2HbY2wj{hP-4* zzD>U|u%)sOjny^oA`3hFtC{pQNuzaxfipAGx&dh;0&IWo!}a3*Y0%p6XU9~+`F^F4 z`%Pp8y>RGy8xO~32837iJR&Es+bxuw%#2XkpC)Ve`~;~j zzcFA;Wgi-84d6E_>V)5mQoTIF%() zpHC2%m;>13m(?q?3%Vh~jYQf=j}ZUEQ|cfZsl`C!fc6mnX2UX|A&aifqr7-$PAB3ySC4X0{-97Dq`fJ38XkG&(@@ZXh- zhhz83#k~#4Z$K^{i)hR#5iRYuMPp#U(G3yq_P-5w4p*h+%^KD`{U4OyxL{~x(`dZ_ zG#rtO-_>vYzFa&UyH_rLJoq^(7r%jMtQhUHzs4qw0sV%3a`83gL%1>3@k<|&G}?8> zn`{~m%f$zv;Y*t-*a;V9YBps?>bMglq~C}zyiln)efpCL7e|xUhUdlcYISOMK`-PR z7^3l9$+*w;_fNw&314Mjs5AQLpuq0S)eiIvG<%4I2?~dGk{q<@rfW z3)nFgeXcPbYJET>b+r6RhtbF&2(C6xLah(g{yOe(y?DnRpwU{^pN135Z@3ldLqq89 z3I8rZM+}WWZ-}5T44MX=Z}{(7)rUhE2bEZBD8-zoXdKiWt&4Q|aKg{?$QJOm@t5EQ z>(5>XFHX)HCZ7zP=P9+}#kB)kD*I?dtrrwn-o_O8&gKx%#;E#QZ30Cjvwa*}8y%TI zqos*Qqc=x8yS^``atqPVH_?MG^omDE42=VX-Dof|RZ1~4>R_qnC?Td2YiwAmS^w)B z@N>M=Wchw;96>jc-1U4B{3_%diZs_JkTy0XO}V;OYJ-OLo(C9Hsr92omP6QXyr}$H zpz&CJonI_Tqw9LtC^n6*>qr~%JQ_Sp<$6xPkI{x8i$kMa*TnS4PL|5sC4ogFMyY~= zhB%XpgUmOoo(VX*>9D+fkM;Y&v;^&YKCRl89cKa>iHJt>=$M{jsiSCUhl?QJ$i#Q8 ziUb;u)YtaX^HkH@_p@nqOa~eU$-p#V4)^J5x~rl`I5ZCEljsZfvQ*xjT>NgU6vDB4 ztx`A}{4#5m!n&nW&jedE299v;6H}@66Sy-Yb)^)IuEb6in?_e6(ncMOXzm|mW+cWx zl%>A6crGd6o*QO&BYta?Q}c zg@B<=@ITdP$IvJX&qr_3Hdwzw$5b)ieK4RdeqJNXL>b2`l z%=CCSaB9-(NfMI9Kre{K!jqm?yKT|1-uM7xs$@B=HQd@TAK&Ub5onBvu9^4-NyC`a z_E*rxF0e!7T0`S0puwrc+E{DoWYR}iX|$8wXS8AM%;2@b(C}W%qA~YNAF+?Fl$^Qk zJB?_}THhMD0WqyHhI5xcZ!q{C>aLp?;6gAK1T5pMMHQ_Dyit%a>YY)jRK za&bd7qYdZaUaY`^`Q>PZtb7Avsv;BJOHoEOI$~&)-&Ql-AFQQf{6FT#LNrw7M}iWh4zg6K%@X7{UdM~D z1kgrM{qfxtjgIX$y!fx985;B|1)il6gy>((pa)%erU)G|G!9)A&=&^nH)tB+-oAZ< zFN|yS5oacfIV%mR`bN~8j{@+wx0I00VcKAGk7%q+8lUPgZKQj`JXN`N1zs$ta`C#g zt4VFN>)ONE+UP))SmWQ=G^qN9GdHFRvC_y7L0x8O?Ds2SM!-(m2=^{#)9Ce4DD#wg z{j_o$xQ{R|#-ACN0Ji_%G zl%?_?8_=ydJfgWUi);kXQuX{py)7#aRw+>VMs`!k^aQC58V@7FU6bHZU@2+Hgn6pj zV5!E{AHPJ^#Z5=MzGTxdAuoQ_z}5y-oA{PWqAwv80o@jUPI6&rln+Z{fY^yf_#Sq? zu^^IK`N_pWM!3o!1Fn1!c)dvg?uaHCJma3w93!N^^uU@!uWW{#x-8j$@B1NA^l$>%p9-&n%X5WJp& zI>694m~3yq;T;povQ)8~GG&qCNXRxcvc5+K5RKV&Ih9ZC2^d9wIjm!9>j2SMSWgfR zqmkN7*2UidFTNCLJQiKsOV!2OuD7LvHg>%lOC0$^FKNE-Byxz0^V4NLxD$dGS10GZF_h{OW7Zf8;b7>KkpcR+ZD^l;3z9K(wKGs>@Ln zjf*$1q+aY%3pOlA%dU$xIdo*Du`HOrU@scpoa}Fo*Lhf>&b3n%jaeUO z1^l}1Qj+g~8<-Zbt4%%cS8dJ`T?fDMGSbFtO@4t1QX4cLhTl-@y->b!1m*E-02x29u^9+sBVWZJov<=o$(eqT^FZ;6m#vIkU zNI$Vkd<3c-R)vr;)vUcem6J6p{P228oje*n?uIwBMPRAiTM&&EC;ik8qmi~2vcGce zUdT6;K*K+}&X;-$lfO!#T?aIN-OQr_F_j04hTA#)iV$kZ3E|EnbY!K0+l_`b7iZIm z%_O}zHyfuS+>cLHMql`BsKVV!;|^=BAzoCKo_!YPaD^KYjf5s|$Hi2s#boX8bMWFR zKtonvtE0TQ(Wh+}+fo^QfJV!`s!j3w&5SlgB9aS3qin)S8q=V)!OT-}YJYPzrSPDc z$k|7nc&tw{^;6sy&|jN254g+qOr^@xoOi< zS?_s(G}d&(dMcFt70cy7V|?`S51Xl(k#XF$hVA2k#-)sbX}Fv8)0nTIsIUlifT2;c ztc2;0ooM)PUCW{|zgMNx$Ra0!I5aXPGe3f8ME&36L4Ue-`OKEfXP6eSV=BB#y=VhT zLyTywZu0rTCXI&@Q-wnIS1M10^;DCA#zg!NA-=%HdgSkwc&nFzp$F_ zvS{Zybi~kjCoZ2Ei^0-ho*3X*s<=YcCNG;aBTYzi-)er%OtygU#V=NE*>Dyj zT$gr4BRyi)m;|W}OZ6~lq}PDoINFd0l~}Y!-8cF~Ayxb9RCRFXsXEa-Rhtbj&ZYs` z-;-7vQzWSK42`!XN#t0-i=AkAFZN|=Beu~GDzVmRin>C2@n~0l&_b>1m*k8HvITq^ zlB(?+M6gEP{bNMqnUat%)=C{jL#2i1Hwv-Yc%IfDXbg|8Ib1ko5?M4P!aB$S%K?X zK1*tO%uKd`M*~078k3OM$fl~7a;qCTzoj95nnPb-C;D+d|^G+VVE^6DvtmfKJ|5tRJF8$p8l4rHSOi(Ab@B^`V2cR7|zWe^uYG3zcC|-#8+p zpQ2$dAR3X&eFF2Rt~U8)S=GsWFVG&J2K9}zJy75HJEAfFnX78fraN+n23fY;71@KpQOwcr=hUSTx*B`lxEU*}~%zbi~joO%pKv zu@{X!zRaM*%FhJ~g-(+vKEgy()Fo0IQ~pxXLIm!JXFmU~dqhKC;(Svtbr6jdG;8>V z_H}%_Tmdx3N7tOCo)~D|aFz3H6Zza<^JN|lNscyvO`}M^B!lj($n_jLVrU$g+(cg( zw577<8*INZM^>m)YLuA@4h}?P)~DKv8>hB-YV)b@1J6>?tzy=wXO%p0)h3OH5mTiW zLnZcYY!PUKtZ&rUE~nPvn%ujbQTF#^JvF0latCenvs6640WsBi$_IlsVuH@o8Y(SB-u0`4g68{?5sG* z{={Uyp)46{I~RW#G&JQRw4O@Euf*5|f2Gb0B5A}^_v(Tz& z!8gcKEjCXO)JPp*+SvO|6*CNWq7n1EerE(?s(bv-h^A`h22_a^UPLt1 zNh54WxJ>J9|LTiCXpC#h$}uS_OMVV`kHkc@EG9 zYDG&f`7}+rR?*K=8A`8y!nRbcpMy58?BRKFsKg3c+JJn6@)q!ayVnsNiw)>4 z+6GI5_8b2GtNMmrIM1t8tlO>B!0%HjWSP5_g}O=3==Z6>i!WReG-IO6JJr;u^I!23 z!HYMXZ71#sF2W^qY|^9xhtbfKOA0KM$e4=I1(CV+R=K!P)+^#Bz%fLmG*$pop25DJW(EquPEJ7z24;pv1_lbw&W=C{r%<;b28PEh3=Con4B89~ zj0_A+KqmWh9yT5ZAm9WFH5?CMVA#R1Y|lbXg_TQ|sVtZvfB^dEn+=#dAAOw6_s-^* zT+7kfw)y|xKmPy!{|}bU^H+E3|NsB<2!l6+{r`U{g@O}}BKUz)4D$cn|I7z6kiZA& zzY_cx|KI=rzrnly|Nr+F7=Q-+a$x`e` S8VT%oN8&)3=0E`SB{KlP$Appq literal 0 HcmV?d00001 diff --git a/examples/BUFR/ISND01_LEMM.bufr b/examples/BUFR/ISND01_LEMM.bufr new file mode 100644 index 0000000000000000000000000000000000000000..8526ad67018b1b8b5d9ca69fa1e780cb8f5409a6 GIT binary patch literal 1374 zcma)6ZERCz6h5~fc6Vbbu2Q2Ww!@BXnV4$EhiJH+u(_p<5!fW)4`$;B&WQZc5HokL zQ=^zg$TDU$LdR%?kw8o!koa|rE&+a+2|-5|_1-KdTcWrc>)N&V-s3r!^n*XfC+#`U zIrpBo&-T;$qu(pVWsQ_s>?H zzu)4UAF40Q1OicC# z5F;RA3KW@M&|STx45w`^D(z#Gl7y7_Vp{QHhD~bYkS`yFyBBUEW+wS5skGRaVc=j{ zfcXAnhke)S^Fw<@89Z|r_c-r={ebVKM=P$p^L+igaWNC%I>WK^i^FFm#HBRcr*ej2 zc*xi?F%}oUnuktwRD@w((pYlzX>&O;bfH8>cEH zl^XjJ7Os{!vA@LO!OB>l=vdk4=&Ovy-Wa}D6a+A`t06XBIXK+1A_^yEG=Zi)>EQ7}7R}>Ewr4o0X{)aqP~Xok3j&l5RoWOIl`n2f48-0w2!DvS>E>9T*QNUHrYU%Xbve& znMG>LtV~Z?CHpd{PnHFUlMhWabTqYxT(`mbw|-7I(DCyJy{|Zz2F2~yr~32naUJ#d zmv3hdGSzD|b_|P$J1H?W?l^{+=F12bJU$k*(bS+xUC2pvhN;4RhEchl+F~2${6*>W e2KRuaA@+As6_UyY_9bNIOPpAC9fAHlo&N$21(dS@ literal 0 HcmV?d00001 diff --git a/examples/BUFR/ISXD03_EDZW.bufr b/examples/BUFR/ISXD03_EDZW.bufr new file mode 100644 index 0000000000000000000000000000000000000000..c8fef8da504f9ab76bc0e9c2a0e88e3cde793921 GIT binary patch literal 453 zcmZSN<>fLnG*tjnp1~0=2F41mE>YnM2Brp<1_la2L5@HPr%<;b2FAH83=Con41Nq? zz$(PS{+x%E6T~^rz`)qRD9$K!FpA|G%k_hiERrJcMc%PUIx)yIG9DFq%Oc6aB*HCn z93ovz3OuQf?8ogLfvQCgIy1;Kh#V3*EOGg5KJBBx!UvLG3e(;&6Hr$x?)ob`Z;^PJ)lk>3t;wy$z_5jZq5eM*bX^p=D}6IoRx-MS0|D5d9}Hsli}}Z$euUNFM`r)OA1w?V49jNx|9{tL|Ns9F|Nme3;j%nP#s3Ek t`~Uw?ZQ=YolOg^4xBpGSkLnMWeq^-ai|<^*H36vi|Njr>Kmd#%W&o8fk(K}e literal 0 HcmV?d00001 diff --git a/examples/BUFR/ISXT14_EGRR.bufr b/examples/BUFR/ISXT14_EGRR.bufr new file mode 100644 index 0000000000000000000000000000000000000000..076104e1098b394ffcbd7b1b2c4514349245bafa GIT binary patch literal 15190 zcmeI(X;f2Jy8z%2a)dylOhIPJgdq^fiA)iKBm~R>f&_?yf-)BrEmCWpS_zX(Qb0sR zw8|s`D&T|@G7EwhMJQ9f>onZ zwH#3wOZ+b^C?ptlu=H9S7--3NM1>3ZI7*b>fdp3pi9j$U5PS$dD; z3(ORjT|jwIQ-J+YfJR0zAkR>vqAuUoONJQ@1?nY3O$f{vG7VyE1Q=g3%uq0YWS9v9 z27!LSOoVZOIJklN6G3t+R@egN!3;xy`-2QM8gQSJF_<|T0rg@~GlRrIOdbHnK}=2s z=L^h~R2EPk+`#-PsSt|<+ouGLico<(!_Cp{+xAfrr~&ojaS#*!FAW27Peq^xSE1~H{Bm=}oA5IL}(LX8aOnFcdmHn9KFV8*ug z`*!}(U?!)!0dY{{>40$%iv-tL8pMo^9f5rsYA6Qu!x(0wSYS;wh8YH)$Bkj86awp{ zF~rOfHtE~Ez%3TYpE&|Cwz(Nl9@H?fUN*!~1jwH`)P%siH%DOxuJ`6B#6)7?{xF9c zIKJj0h+#-PkQa#25e&#P)MzM>XFA+~{NZsB!+`sUPJ@^Tjrq18bf}4d`y}8}@Cz zJQ~Enac1=mv)W;7a* z7ZhsX85pHO%$yF)Qxs-wNc*5hrF=VXC>v%%FfVL~Im&?Z3kow4)eNW?W?}@Ge<;Kp z5%9ANxS_!Q;fO#Ds27iem=G8bM+9cF;cxrvh`)1qw4_Wp%4>+dxay!gh(i`o)rvJOkjqltYaM*1>}q zWbcu~Ob9*)$YCZjW&wGDnh=}^BA6+FeM>Ham>67}~aOv!2BsNm<0oQQDC4cU50@;sL2MwIEX>^csaxrh!WV}phnXL z?n?#Kz%!wO1~E3cjw@h>@`3eF0W~==Ulb_BMBsX*fExIWqYyz%44%UjFvE0#`%(cl zKCr(jFo-G1;J86dNkxGD4QeR(e6EC8D1rich8sBMp^)c=P-Cj)xAPflZ1DLv6l#9J zIv&b~7z&;@LZJqJE+2}577qDb{(mrFjlknz2AogvIM8V9uy6Ah4mI!@F&y&yHqXs@(rWV4Y^h1xial-jfNj0d`>;yy=I`sZ%An;JYEGtRVTjI7>`mFOXRd?pKONlmc2?*le@ZNig{Oy7VTpll|EaZJyhCs7>)M(?oH|M z&kRroCrX-bm8~f^Fge3o`3p zKUQMVKQ8RQlO!e`>5dtuCJZMBY5Qn)TBlygOf~P%;Jn92j0cc{cx}s@QMYQl>%9}L z@`mM%fI;b@Lknsrzf;`iq~|Hf+H2mI{PD^=*(h&vb9?Z?!LK|*jIO_I27eHM`7k&MXuS*?$pJ(q&;^M?3*9|W_{H?NI5oD-#z^2 zYQ@;i>OAi?MtOagJ9hn=vFcgYr6JvUa|G@d(Vk{CnZkf~F^8u5EY`1c8+$OvddwoI zePD{Yci8{_KjvrumKMCrroUj{uJAo7<6qhP2&nQibpDIPaQz!o@2tX*Xr4&iJrZL4%wgs&l@K{*%KzHI6&cH}&)$ zWG$x6Dl)`Od$gSM3=WTHFS%^`_^R{OGT*+$Jwa4h%n~a`LqjyFhEa*J9%D}yq^Ya4ISRA=aX&GPValMjO>22fp<4o!EKWXb$bMN2m zoJiQw+h3`%$9vAo*8>-yln!Kt?$tS2za3lpxvb6KF{K>iMHDS>`(k|d^MYjBf-3HJ zON;xCTJK^QwMfxqHLYz#gV(>!KJ0GpQ^KOz%0ly3pD?>+67S2)n|^*4_3`&dzs~Eo@PAyh z|Ek|^`XR2TB&VjECeP<~mMeESH*vm?3K+L;3zu>#vJ_I~#rXOyH|fe+N&V*YKCd_K z-e}03vQ_RDJm|3ciwsHke{4j zHqmL*m$_{B+AsiXt2-Ne4+?f3jFz%`vM`3Oo|M5ZOYn8L zv3uRVx(Ch@tBG`Z-_@o2%8oidzuf=Fue+!UE0c=ggN;YTk%7nbs>7^CKD=mu{h-L^ zJZqUknpu%e?baL0VOO_0*ldeeZhKqz;dv;%q}h{ampc|dx9j;?>pzl%HZQ9AYkN?f zq-G!^N$=bU`^OO0WFvM6JO)ZGk8a8ddZtHO&^<`&xX3pRgjlD;;{e(X43N%XHH zWw~KGn|7Z+Z1a}hj`C9((V;m# zC+6NZKi=RP@e8NzlOl+Du4eI1NVuKWxsab<@jpIPIvCW&TL&tOs_N&LOwZws{O6yb z%J#MuE>K0MrC!z`p7!oe+*yd;U_@6f-{f*~di|=Nu}eFyE6!shItWQ31w;2*;qt&(4%>I(gs$cf?u5vDD&A<4GY125mHg)VC(j^yUmBs2`Vb3r2S-Iiqg=g1)EU$Ym43?YLmN^IQX`F8# z!HN7Up!1RHey94`3}c0spu+gl#}nJ%huExaT-xeiF|psP=dDCOnO=HzQ&ZWWh3_9Q zU!9#x)w45UWuLOvUmeJvX8DeU4mX}?isXh0f}3LxKJ6D@FV)$~w|wWBleoj+&it@R z|Fgq_k?Gj>q0*N5Xp$;Eh&Wh7&2}@@+x&!g;@XN;wLzBx4zHMf^R#&#mFHlxJm&Ki z@;}9#m3sp&3YPs`a_pn6q-iiC`pEtu{e?A#J}wQ#>oXrjQ_pq#=P~vhd0md`&+jhc zK4d8`_&wOP_uMq*ZdA0*IlbL!h|9KGg^ zajPN{7)4%#ND`H=j!`O)(*(mfaQ>7b7f55K>- zxmWPIAf_cZd;Q$G?QX|YHwWySKIA;QYDDi+o3%ePu-v;Yly$Qi^CM1_B7I9EB@G=J z|I+PmW$ruIcjm`k$p^TP+>~}Qldu9Om-)6y{)z3s-AP-taplO=#db5NyDn?bJUR9K z@2dw_&OLa+zVP5-rda;m#^(0Y;MaSOJh5N)^V(y7-@DlO+tx@?!RwFoCvU{*S!rZ; z$Ho4PtqJcZhhE+MJl*uqXxyuJYyUMh^7kw8+lL`aN@JbzHtO9O(^J<&Y$_T<%2`b= z*seK3D~t65yY=^SIWf&ieKEJ6rtz-NQWm7w)5+`6g=0}ag@wFqX&SBe>h#G~XGKUz zma|gg?T90DRhy8M(3P|igb*;Fn8Z!3B@+tpw+z)Hhy+VrH60_h6jN0T{sKt}(NU2L zjde%_57)F7{6|x@M*O=L;h;Qy_^8(4N@OH1Gd^Kffd(O|EW6e*Eo64e(@qM%ReQvS z*hor|8LJhj6Zi&d9{MU5V^yg(p;3Y8;CDIxR#lFSLrgJK^*X6X_@vib6AS1>9sF_& z)l*wyo36?TZ`B;aw2n&i#~mT~k6Sew@#YazIA&^X=Bi;+)rBhqwVO2xW)UJhNCdoO zyiEKLUcwI={6*^1_;_FdBd*U?SQ%O1pZ!n997wmz@;E@(;BS||n@s58jlaN$Gj^Lzb;i)rJ#T%$( zcv`emS~OC&Id#V4g?}xL5MY%WZ=}k?U*Ii#Gsk05+6G)y9h6H9u-GJ2UrKHg6JT3+ff-d{W(?>>I*0(Cy#_r>X(@S^aZ zZrjm|cM5MFUPVt{JShdsFMW!ahQEckQ=o3CM$o}CjQ{!1*s+KbqeIzS_SE8+tjIXa zEvmI767aX2)sygH!c&cpApREq8h1Sv2X6zOSUgww7&)p3_@_TLR~t;t;^1>BFRK@y zJh_^U_(0=*$NLh15b(a>Lx#5&uU(Un;!TRc3&%UoAX?&+ikdR)l^&w65<2XXEz8*4 zU*58H$JuVPpp&j%Z7I`>zNmj``>gR< zcg(r%qreo+q~7`fxdi_(;2ztrR1niUK5uF>n%KS1s^I`?(X~c@%BR_>&zGEMjXHFd zku)i&f7jhFd71*!<$In~>WSr7?{WR=j%l@U_vdUl`l&^)zh7{yuH`k)_S8}C$rG9s z%ckMWCb?YR@w(O_N+OEZ^mK|}XgOCrw3(Y1-q_IboHF3x@_Bsnp6^eB)GNE{>}u+h zkM>h33;B0D(@3`D6Lr_t(DPJ{`5i%)4JnNeJJl%zoF?~frcK`QX0zTPas%7WqD7n8 zkXJu&+?ADB(%f++*T653zokQ;F_oV}An0Ax`rMp(B~4?1-X>H_o0I1u*w{A|{9@_x z)2)Af?0;q3&^$!RvqL{$u2-AVZEEOQxQhIX;8@?4om^q_?S5ZAwLO|4k<0AkFBZ*c|IzqM$BDU=Ky+L8KK-dcesAXjy#f4Jc3oq) zZ92+~n8YSS$%%;66~lb?wV$ zhu(@0xv!3|yjV}4$`Ty7(s6Y4nMOW--INBuren*eHIq$~mluBVBQ#TbnIwuOe?$AY z;lT37o{oiHM+}>iyXyk_q(>#EObr^83nH#oT2IxVwjJ)Dm$=1s>Q%*)HT%(by{4MB z^NtT*V!T=NJD=2cLSu@#;J~$Lr>T4P7g|?XQA)U++7k@FT>9*^A!cH3)9lXYzDKR? zH?(@PD8Dwgbku#enGwwD_0^^LHy!H=Rh^R&oc93XFPvjT=^?I$c3Vc`^1^E)|{U zzC7O6dZRw4UQm7|>aq>n?&P`g*@+I#{Jt9xwAbL1foVb6$S>}QwrjZLU@{o^>UrCO zylYxt)_iW7zP!&mueNz*U*O~CrA_%gKJsULJK~wmZ@#h#!@!yUw8%YfOI`jdADJWa3*pH8rL{$aBIawkdWVCs}r<*|e5iqZs9 z`Bt7$NTtKw)eE1$sw%=R%vk;_Yoc~OhBQ(xM?d{KSD*BpGsQQ`loe|An7vS#-t$Yqc?RMuWBIz#`8OhnlfL!M zX(br5VtSFV7>XEi3Q8Avdaia%UCOR)&h2b-9hF=IT%c`V(U8R6vf^q3@u z(`^ux$n9Y1h}9o28sFlg{@ATnb9;K$;%Jtj^7EqTCWaDb@v4m9$xs}P9jKR zmP)@KOAc_WEduHStD}wh9qX(;9?xOMJ5`4z+^kU<5BMJLjc=XFf&w_VX#d+nwd|CBwiD#o5GAn@L z#`NrRt&DN$@L)|g)Wvp$scoZ06>!pYP|q+YiG-0}fmO6}t5*i6_SmsV9!x{N>ZUpK zu6uNuo3s&2pOAaAVydDROB!vY7kox1&BvnHRUuv#kNN72q9{I#74J2=P(QNBC_^hS z?U*a6)Q9xaKeHxYTHzDBVOiTtoiulc=o$A8!aQ%2$mmvgwLD{@9ldh(!ch<139iI5 z%uym>#>OqnbdOHYlx5rxXW@&~rzLDXMb|lBbqkcu2#juU8T-tuwB(Op5viYcSGPn{ zMp?;1gS(Z?_IW6jl;sD!}{?(_8e*yDuq07nzDG)-J+TQ0U_ zC>B?aa!Lfo+DO{yOhZUCuUKo=4?@2hiDhoam0Rm=ON8cT*K)cQMmealPL}-i;Dwmz z1@7}mW9uCbNEy4Z)ikxllkW5M`??rvZY835COf^OIfL=vI~C_J4h4KlS;n1nd0c$t z<`hEA$*3Ar^*;HzP1~z0O29Tq78YfDMz$2o7FiT!R~4b8)D<>!wX6U7aN(i5r3vMc z;NU>iGH!8Br=s9Oi23Y%@5itAlC~+Wb26e11>9fvouD8qYQ2DKD*dw@q)b$2U=Irt_)chrnkx1MtO|(NKUbYsf`@1x!yKnPNjIOw?EU& zaey`&ov#j-7KMOc&?p*GNqJ&RZ^*h@+0(#o#x x7#fXfPIkcuWr1$Nm6dkCYGiuS&2n7L=IluCh-8(zns5I^AUNY+`2RGk{$KyMF8}}l literal 0 HcmV?d00001 diff --git a/examples/BUFR/IS_2023120401.bufr b/examples/BUFR/IS_2023120401.bufr new file mode 100644 index 0000000000000000000000000000000000000000..af2581b305d8d72bdfc120cb173cb5a4ea547e9e GIT binary patch literal 1644 zcmZSN<>fLmF;)Olp23cuhNcR^&cV(K1||kTpb!+~43uyRbqiu(b6{a$5MyA_WdH%T z|B4Lk&v}4CK)?;;GaQ_DJeq+`n1O+_fq}U>*qPag2@d%GFlaVPt(mpid-i7U$)1cz zfZ@*%1qO55S)wAdy*~B-hkyqR4EzjC4h*6U=l?(dQ7a4;-mbst?Q|%c+SCV;m#qBM zG4uhW0Ru}?K|(@Co&yXx{%>Hdn;$yMu`013Q2`am&u3v^JHKtw{H+xUJE2ba!@$7! zfWd=-X@TVbKVRYQ`pyyd$r8?^BI5@GV?FhZ`@^8dz`)aUhKGlzg98TaKQKI)vub0Fl~9#(W2!$8@`b~mI)*l|9AFTfd9bl@=1O>KXaC2bmm(YO>3j8L@16!!P*)?s zAbja_m+L`>&*e}jd|+T;Z(vkl;A%+!|M?-@U3ZwHix$Mm1OD?(3{2(58{KzCN%lb9@sELl;RAyT17iRWvdezg zUQWAwA6bM-?B53L^;9wnXv{wbU~%Y{&c?>pt^!NyEb>1Xc1|+Qbv1VrdIT>ZAq>W! z9~>C`CP!!r1|GJ#4|T@@1_n6>mIekb#`*ux!wSg%{|j<+O-iZfRD? high: + return None + return val + + +def load_xref(txn): + """Build out WIGOS2IEMID.""" + txn.execute( + "SELECT wigos, iemid, tzname from stations " + "WHERE wigos is not null and network = %s", + (NETWORK,), + ) + for row in txn.fetchall(): + WIGOS[row["wigos"]] = { + "iemid": row["iemid"], + "tzname": row["tzname"], + } + LOG.info("Loaded %s WIGOS2IEMID entries", len(WIGOS)) + + +def add_station(txn, sid, data) -> int: + """Add a mesosite station entry.""" + if "lon" not in data or "lat" not in data: + LOG.info("Skipping %s as no location data", sid) + WIGOS[sid] = {"iemid": -2} + return None + sname = data.get("sname") + if sname is None: + LOG.info("Skipping %s as no station name %s", sid, data) + WIGOS[sid] = {"iemid": -2} + return None + sname = sname.replace(",", " ").replace("\x00", "") + elev = data.get("elevation") + txn.execute( + """ + INSERT into stations(id, wigos, name, network, online, + geom, elevation, metasite, plot_name, country) + VALUES (%s, %s, %s, %s, 't', + ST_Point(%s, %s, 4326), %s, 'f', %s, 'UN') returning iemid + """, + ( + sid, + sid, + sname, + NETWORK, + data["lon"], + data["lat"], + elev, + sname, + ), + ) + iemid = txn.fetchone()["iemid"] + WIGOS[sid] = {"iemid": iemid, "tzname": None} + return iemid + + +def process_messages(txn, prod, msgs): + """Do what we can do.""" + data = glean_data(msgs, prod.source) + if not data: + return + sid = data.get("sid") + if sid is None: + return + # Remove null bytes + data["sid"] = sid.replace("\x00", "") + valid = data["valid"] + # Don't allow products from the future + if valid > (common.utcnow() + timedelta(hours=2)): + LOG.warning( + "%s %s is from the future %s > %s", + prod.get_product_id(), + sid, + valid, + common.utcnow() + timedelta(hours=2), + ) + return + meta = WIGOS.get(sid, {"iemid": None}) + if meta["iemid"] is None: + # prevent race condition + WIGOS[sid] = {"iemid": -1} + meta = WIGOS[sid] + df = MESOSITEDB.runInteraction(add_station, sid, data) + msg = f"{sid} {prod.get_product_id()}, {data}" + df.addErrback(common.email_error, msg) + if meta["iemid"] < -1: + # add station failed, so do nothing + return + if meta["iemid"] == -1: + LOG.info("Skipping %s as iemid is currently -1", sid) + return + LOG.debug("%s %s %s %s", sid, meta["iemid"], prod.get_product_id(), data) + # This likely means that IEM station metadata has yet to sync, so there + # is no iemaccess entry + if meta["tzname"] is None: + LOG.info("Skipping %s as tzname is None", sid) + return + ob = Observation(valid=valid, iemid=meta["iemid"], tzname=meta["tzname"]) + ob.data.update(data) + ob.data["raw"] = f"BUFR: {prod.get_product_id()}" + ob.save(txn) + + +def render_members(members, msgs): + """recursive.""" + for member in members: + if isinstance(member, list): + render_members(member, msgs) + elif isinstance(member, dict): + if "factor" in member: + LOG.debug("FACTOR: %s", member["factor"]) + msgs.append(member["factor"]) + if "value" in member: + if member["value"] is not None: + LOG.debug(member) + msgs.append(member) + elif "members" in member: + render_members(member["members"], msgs) + else: + LOG.debug("Dead end", member) + else: + LOG.debug(member) + msgs.append(member) + + +def glean_data(msgs, source): + """see what we can do with this.""" + data = {} + displacement = 0 + for msg in msgs: + LOG.debug("%s %s %s", msg["id"], msg["description"], msg["value"]) + if msg["id"].startswith("001"): + data[msg["id"]] = msg["value"] + continue + if msg["id"] == "004024": # TIME PERIOD OR DISPLACEMENT + displacement = msg["value"] + continue + if msg["id"] in DIRECTS: + data[DIRECTS[msg["id"]]] = msg["value"] + continue + if msg["id"] == "010004": + data["pres"] = msg["value"] / 100.0 + continue + if msg["id"] == "010051": + data["mslp"] = msg["value"] / 100.0 + + continue + if msg["id"] == "012101": + data["tmpf"] = bounds_check( + convert_value(msg["value"], "degK", "degF"), + -100, + 150, + ) + continue + if msg["id"] == "012103": + data["dwpf"] = bounds_check( + convert_value(msg["value"], "degK", "degF"), + -100, + 150, + ) + continue + if msg["id"] == "013003": + data["relh"] = bounds_check( + msg["value"], + 0, + 100, + ) + continue + if msg["id"] == "020001": + data["vsby"] = bounds_check( + convert_value(msg["value"], "m", "mile"), 0, 100 + ) + continue + if msg["id"] == "011002" and displacement >= -10: + data["sknt"] = bounds_check( + convert_value(msg["value"], "meter per second", "knot"), + 0, + 200, + ) + continue + + if msg["id"] == "011001" and displacement >= -10: + data["drct"] = bounds_check(msg["value"], 0, 360) + continue + if msg["id"] == "011041" and displacement >= -10: + data["gust"] = bounds_check( + convert_value(msg["value"], "meter per second", "knot"), 0, 200 + ) + continue + if msg["id"] == "011043" and displacement >= -10: + data["gust_drct"] = bounds_check(msg["value"], 0, 360) + continue + if "year" not in data: + return {} + try: + data["valid"] = utc( + data["year"], + data["month"], + data["day"], + data.get("hour", 0), # Unsure if this is too forgiving + data.get("minute", 0), + ) + except ValueError as exp: + LOG.info("ValueError in utc(): %s %s", data, exp) + return {} + # Attempt to compute a station ID + if "001125" in data: + data["sid"] = ( + f"{data['001125']}-" + f"{data['001126']}-" + f"{data['001127']}-" + f"{data['001128'].decode('ascii', 'ignore').strip()}" + ) + if "001002" in data: + ccode = WMO2ISO3166.get(source) + if ccode is None: + if source not in UNKNOWNS: + UNKNOWNS.append(source) + raise ValueError(f"Unknown WMO2ISO3166 {source}") + return {} + data["sid"] = f"0-{ccode}-0-{data['001002']}" + if "001015" in data: + data["sname"] = data["001015"].decode("ascii", "ignore").strip() + return data + + +def processor(txn, prod, prodbytes) -> int: + """Protect the realprocessor""" + # ATTM we are sending this to the general text parser, so to set + # various needed attributes. + try: + msgs = [] + for bufr_message in generate_bufr_message(Decoder(), prodbytes): + msgs.append(bufr_message) + except Exception as exp: + LOG.info( + "%s %s %s", + exp, + prod.get_product_id(), + prod.source, + ) + return 0 + total = 0 + for bufmsg in msgs: + jdata = NestedJsonRenderer().render(bufmsg) + for section in jdata: + for parameter in section: + if isinstance(parameter["value"], list): + LOG.debug( + "--> %s %s", + parameter["name"], + type(parameter["value"]), + ) + for entry in parameter["value"]: + LOG.debug("----> entry") + if isinstance(entry, list): + msgs = [] + render_members(entry, msgs) + total += len(msgs) + process_messages(txn, prod, msgs) + else: + # unexpanded_descriptors + LOG.debug(entry) + else: + LOG.debug( + "--> %s %s", parameter["name"], parameter["value"] + ) + return total + + +def ingest(prodbytes): + """Gets called by the LDM Bridge.""" + pos = prodbytes.find(b"BUFR") + if pos == -1: + if prodbytes != b"" and prodbytes.find(b"NIL") == -1: + LOG.warning("No BUFR found in %s", prodbytes[:100]) + return None, None + header = prodbytes[:pos].decode("ascii") + prod = TextProduct(header, utcnow=common.utcnow(), parse_segments=False) + meat = prodbytes[pos:] + defer = IEMDB.runInteraction(processor, prod, meat) + defer.addErrback(common.email_error, prod.get_product_id()) + defer.addErrback(LOG.warning) + return prod, meat + + +def ready(_): + """Callback once we are ready.""" + bridge(ingest, isbinary=True) + lc = LoopingCall(MESOSITEDB.runInteraction, load_xref) + df = lc.start(10800, now=False) + df.addErrback(common.email_error) + + +def main(): + """Go Main Go.""" + common.main(with_jabber=False) + df = MESOSITEDB.runInteraction(load_xref) + df.addCallback(ready) + df.addErrback(common.email_error) + reactor.run() + + +if __name__ == "__main__": + # Do Stuff + main() diff --git a/pqact.d/pqact_iemingest.conf b/pqact.d/pqact_iemingest.conf index c94c3324..96c4b76d 100644 --- a/pqact.d/pqact_iemingest.conf +++ b/pqact.d/pqact_iemingest.conf @@ -2,6 +2,12 @@ # pqact entries only run for IEM, not NWSChat # +### +# BUFR Surface Observations +# +WMO ^IS + PIPE python pyWWA/parsers/bufr_surface.py + ### # Temp Wind Aloft Forecasts WMO ^FB..3. KWNO diff --git a/tests/test_workflows.py b/tests/test_workflows.py index efa9c57d..52a1af66 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -12,6 +12,7 @@ reactor.run = mock.Mock() _parsers = [ "afos_dump", + "bufr_surface", "dsm_parser", "hml_parser", "mos_parser", diff --git a/tests/workflows/test_bufr_surface.py b/tests/workflows/test_bufr_surface.py new file mode 100644 index 00000000..8bb2fede --- /dev/null +++ b/tests/workflows/test_bufr_surface.py @@ -0,0 +1,112 @@ +"""Test bufr_surface workflow.""" + +import pytest +import pywwa +from pyiem.util import utc +from pywwa.testing import get_example_filepath +from pywwa.workflows import bufr_surface + + +def processor(cursor, buffn) -> int: + """Helper for testing.""" + with open(get_example_filepath(f"BUFR/{buffn}.bufr"), "rb") as fh: + prod, meat = bufr_surface.ingest(fh.read()) + return bufr_surface.processor(cursor, prod, meat) + + +@pytest.mark.parametrize("database", ["iem"]) +def test_vsby(cursor): + """Ob has visibility.""" + assert processor(cursor, "ISMI60_SBBR") == 60 + + +def test_bounds_check(): + """Test that we can handle bad bounds.""" + assert bufr_surface.bounds_check(10, 0, 3) is None + assert bufr_surface.bounds_check(None, 0, 10) is None + + +def test_api(): + """Test API.""" + bufr_surface.ready(None) + bufr_surface.ingest(b"") + + +@pytest.mark.parametrize("database", ["mesosite"]) +def test_add_station(cursor): + """Exercise the add station logic.""" + assert bufr_surface.add_station(cursor, "46_&_2", {}) is None + data = {"lon": -99, "lat": 42} + assert bufr_surface.add_station(cursor, "46_&_2", data) is None + data["sname"] = "TEST" + assert bufr_surface.add_station(cursor, "46_&_2", data) is not None + + +@pytest.mark.parametrize("database", ["iem"]) +def test_unknown_source(cursor): + """Test that invalid descriptor is handled.""" + with open(get_example_filepath("BUFR/ISIA14_CWAO.bufr"), "rb") as fh: + payload = fh.read() + prod, meat = bufr_surface.ingest(payload.replace(b"CWAO", b"XXXX")) + with pytest.raises(ValueError): + bufr_surface.processor(cursor, prod, meat) + + +@pytest.mark.parametrize("database", ["iem"]) +def test_231206_assertion_error(cursor): + """Test something found in the wild.""" + processor(cursor, "ISMA01_FNLU") + + +@pytest.mark.parametrize("database", ["iem"]) +def test_231206_nullbytes(cursor): + """Test something found in the wild.""" + processor(cursor, "ISMA01_FNLU_badid") + + +@pytest.mark.parametrize("database", ["iem"]) +def test_invalid_descriptor(cursor): + """Test that invalid descriptor is handled.""" + processor(cursor, "ISXD03_EDZW") + + +@pytest.mark.parametrize("database", ["iem"]) +def test_badmonth(cursor): + """Test that month of 13 is handled.""" + processor(cursor, "ISIA14_CWAO") + + +@pytest.mark.parametrize("database", ["iem"]) +def test_nonascii(cursor): + """Test that non-ascii characters are handled.""" + processor(cursor, "ISAB02_CWAO") + + +@pytest.mark.parametrize("database", ["iem"]) +def test_nosid(cursor): + """Test that having no station id is handled.""" + processor(cursor, "ISXT14_EGRR") + + +@pytest.mark.parametrize("database", ["iem"]) +def test_notimestamp(cursor): + """Test that we don't error when no timestamp is found.""" + processor(cursor, "ISND01_LEMM") + + +@pytest.mark.parametrize("database", ["mesosite"]) +def test_load_xref(cursor): + """Test loading the xref.""" + cursor.execute( + "insert into stations(id, wigos, network, iemid) " + "values ('SCSC', 'SCSC', %s, -1)", + (bufr_surface.NETWORK,), + ) + bufr_surface.load_xref(cursor) + + +@pytest.mark.parametrize("database", ["iem"]) +def test_simple(cursor): + """Test simple.""" + pywwa.CTX.utcnow = utc(2023, 12, 4, 2) + processor(cursor, "IS_2023120401")