From 2f64b465352d3b0c1f5d8ef5d3ecff26497fde0b Mon Sep 17 00:00:00 2001 From: Brian Coutinho Date: Fri, 13 Jan 2023 14:57:04 -0800 Subject: [PATCH 1/8] [hta] Fix queue length summary to handle missing data, and avoid -ve memory bandwidth --- hta/analyzers/trace_counters.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/hta/analyzers/trace_counters.py b/hta/analyzers/trace_counters.py index 8e16abc..990b501 100644 --- a/hta/analyzers/trace_counters.py +++ b/hta/analyzers/trace_counters.py @@ -16,7 +16,7 @@ def __init__(self): pass @classmethod - def _get_queue_length_time_series_for_rank(cls, t: "Trace", rank: int) -> pd.DataFrame: + def _get_queue_length_time_series_for_rank(cls, t: "Trace", rank: int) -> Optional[pd.DataFrame]: # get trace for a rank trace_df: pd.DataFrame = t.get_trace(rank) @@ -62,7 +62,11 @@ def _get_queue_length_time_series_for_rank(cls, t: "Trace", rank: int) -> pd.Dat stream_df["queue_length"] = stream_df["queue"].cumsum() result_df_list.append(stream_df) - return pd.concat(result_df_list)[["ts", "pid", "tid", "stream", "queue_length"]] + return ( + pd.concat(result_df_list)[["ts", "pid", "tid", "stream", "queue_length"]] + if len(result_df_list) > 0 + else None + ) @classmethod def get_queue_length_time_series( @@ -98,14 +102,15 @@ def get_queue_length_time_series( "stays constant until the next update." ) - return {rank: TraceCounters._get_queue_length_time_series_for_rank(t, rank) for rank in ranks} + result = {rank: TraceCounters._get_queue_length_time_series_for_rank(t, rank) for rank in ranks} + return dict(filter(lambda x: x[1] is not None, result.items())) @classmethod def get_queue_length_summary( cls, t: "Trace", ranks: Optional[List[int]] = None, - ) -> pd.DataFrame: + ) -> Optional[pd.DataFrame]: """ Returns a dataframe with queue length statistics per CUDA stream and rank. We summarize queue length per stream and rank using- @@ -125,7 +130,7 @@ def get_queue_length_summary( rank_df["rank"] = rank result = rank_df[["rank", "stream", "queue_length"]].groupby(["rank", "stream"]).describe() results_list.append(result) - return pd.concat(results_list) + return pd.concat(results_list) if len(results_list) > 0 else None @classmethod def _get_memory_bw_time_series_for_rank(cls, t: "Trace", rank: int) -> pd.DataFrame: @@ -143,6 +148,10 @@ def _get_memory_bw_time_series_for_rank(cls, t: "Trace", rank: int) -> pd.DataFr lambda x: get_memory_kernel_type(sym_table[x["name"]]), axis=1 ) + # In case of 0 us duration events round it up to 1us to avoid -ve values + # see https://github.com/facebookresearch/HolisticTraceAnalysis/issues/20 + memcpy_kernels.loc[memcpy_kernels.dur == 0, ["dur"]] = 1 + membw_time_series_a = memcpy_kernels[["ts", "name", "pid", "memory_bw_gbps"]] membw_time_series_b = memcpy_kernels[["ts", "name", "dur", "pid", "memory_bw_gbps"]].copy() From 6ce6d8abf1a55dcbf40770de0cfab889e26db76a Mon Sep 17 00:00:00 2001 From: Brian Coutinho Date: Fri, 13 Jan 2023 14:59:48 -0800 Subject: [PATCH 2/8] minor typo --- hta/analyzers/trace_counters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hta/analyzers/trace_counters.py b/hta/analyzers/trace_counters.py index 990b501..73b4091 100644 --- a/hta/analyzers/trace_counters.py +++ b/hta/analyzers/trace_counters.py @@ -148,7 +148,7 @@ def _get_memory_bw_time_series_for_rank(cls, t: "Trace", rank: int) -> pd.DataFr lambda x: get_memory_kernel_type(sym_table[x["name"]]), axis=1 ) - # In case of 0 us duration events round it up to 1us to avoid -ve values + # In case of 0 us duration events round it up to 1 us to avoid -ve values # see https://github.com/facebookresearch/HolisticTraceAnalysis/issues/20 memcpy_kernels.loc[memcpy_kernels.dur == 0, ["dur"]] = 1 From 56bcd691072605e428eba4f05d54b74ccdd5ffa5 Mon Sep 17 00:00:00 2001 From: Brian Coutinho Date: Fri, 3 Feb 2023 16:21:22 -0800 Subject: [PATCH 3/8] updated documentation to standard format, added test case with no gpu kernels --- hta/analyzers/trace_counters.py | 93 +++++++++++++++---- tests/data/rank_non_gpu/rank_non_gpu.json.gz | Bin 0 -> 46121 bytes tests/test_trace_analysis.py | 13 +++ 3 files changed, 86 insertions(+), 20 deletions(-) create mode 100644 tests/data/rank_non_gpu/rank_non_gpu.json.gz diff --git a/hta/analyzers/trace_counters.py b/hta/analyzers/trace_counters.py index 73b4091..52cd1ac 100644 --- a/hta/analyzers/trace_counters.py +++ b/hta/analyzers/trace_counters.py @@ -17,6 +17,28 @@ def __init__(self): @classmethod def _get_queue_length_time_series_for_rank(cls, t: "Trace", rank: int) -> Optional[pd.DataFrame]: + """ + Returns a dataframe (optional) with time series for the queue length + on a CUDA streams within requested rank. + + Queue length is defined as the number of outstanding CUDA operations on a stream + The value of the queue length is: + 1. Incremented when a CUDA runtime operation enqueues a kernel on a stream. + 3. Decremented when a CUDA kernel/memcopy operation executes on a stream. + + Args: + t (Trace): Input trace data structure. + rank (int): rank to generate the time series for. + + Returns: + Optional[pd.DataFrame] + Returns an optional dataframe containing time series points with columns + - ts (timestamp), pid, tid (of corresponding GPU,stream), stream and queue_length. + + Note that each row or time point shows a changes in the value of the + time series. The value remains constant until the next time point. + In essence, you can think of it like a step function changes. + """ # get trace for a rank trace_df: pd.DataFrame = t.get_trace(rank) @@ -82,16 +104,19 @@ def get_queue_length_time_series( 1. Incremented when a CUDA runtime operation enqueues a kernel on a stream. 3. Decremented when a CUDA kernel/memcopy operation executes on a stream. - The dataframe returned contains time series points with columns - - ts (timestamp), pid, tid (of corresponding GPU,stream), stream, - and queue_length. - Note that each row or time point shows a changes in the value of the time series. - The value remains constant until the next time point. In essence, you can think - of it like a step function that keeps changing. - Args: t (Trace): Input trace data structure. - ranks (list of int): ranks to perform this analysis for. + rank (int): rank to perform this analysis for. + + Returns: + Dict[int, pd.DataFrame]: + A dictionary of rank -> time series with the queue length of each CUDA stream. + Each dataframe contains time series points with columns + - ts (timestamp), pid, tid (of corresponding GPU,stream), stream and queue_length. + + Note that each row or time point shows a changes in the value of the + time series. The value remains constant until the next time point. + In essence, you can think of it like a step function changes. """ if ranks is None or len(ranks) == 0: ranks = [0] @@ -112,14 +137,19 @@ def get_queue_length_summary( ranks: Optional[List[int]] = None, ) -> Optional[pd.DataFrame]: """ - Returns a dataframe with queue length statistics per CUDA stream and rank. - We summarize queue length per stream and rank using- - count, min, max, std-deviation, 25, 50th and 75th percentiles. - The summary uses the pandas describe() function. + Returns a dataframe (optional) with queue length statistics per CUDA stream + and rank. Args: t (Trace): Input trace data structure. ranks (list of int): ranks to perform this analysis for. + + Returns: + Optional[pd.DataFrame] + A dataframe (optional) summarizing queue length per stream and rank + using- + count, min, max, std-deviation, 25th, 50th and 75th percentiles. + This summary uses the pandas describe() function. """ if ranks is None or len(ranks) == 0: ranks = [0] @@ -134,6 +164,21 @@ def get_queue_length_summary( @classmethod def _get_memory_bw_time_series_for_rank(cls, t: "Trace", rank: int) -> pd.DataFrame: + """ + Returns time series for the memory bandwidth of memory copy and memory set operations + for specified rank. + + Args: + t (Trace): Input trace data structure. + rank (int): rank to generate the time series for. + + Returns: + pd.DataFrame + Returns time series for the memory bandwidth. + The dataframe returned contains time series points with columns + - ts (timestamp), pid (of corresponding GPU), name of memory copy type + and memory_bw_gbps - memory bandwidth in GB/sec + """ # get trace for a rank trace_df: pd.DataFrame = t.get_trace(rank) sym_table = t.symbol_table.get_sym_table() @@ -184,12 +229,16 @@ def get_memory_bw_time_series( """ Returns a dictionary of rank -> time series for the memory bandwidth. - The dataframe returned contains time series points with columns - - ts (timestamp), pid (of corresponding GPU), name of memory copy type - and memory_bw_gbps - memory bandwidth in GB/sec Args: t (Trace): Input trace data structure. ranks (list of int): ranks to perform this analysis for. + + Returns: + Dict[int, pd.DataFrame] + Returns a dictionary of rank -> time series for the memory bandwidth. + The dataframe returned contains time series points with columns + - ts (timestamp), pid (of corresponding GPU), name of memory copy type + and memory_bw_gbps - memory bandwidth in GB/sec """ if ranks is None or len(ranks) == 0: ranks = [0] @@ -208,14 +257,18 @@ def get_memory_bw_summary( ranks: Optional[List[int]] = None, ) -> pd.DataFrame: """ - Returns a dataframe with memory copy bandwidth statistic per rank and - memory/memset copy type. - We summarize memory bandwidth by - count, min, max, std-deviation, 25, 50th and 75th percentiles. - The summary uses the pandas describe() function. + Returns a dataframe with memory copy bandwidth statistic per rank and memory copy/memset type. + Args: t (Trace): Input trace data structure. ranks (list of int): ranks to perform this analysis for. + + Returns: + Optional[pd.DataFrame] + A dataframe (optional) summarizing memory bandwidth per stream and + memory/memset copy type using - + count, min, max, std-deviation, 25th, 50th and 75th percentiles. + This summary uses the pandas describe() function. """ if ranks is None or len(ranks) == 0: ranks = [0] diff --git a/tests/data/rank_non_gpu/rank_non_gpu.json.gz b/tests/data/rank_non_gpu/rank_non_gpu.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..f702daa4ab9013d42a6b562a88a5e26d3783bedd GIT binary patch literal 46121 zcmY(p1yodD)IN-KcgMicB_++!El5ZsEdtW*3?SVNA|W^kBHc)ebV>|JD%~wJ3?R)n z>ihe@@3)p~;kmQ#Irr|p&+|NcpUa$pi@TAx^c)St+t%I1+TFw5+R@Wbz}d&c9qsU| zrH6%PdNHK4Q>nE52NrqK#>y{0iFs(hxL$;tD@`qdC^n`+ydil%36bUtf1~Gj*LgOn zz66u9qpXV?Nj^G#;|?4gFB~5uzAudd0>=*J@0)e=?@qV2_bw6JS;w#n!F%ufgWmkR zn?3paQ}FBiW<8{9{%y}X@Taqmc!cgYYLzCaAZp2;uY zxLDsk`WE;J=lNHE9&C!ikn@q>gJ*(8b0LRUh=VKU#T%;u=6k=}g`3m!mD{<_`?EC! z(j))w46!zsyD+nHCEsz$68t+4E}O;8nTku`2`_y_xp_5?l$Enf3CSX9}2CJzdzb?v5lommdai&9_P9gJ{CNkCTHQ(q3R9yY8^;2b zgNISZ;ZGbJi;BK%+6>x8FpY5zx23Y{a?D?bbjYqOE+sNw%6}aM$=_xP&K-2_m%H~$|^Bqcvjqw^G8FJuU5vQpSX=d+TN;joFpa3`V;1Ue~0|c zm-5&?ke~IZu&bQQ_(Cm+m%R2E``8p(`vT&3f9rnhXmNXdA9NdZnx4dmW zZiY#0Fz+Du_x85$T<#h}={5qMRFq;(c2ITub>Zds7q${07;YWbJp}Fi{qgq(UEVN-hMJI) zws0~QM>X1G;)47vn%YtcJP3;*iBxOydcSHipY)vPWpYF_MqjckQviczi?0} z^o3Bh=rBC#^)IyiMOMs#}J&<&!pzAx0b%9{wepZPdi+_hJ(7Yjt%OVl=BKawWN-Dfz{lW#|&rkGk z)@%geX4+CgpJVCuZ4jE=s}z2F@w?~jJyecy+T3J0TF+EAe0GO9StBncYTHp$2&*ui zFe*_C)q^=R;DkpBrtZ%pxhS3jJw-LV2ESj1RpuzjT~lNpP<$A z?(k8g{`CEs!HlWi2=JYT)=ZS*vD5Yu;dXCJZ3!&mAUv&~v(7Hz@^b2~W(6yOS`23n zn{C9-?rs0!Z%nHXyChFzT)PzXnuku_gX;!K{kqP&hkCM2>jvAI`l^&$K`i?eFHOT3 zX{mGj;-74vFrEhx@5SP)6qA}!Q}7w*jQGr7Fc#-gu!2_8wm=BGEQkpgAq0}8MChNy zpA|>-Wi$sA6P8HYZIQ&U5@)n9x+QgZ!ie3bfZ?UdnK{~5{t`836UxM}fb@{7?vTrp z|B)Hf`+Vid0-m}IPT>rPy&9lp5w3pZC!Vs+%+~g3CTr!EZT><{!a~#_=JERoOO3_e z!hBz^QG36kbImBBYL3kB?9t!(Z>4-!H-+j%IgGHsSwMTOUA^#l$=&(jhiG zeIsy}Q<)4K1{UT&A>#j}N%+a^%12|nLJ_JgRx7DWL?bws#@vSnfXkE*Rs?%ZG45!x zH_Vdi7Sfxd6-6iHa~@FmzzDYW?QN6j#~1V{$jkQZtW>EhDqnnQk5_+9WdQw#6+mTX z{xLeCY+JI%?pv7PwvoM5D!o7xeW7ksAR8~OvrW#Jd5=#p72C`4sCn&~uJ3OfU9(IO z*)?`WQ>4I}Lc4+_dV%G4AKT;gx4LLfczL1NWeH!Rci$ky*-Z>V&H6h>HF5S19Gs|u zwLVSESemt}NDuib@5!{9B}jY_LiL0=OvoH5s$pp%u9C0aB3#1?bDM@NxE69eDaluXTuIm`V)hb_)68W~UOU zCk8X|opF19x@B~T;8pvKV}zVmGbtM*o)N4mR5)b3u=~9IGPR?{tzM3Y!hN)2$cMpl zOFXxs#ZMsXH!o*B8|6mRB*=R5>`YhM3ffN$v!kErPF<@@sy+_+3B2C2q%2K3S~iK^ zQQTf{9=_uC^oU*?PAc;V{mD3R^hRnd3&C757*m4bhv*fWzmc>P(1h00;=TCc<)GuQ z_O5a06>TJ#4qBRxuAdV{EqMGmf!h8#w+aEYD4SU4?PI4<8RnP{py0brX&yVdQW4OO z5XZQF2=7(M?Eh6r#FrBxS9?>;a}qFW_`q&3*-{AILltW!|65j?hFoHgUs^#QC!Db~S6$f8>8}+ee4()H2;6F2PyW6_R zjLHk&%`u5iJ^3Ag=in2FwFzkoa`$-L`ZYmr-}DK~xVqp8SmAVi%IBE|EM^4pbLefC zH{wUzL;Hi?9|T+8tq*)iPWHY$EIKwSSneLC(|Fqu*jLZ|;_`6W`%4z=(&_R+&}T5Z zjg}RDw{y=U(vk4~^~umGQx*Y}g2HE9KxYI?x3RgofOA>scq7=;mM=3p!91R@tT&Pl z^sUyfL78?@^$}IKe*4DD=q>DsyjDBnN?O<>#<^hc?Rbt_JJ5Gei-REupIR}l9p%Q4 z=#7Cel+7x6)&^P3h8iTV(JlZ@nRpQOFUe8Xr(a|*wC{H^7gI7rRls(90UHU_U(!8v z5W-hHk%prSVw4El7^T8kIAhe6Wt7l6Yp0r89qL7n_F85S>hyOt7w+W5)QFHM$}G@v zQ+LH9Mtgmp&^&ruIl18KmibM=+YvTJZ2f_0mQJjc_ElgW7tiE{W4l{gY=9ffR}ttS zLr18@G6(rN@p!S~K>8(hDyKYb3SaQ$w`{uz?FQ;q>5n^7k%FGFW=@my<~D5EUjm>) z7sSfI;E9c++x)=@0GlZV6nH_doUPOFy@n(u#6mv!<^I$9INGr{ld0~+u-;Ib!6}dc zg)bQwJg!xceY;`b;41iD6uYw`q>^~9e~u+mL~w$ph%b{iNL?5D(4jz;E#WMzGBMhzyCGrT7H5DY;09FElgt$; z$nwB^GfkNv7&A=ekJ9`{=i8V2dkbe9Q~UdKGiS)(gEIyOt%HL>W>|^zATO{N&*(;Q z-0pQ0Z2x0Xto)X_?(+dMN-)0bX2Z4Vw_nW;la}?wWo@g4dYe<%rjL&k_H~4iPO@&& zCNV;j0{JV0PlI)F@T`b)4J|xNGcPFRy$x7tf@TRN8?=Km^B)sOkBs=Iwym0qAQoBP z9un`I)aQ&!0KM$je<<=^0t5sXfa~E)pW)$j_Uk+<(o+MGtKDw# zagOwR+6)xvbR%(iU69PQ@pfS#7-u$;i}WRR5?;vNIo$V5ouY=gsKtS3J77Ib`P~yA zB^E5hu#@iL+qAu9)U%VA91tNN#-|r^ z4~={@6|K#=;nnwLe(?%zNYnY+JEBP)GQ3U7^idE-mr=1HsHdiu(oSNAqOKwF}5DO_~Y{0vH+C%I*8SjW+-bLv^d*w!tch>^BJKEpdX0GV1=y#rw za_zD(2g&GR`PWL(%S=OLSbaj}e$S<(ki=v$PTSP_C!G0u8GlB%pQlbDORf_c>-G`H zk2te(>Q}}#3MFKYlCsD%7@7`ymuzmUzT?#Y2G59vB9N)@4HnG0CO>2$h!v+?aZcQ> z5DTDDik|A(n@en#U9SK9Amhd}lk-~IPs*aG7N1Ac37}Sro=D16Z9p7yz^c|WF7W*w z7fzc3@yX=7Lh}3?lA0P`)z#22xbax;1It`qy^GEJwziM}@Xg4WLGQ)mN4aQ$4b1qo zHZB0hU3AI$m9_I=V*j&eyu`EOJ*KwZ}j$0@nj^T6LQ+D-YjrCydEj(%% zKx@Qv3Te02QqO_7C(;K(d7LK)%4Rr;mE?FpxD!d#QQ?4jr`9p` zPoD?G$DPKe&@Ab-guZen?Z{VmEX&<^zrGfCa=Nv4fB92S+J3PzBfOm{8n$E2OM`5& z9U9`-KzF~uToldxwX@V%kjRNAr9v*pEb+K;?Dt$?F?KbuwRFoW)SA*=b8tPFS|BCkPmaF zaTN`tb-SMcvGUh)S0-8u7E@Cqq8V{UO4ij3&DERSSF4-1XOr}wuG2?E&SmakG*$>hq~hA;|pJ~5v5pxrmGbLevlxHc*&F|DKk>GR*WF@C+1 z>Ar1xDPWn@6LRV8bMKdU*HPI~{j0{E8E|)gi)w}SGKVZyx^Y-ncgk6Xv|gQE-|k4r ze6w}N(&c2ZnO?t);e)4S-9>K3tZgNX-QV221b1Rs-#HSbw%>mMd$}Vrj<#dYI+N~R zopf50cee)`y$V8Hg=Vfyt>4}t9Odr5Woj8`T-A0yN)x`Kv^8V1A+f)tjg*oE;J2!A zi;{IMZxh7>K!lSBp`o|p>Xpxwl2?hny9T!FF+HbiJ+W zV|JA~H*Dx$vMYC2Hq4KeMb~akn(md;W4C<)h0UD;=LPR^SVHGqFrk!DFI)}W3>~H{&=2#7 zL9F;mQ&0Vi${Hs6oQP&4OH>Lb3%tFbn5)^jDC^)kVA-w$bxRwGsMvRf$rqH7nbE#e z#5p)NRt(U&$W~&0w^{VpSqLXi4eU%E{e*#6{lu$D@?j-H1zwN84X?aa8?cMX8&WRE zNmJ)nj5pPh0r}sU93X0%@#>$;t%yh&KU0mKGQ<594>EuV$G|rHfse9T64_1?+5BC| z!W>`Szqt7YxNp0fi-}p7$*Q=2btX?=gkTTq*JeoyLI?0sXoTERemwbdoGaTiizcJ+ z~;?5p(|TP{%J6I^dEH+kkLYhIUw_$HA-)%q&GE)Jr^vX-kbL`e9n~7i@=^;j1|_hF zq{pL0)^o_go}|YkDIFWn2U0v)5tLx0uByt`O@~~c9s&R{+`N|~dUB zmW-cwVO+_K@R$Ch+RG|B z=}Ks}AMR$|Lef}Q@vs9o5pA9Z?kP;-Ab85UL%yct26=`&>$H+jzK@zYlK(oO8bgq# z@v%+|qmbGGTij1v5L$1xD4FCIsCJ88joW{_GrY|r@6Jyz3LfgA@+Hsu0KT26U*%y-!c1y1>Xg&S6*Yz zx?_<^E(vnuz%F}?u^G!PE@e$%{Pg+lrH8!M0ZJ3@on@IqN#6eQv&C;*IJhDORQlZ#$0-o~0 zwVgH9ZBEtodwojYXnzDuIAP*8&Vj7iTZS65C-0nv=J<8_4W9>IRuA0gkePfGyQ167 zi8UV(iF9~TpGHOzePXI2_VWQ}p|(MO@{dC%bZQ<-U1`H%3ZV|8n~$RTy0X@Wq~Hp$ zXTjrKF0S)3`K6$*fh{jXDc&94L~5Ui$Cpr)+dGG(j7g`-PObbfH`x1iy=FX%2+_o1 z=t&Vptz8zArv^iB4VEL!W=PTkGgL5V^1sAOz0g9^DrF5^xM`{br`$8|gm-=zXj%Bx z6TYZN^(1uQZ6~ImC{>$H>c&jW6uDrEIz9M$y7|_D(fypR}I&m>Ritcal*gjtdYT@ToW}=sm zBUE@Y9RbBXpkn(F=i6zdSE%VNzpjvLl#TYDfk$;zpF+nHa4lzxC#uRI{=K1pr_jiH z`|Xp)R27L-C1+r!I!)8h1_VY|s_m)l+ulp8XD&dVbS3#n88fK*eENZ%R{Pm*!_66D zpM`pd#kDlTH4b+&m5ZiHL15YFz)u-{=4DMdyGSFAopb}psTONO0XF;Hg@vc^<4XYP zJUYw8hv5@m7M?WAq->3pm&N(C-}(P6vl;W#A8xpiMZ<5F$bYbHzo25~&+3?;Dn<)2kTN;Qv_xT0e_*m6j4iVVW>CxHxeRXD5i zHIQn1Xex=NfFVmFp;V(OM^R*V$S1jlWXJR_mx{1OQ}85}jhbaJr^75v8|`U!U@8}# z7s;0asjp_fsx0hx2_RT3`P9JH7;i-^qybz@_Pfp_uRC{Z=*B))1@&7}u6E|BgH*jB zMh`Yy3hz3gbUkmmG7`E&1#uwb5KkJ(RW%yrBBtm=+OYk4J}(&|!58LO$q9ArLx)LD zN${mJf$>Ecc@d?oM`66v{Pi+^vdkXR3wK^s)A5%?kAC!k>!V!WQC+6ERnHxjmvCGE-u*QN;o2zm8c zmAQmHs9}%3_eZ2{4cEzR({F03wwVgWKTX^6OBEd<2GwiO5Dawi+BJ79gl*CLx0nhw zaeRBlx;|Wwk?`oyv(Bb{WJ4nty)f?uPNrccM|-;&R@H>l1TlYoZEt^gT}`}{nVPB4 zVV5_Nj0HdK0=ezv2PV@z+n5Bb4NHJ1{bvj<-aoTYe)f*wG`C&mr#7qHLQN7=j`FH$ zZQbnUiGzNe$V6ySPy8003zeY9Y%a)to2e(d34^qEYX&yajxZODkRqHvIELy0J40HK7Ld6WHO;@U>B&`BK$- z_aDP1j-EgN|DT%*e2!RL^$?{>mLOWyD$$ppA~wz5IZ~JJcVF11E^Uonm9j<>IRjjG zdFQ#c*+3Bz30+(orUejafgm996-3)#80Z>dj4V<*7KR~jcVMf-C-z^M)c63XW}+bs zl>#qDBWaKh2pi>Nu@xjC;9-jzXkvnfV&Dttfm_c6sA9s%SYJ~4lBW39eQ9~IM!Ykk zH{*K|U-$E6e$*TT;cK&>(tiuS;yvx9dGopj`iysLE`*s?j?4-JQM5+lEx5al|SJ&*DWYS9( zwEjikgVkpKQ`tE#+L5%li)FTksK0h&?jY~@tic{h|Pp5FE9kc)&tMee_^?rHE zHbkoEx|BnhzA{J{CemYORQoS+@g=N zMMLoQb`fCcu^=9zAH5*?zNA7g=S_>3=dO`>_)3kM?kjP|D<&31#sanZkc| zg!aRZTEwEG#GNb)E^aL2I{F<;Bghp(Xj|bPZ?ZS*GrrsjvTJ1P_H`}w*TVyPz1#AN zdbk=dF&SW)tl;&tg2{J8B2v&4DmagB&BEofH$%+u=y^=!WF{9zlLB^dhCSrZ6|b{Y zcg7KAv}d7c8_yf4B-@RaCFE0T{M)rZzQfY4jYBhz?Ez`mF4^>yl`|!E=Oc?k@F|*+ z?A;vM$OQ@hoEk@Y> zy63HU)l72fq^uEikQ91wfM#aNI6kqWcN9zSy8Rne4VST?{3@XSLGlP?)qQRcLt+WS zjrd!BlI=U>B?Hi!IdBgzp3v|rvycw*WDECObNr&uEND(-D^-Fi8 zfsu@PM5oHnQI>K{C_C68jAwK|99hwf8G?7{LkxP0`)8qW`45sz;hhT&Xdd84X>O{+hFYC9@IRfN4`{J&RsvPU z=2@?UjcHO2nqOA>UnD(8`!UY|AtY<-ypn=F_F0m6;DtHnDj9T#9L*t>E&%Z z$E<{E+BCAA1Of4bs79*%FNpdSq9;Lgu_)K~R0#z8WLkR&B~pSgW3h0B#3~YSJaAl6 zB}0XE*KL&!u_<(nj+_YZC=pVqz{hdnl}QUyxcDI?I|G#A>KuLCLk4l5dsQd-E->P! z05?4KT^Qsy8~_{kuo}DWZf47B!U9Mf7j+pb@0(`&`Os5@KF#$}#2T`Y8Z@+Bw7SmH za$wKdQ0Q_$Z62d9!N2?IMq-s#{4#Nd17XVgC+U-Bu;!d5VEZD@P?-&(*`7Y16vHL+ zfPjh-i=Ue9k5{pz;!3#pVvXZ-%q-Pj*M11T+1@%ci!LbCr;~Ns9rnT)QJh_WsF=e! z6*Wgcvb)r^p4907M#A!Q@cyMFOQ90#(|{f#s^0z#xijO6ggF5kC6-6Zi(Phcjgn{; z1b>shP&pYYw(VN`+3w-`Fe+t&b7!4nfX(j)L7e>2rWG=H6H$KZ!42Z^Wx8!q@)>)98IxTw@z%p7mGYMT0$~Y@yB} zr7p)b3V3Q(Nw`tK(`*(UvtYGHNg|ZdWm31g>e0YOM0>*nH7)(S0cf<~0Q-Vy15lVA z7P_!lg;w^615sxVGkzsI4^*$zJy%TU?+%buj_hvN24(8u%mZ}n{P0@TB9e`19KD6? zfZUG}i^@jLnb5_3bi-D}JHOT8G94tPD%0N$kkDW{STY5|m>dx@{LH&iOT?ywWK^M% z93p)C@{#4lQE{s16C_v}*5x~SRkl-0`gD8_R5S4}01g!iCicHE*u|$nnG^p%)ag(v zZj&6F!XLZjNOnX$&7Tmc>-BLxDD!thQD}7y#tXmhzcDaW(xB2uwkREY>YuhThOK`# zkqJkCW+@7z`TWNpsJm<`64r)81+Mrq-Tm<5O)XjaX5gebdp&stG9(@7Sh7VE@7@V8%FYc317+~kE;YIS0#+I2`u3Lz^T%^|WWxij- zi`r{4+c505##vB4by|p6tBaKSi#l3;e^Ez8w<%5$-z;+nP~6sdx@k=F;ak$2DuA&F zbUWQ`Of!PSeed>!IKsNVf*50o`3wMQ{H@T8jmydrZ zOAesbI2Yamirn`(%%z;lHTEvHllz)bP3;yhb+gzo4Mtz#B7u%HoJt^_!8Z zhVTj^sQVx8L@I8fQ?gfjg6KT|A7x!oiHXv?-iHPX};8J3;QLXeoZffva~8ar%exm|C@MrhxFy`8p>7j^J20$+}R+`8@K zsb~`@cl=-UAvYY_q8w#shWU_K|3CCmd(qt*y2BGMkBWhVan=WoB8;+h2sG^CZwxdy zbt1WYeMqfWSsob{5Brjxq5(SgAnj{^qHr^9Bu7kIk+1`as-YDqJ}qlDw+t6A;=)&o z2c>Y_=&-2@euz1+bg65@JeuSy2I=yF{zYN!%FVojDLPTS%hVE-0tC&?J*aPE&nxIG z_-Q*d&8I;n3~%W{<-TtlW7?nGC#17@0O>`C(S-la{ussM2Sgw`+<&b5jBj4(DV&rA zmHX3`0(gH(cOX;Cizc@Zb{Kh9^(xTPX!Zku=*kj;Y;#zk77z*6ks zr|I*fl8k>38ETl$s>`kP5BX7Yr;=^yFg1(k#FQ2ASMG4E(Gy9VTUN5&`43IeKz@-` zJk!#3_j^#j7%&z9PhTA}$N;PG^)FTm%iN~>iE-B0 zgx4ne$DyhCAr$C1`E{Qyd^VN=?H#~L`M~SKDw|i*dd4(=J-?=zi>;;zFe&6gYfRf_`ltNaHEvSDxT*N0Hr}X7Xxhj)FS0qJ zZCd(zBd3Wt^U8lt(%xW+2vR)v(*$X1%s_JVDiY{DgU_N@1yL214(P20O7Of6moJq( z>b7Pf2o~;UCJ{>ii}-@3BC*u0c)I*ge^}RfR%q3>qG9Ijuj4=!ico4Zdq$P$kNd;f zWDHpF8zOs1mc(81crOlY0&%AuqFbYtyEqINLV9)H+wuSjZRq?VAOD4()FgF=R_R4g+- zqkZ%r^;6jG&z^X%X_fs?bOte!8q>(52$-51g(1=}hoSXuYC&dYpw1?ow-2g}vk(zn;{vg1J#bT{`eG$3 zmiTmF?{nPVGr!PE0er_@`Szc64}mJEt&shn^S7PV8Dp98o*oH-tc6hS_pV?{b5BK; zVbNR|#i-rTGQ&Y^Z^LRImL>th6#Az?dmsf2w&C=(y@6lXY{U2d)1Vn_kvJ66|47)b zN&tVy!~^#(qgMbM8U25FSoXreUKf-7q5;(PXT9(Uc~wH{)Y#sWzMjm#Opy3^(}6*| zb?F4NhZ?sJ1$@o|xnhMUHcGY-CEG7vQ%UgfJb5l7wM@t8>$B2Y~N`F}Ooz+W|FDq>CX2(`*$ zugU*P(8gUXDQF4z9jXjyHM6WlZ@vcqN5hlYl_(|%`i}|3is}`m2t z>ZT;Toz;-=Pm3bAN?Or@f&x=jf`f((ZqWpU2V7ylE5<(!@m(!Q;W-6j_~e zb{8=|l@gI5hAJArs2~DlE^_gWu3y*XKS&=T^4SuKg}9O&Y3m%Sz_J!vBxp=^af;|& zLk0<^=ccYM_3gM5X+Rm;sTZ5VD*QIF5B*QehewQjr3Xh`sa+*WpwgreJ+dzk=Ov8@ z=euuLhqoybH#kmwSploTl5rdTt5-zQ(pDgZDy_yi6Xb@hVnj>#7Z^p$;4ydVSk@Xd zKye?NH1GB{7|~}EHW{N1^rGvCcX-idRn)hUYT5-v38JyUwTCTyVu(bxv97>Br9cek zI#?6bph30rrxtS9J#Y)Z6tvs?h>DU5KcTrmgb)8b>t+uN6CV87QP_we@1jZ~7` zl6<+_EBM9W*+!Q9z1M4Gkl5Yz*~Q$<+(L|XwMUKdtInXSy%KCGi*c#9gCGFEgsF>o z79Td~88@NpI3U`ck85>pB80B<%V1~q-Pzij|Jm05JwxXP!{>sJ<-iTWo-WPJvF z*Vzt1Uu_2)E4$e$G=QygQwKy3iSu7nC}qBGUA8)7%|!^!G2};9C?W0=t0k(sxMFn= zjevFXumd6i;{3S^OUUgHYY!>DehZJFfEs+NE1$OIS_7rXmdn{Z^ya@V>Ol#!c9 zN^<1jA&oxET<*4QcxF#zl$A%$S4DX8wpZA%Fal05hj%p+pCv(36HmQ8Gv^pu>}w#| zA3h1$7R)_E%Vz=5>;hVB$Q_F>1C$eb-(WoCz;T3p-8wfNJsar7d>DmC@O8OXmRb?n zqVT&5>y6~2s=3^5|7VzEe@%D5U+FwwcEg1ff7b#>e+N98FMxEzTe?pFnvS7X`)mXD zwkg_N<*$+WYuJ5S3(PwVaLKJs*DDq`BH?0EdOrAt5xkjc-$AB#&r_u7RfR*xhiIme~+h7P_R4x zCFIl{*In}!Xm7R$R2}ro4Zro3_RFQX+9DD;o6e1g`6{_e8)?ei0QR)03f+Mo_Mg3G4OLp`SLGOQnN4^ZE3 zbXVEa-%&O?zZb?Up4 z()f3W6Wix&cPj^Vy zfwHV%g9bfksKSn2_s%XlgMCt&kL;d!tkCp#Mm##KSjg=Z)3Z?JzFw$JWIGsmL}FqC z_##xDK?G&tUUr(+cBkTKCKcTRGpvbi zC9aTA%TBC1Q1CYgckQcZdokrlK=x})u#C+FF~ zd0=vOCktNg4q0L+EnmF?{%dEj_qyOf{8D<@mnB)4!p&k5)fmltbcyyrs0J9SkvAzD zGaJK260CdT>HeEBps;#}CsI%^7TLBV+JcmYPjnL$^qMXDc)kYLlAfaxI6`1yV~$~t zM7AWewG)BF?O-9eDA2bM?~SC?*zY_nW_l=Jq-;x%*|Lw*FnF5uT!Dz->rU4rE_4zK zC9Z6txHnTlu!6I-R;Bp}zPv4294lX{|21Q?`oCsjXK^4C+(6H$hW4GVU;9HJqvwZv zg;oNSN0~x5tI}fd>L|3c`-WeB-I?g+VS`bv11#1UkEKq7zWp`r5!ygKpAFB65YRvT z-}5n=Gei0M{ypBR+{`~S&-%b>FO5ijGm4?EU4S z=;myE_cTUu_I?*&>48^l-wQI-V!^kM)LTcM8Gf5vv0u1U(>8cNcyL|fL841!_C;*9 zskzVbs4S!LO>bjJxm?ZXMS9=Tk&>;ljLiGexE6gt>0p-Nt5-x%=PfMava@1!2Mx+T z%gad~yo!<25bV61v$koxyii3@4j(JZ+B2=uDv-zwLeA zyXRvcFtl)S_>q?aM{ZBoCGwh})yX@;6XnF%1dI{Bbv-xL`9f+D2h@ocHtIwRrCWX^?hAjf zqeay+2B)Sp6(@6pd;#r;R_}r)d*3kP=%2>MV^Mo?KS+@H5wMh^;P*2DupGAWvY!px zNQrRM^WoRsH70+EsUe6-?~`~mvzVC`>b~15;y*WXJt&{9Pxbw@TXz{nK`G0qy{T5b zs!S{)BuE9LG?XazEH6}lPDrb9ao4m6A1a3{C>rH5jvib-n+rqEsV6@0I(Wfg*QKBh zAx$m^g9@B8@f-LY@~fqfgl$-k22(}etm{XY=kO=OWpg2fw4(lAL584qy%!DrO`IbM z4=}ur75#}+`loP>3dtyeU!Cn{g|GG!1zY+bCkd}Q$b#UXzJsdtl4Pk$QXa@;v!MU@ z=?eT{iHuZYwh5O^LSVI073Cs0R8EsBAn1h^5sW$>LEUAju8&%6of1+KRqmP!OKgl3 zH)52Lq$r(=9HMHi%}6x#eX$f}-@ow_Cr~rAF|UCPpo-mYRMWF=EM6Zus$_EbSLP=P zrV$@TPk4{L!TUO8JV~&>*7oCFq9EBYz`tATt%t6P?hXE@2E95w>vxF%? z-^P)Rs);%W?aPA=Dp|(y)fK(dSb)W;5U5s2OiOjJ5a;Rp^FsC0)xwxHvWT)A{_&z+ zGTD}0Td=gvH;05zG_rO>wt(W_qj2`FjBOpgjS=I?4MWsAk2EAiqBJD=*!4s~20}~O zJ!SELBw?Gvx7@gU!XQ1NnY}de^f*Dp1%)-^_`otL2B)voBplphxMSSszU^4_Cn)u; zf%wP)1b6SWHg!#m%O2{AO`bCtY>Qgi?MWsL24}7TJAvZs%o4I<6}BeM4p-4SL)CfZ<#0`gmr5!(h!1<9F>37XGCBb@D}dyhEu$%Y0d5^jqzLqy@T5+PC^-58DR!woP$$@CP=e(rkei2FgZuP)cIai(fpQ(Mn1N_OJRb7fEiO04A8tFPoH1c@E}Q> zEIIAG$f;bXeL=jM{0gQ_u;NBFSf4Ti`912K66kx{>GY(S z?D;PXK;*@K)6|=5>rh?^tA=-@4ZRi(AR~nOsmbWreIXHDGyY@@7i%z`K!Wj$q2(tU z`cp{5KAO@$CpA8HPFX-!PPsM3nEWt`j|c-U9uZEZ)z+w*79mu(TH@OJv-?u3y1nr_o=t}wqg5LLUyz?}7VRjf=tBbL42^%CWSUyCkNKP{*8A_5`##h1-o$R2^iX|*}2w2S&w(CFHU z2n*xo(7>h{ip*yoxAx31Lk!&wLw9!% z-QAtiAYFnE4bmlzlz=qSDKUgdw}?ncD_x@KH@@$^-~Bgx&e?0P9p{|2p0)O~Lp2E9 zPygdM!%HP;cLp&_{t|_j6QUYk9cjPk=Z|Wb%Rsr&-+R_poFet@ruYb*aaF&SDRIju zHjJuLYbVRS{riNKT0$j*cnwM$>O|P<_3b}SGq$~Uha&aF@uvyHmmvu99pOLZR(uMo zf0INc&d|SfNrMxY-&9xMy`QSuMOvJf+)SY5gebij-T2M7QvO<=E*ANui`yc{m#iKq zP0``MZ#1?z+)yVXR>NIN)gsiKpb#W{tV+G25NARz-sFEk_8)TUxzm@(ti-lO>v^nW zh?S#d*5A=gBQl}CEEj!@zQjUl2QRR?Sc_K5<12RW_*bdY7!2A!py>+fbJudLtMVo6MDywTXX$Lit*Q5C)GDYa*GDNx<2A{lJuFK<@ zQZ({9Mvz$uqXLDN_WJ}f8J#ELSmLiyPm537nim7rl@>#wBeb;=ix+|6#8$#uc^sfk z9WVdt%c*a;PteWNy7At#SsrUbWFV=Zu*;abSplsSH_QFFy)u@`Fu}`?!lM&ZgR6d5d zv?5GRp02ICySs0US5)`qD6t#+R^q3gAF^`czkV4<%d(EYNVk?$zu;Fewg;<6+;vlF z1~aRa)ScbwuD)+X;Mo%8yB2~ z3Di0!11k!qK2BsH6|s_MHJo>PD+6G*u>ObG-&-L$jR{xN!bUwc$ppcH2GS6H2sAVH z#OQ~@@uJNSD4o;>w_8a>4d~aJq^%}j2qcZx`<3vlR6t~KILQ;#UNV(<7nq{~Pk9EL zTZzXZXw^(kvRnR}arVvE7k{oFG(*x{koikgls+_qZShL91#FCAr_nrgrk!7@nHhk6 zV@1+*0f)zGNyYMCOUBU;NPmYvi@5u7e7Zj%&!%a^g*?ot44o{i%ijGVNb-u~AVWmC zCl^XAc%H6c$}n1o9z=1tsydbLyp6YklMabuohHV%hLN4UyXiS~-oxLRqJiuLPhVkK zZxLVn@W|M|Z2L@Cj)hMh02B?Ma|->r;QVdV1BS~0jbnrMz)8Nt-LIPp`k33A-upy( zq7qo#2_zz0Sw_&Hh5!X30>YF8o$w?vU(#Z9n8P1 zPPS0zyT1Ooa5=zYfU85mX&=IAUu(3ZDzx6iz4#(X3mNKaQK-KvFSUu;)!pY^D!N;}85rap0N$Ao zwGuBu9qtw-+H0UfakRSq8c4nmF!Jct0a3>wOO@-gmVb+TV>I%0*%RK6S2$94zC{gX zrSFiexB@_n5|cGf6fo*+-yh2NA_55~=(-q4OCT^f42iPG2Jc#T*!aTqif_gBGFCX% zXbwY?x1ki#bd7J~BPZ=UsaeMTu}!zg=!MN~NNTL;a8vUA_e0pXp?VIFTO_!y|K^l} zzy;}*0+b2*6YMAb&IjToP)Fe-9TLxlvJore!HCpx;>P2;HOcU&7~Zz;2F5=On@fEL z^sO+b#@9?-;h5f7&^R_fAEuly=+{~|YqpR+_c3M~^F=>lE4r&!t*p_GGs-)RsU~kc z9y9xG7x6~@&S99jU%JaD%C|VZ`yp)596~(^AtfZj3!}5@T8PRh@xLisZ*#=?AK3!eJi^k)aJwP%S0&o8(`i-v#nTsX0m=A zsjk|3()%?RA;aY7$ZpAUkz%3v4}~EV+Yl+08Mo$vg^WHkf{YOOHp?%iurYFR!vM`6 z`s3l-_6|dX(U`UXLc|cNE8QNyXoM^LqA^vZOY~wuW#TJ#5i(oS2<_;bocV9tQP*P- zI_sF?k1G7pH#dk@K@pdkI0AXKYr>F2!XSk-CK`#Nz4FD0Onp(GL=@7`^!*#HDs_EY zc2oP@6PgxnMW3fzL#j1V_#v&Y=zQo>;tXPbT|iGoCnu4{vUD61rh5?}NP zoh~BAk8AR$um>f65ULa$13DFtcji%~Daw<`h#aRX>w_^fRl5m$903pXJ(LDn5stqn z&@$${*vAr6Z5FK}o)`~6M3@n2(gz945)_XQ;n#>(Xvgd_eV~G3>V#sdSuBlaQJtL4 zRerzYIu+9_yF-pX;^5dr1UtC2J=6!p{N$x{a{|O}KD1HRsQ5Pd!#WLt!p_jdZlc*B zv5T;nlaEpkQ*P5+h_Su;Q;(M1ynZW9SmjhF5yXDS;*}gK9}#+EU|_#sNPkU`2c$ zgjkowAV&kff9w(4IwyQrF8MABuYz8eYe7d-<*fU`0{AFITmFoE0&bPyApKcQ>XG~- z>Z8SYv_FP99R1QL(1gbEp}}Ngxmr7gIW1u?!rrd^@h z?3#nDv$k#(63%*3XTJqhsRiNmSsG3cH>d|HArmT+xtBvS9`2uOSHgcRtLk=u7#qj+ z!DP05389;*+E4xE!5M^U^b#N0Z7QWSXRFv$pb0MiziZ4DE*L*?TNS#CHfo=}n@hk6 zT}ppK&G!}is!>T)HP-1zkLptV_-v6fIp4ar=X} z%@F}HYdw?K5@M8!&!Xg}CPJ6yHAk6gW_Xh*r_DDCqq)xoS+ zGDh_9!-7f9NmWr%II^k6e#9ed@T8N^GMV2vH+aulpO4qCX-UDOU^E zYGaH7^7femEO5%&o~W2w(fQ>1Pbw5xEHM;|WB68sJWDg(B^CT%(uf*2zPhp`gE6V* z1c>-?+x;i1m0Cs|aL&nkRHW>gzsO^rE!H3qh?-jCX!qd8)M2#=-u<{GP)S+uMKTuq zk}HRnQ*r4SMQIzt`>+_7rD@##y_DbS*sv0jqgIX*_F?SiCsn^>S2UMkcOE9Ttf%i| zu*BK`PYywoA+&+NYjY_la>wf!J&JTIfNEaO&5B?7f!lMC;R{wvJ_pR~>i&d2N^IPI zI*64bc^A5bwPq8;(=lnLmRK@7Z0kWXxtD_ScJpQmn_6j*85TM@S775D#LC{rD(3Uk z1#EpmwgS&`-{%Em$y^y#Qppv1*qHIW1gYl6>6Z4}_jgvG#+3+F;CulQPZE!0zaY0f zDT=EKlV8lrL&nnV!M}%>f^#tWCfw>)^Ma2eak2vPh~5$M;~LT*Q97O;Gs=nE96v!+ zrI0bq--a;3>8e(8F3YW|zhQIe`Cp1aV}s7AfQS5h&vU|&WF^m2-j(w7sUUc@fY#yJ zPkgb~YE--A7dnky(jLsX?gHw}@!2I9ys6V-l> z-!K6S)4FtqoR66NrV})iPMejX$I_~|gK&?Sj794l`M>D-8FuHhILJjZusj1r#5M$F z1$cfE>!Qo-J?q4!QhxWD5N)*q;b7)18V7HFjqh~SWs&6xY+ zn~6o+#2a;m6kAfRNiJn#M^JQh*tZ|>U@cYmbH6i>N$^Z<1AVvq_|bIsNps=~w71v{ zR|k&c8A%bY_asy~V_NTO6-;u)C;_DP+51rZITDGFzlf#5D9#toX9f<{x-hI`iQ^g$B`XQJ*@ep9+FV4w1NYmH@*}G=R=++{oZ67_2qJKiH zN3!Zwd-z+Eu>fZqz`<3NkuEn49xN{~_C<+ftQ`!vjhX=kr zp4VgylAUH88`5O7L9mARn}0Lzb6ReGtk}&_{?5Iwocysrt<;enKsf*wd>2xDs!^%% zBbIPiC}I%C#)rTk0MmD*ep=9E8mrJ`vVr4;e|T}BpSwq@n#?xvWG;nu3q0_xJtP$I zaxde~c9>l4yJwgRPVo#v-vUnid*hi(!LQqfs1rsw-oQFvCO&l`@XR;a=L?`;@~&ah zUWUDv9ENTC<38Osb8+P?_to8uyC6UA?q3|x+BS=j?1v4<`KdM;Qe?NHC9%7z1yp*h zm<%k;yyS)%{tz`Pp+ddjrrJ!VrV6CMYBz3>spf-0+-Iwb<Dv9h@o~VM|;Yjz7^RKxJTaf|R8~7*Nj2Uo~raZc!#Z&rA}^uCgJ)u-j## zSft+8E=ZJ+Le&<@0G(6%rs6VK#V20m)mFx<@B|4@MlJOgfIyh8Ega=zkzqmm3E82| ziW?#ZyZk{88}n zoSqmKy9TFTwm(l|Or9?Np#%wkb^!7xh5rJpYGvLxNh|_&i!rBUQDJ(FVLE{Y%r4d1 zJh7^neM`CE>!l>-EebbuL%sq{#*AU;Tp)8PsoTL$x=l0ws_=plQR4X#wmSiTa+|-x zn7$^XH)?HX=-48V4(?m?NDI3vW`!?Zy;AtH`|xMA5=qX4oXkjTnMsDZmL={>UhAeL zKND1iWE%v5t;qQ|d!Bu~vYAqDBAG6;H|Ql)^>2tL-u3mUUB_%A9H4ZB^( zzv>?pKYbETQo61&5;^aZ$Mt_o=0?|>KCuRw`?+f#O?8~Y_|zq3I{Z#txr-NJV@RF} z3NEjWR3-IrllYX%9{^TGhH}uIR4s=*LaaFe$!*oJ|+w`)hVPNqbwljZ?18 zBuw;4hE_nVddDq+s#f^h69<(``fuOylzn99ZJ##19_Pa&o5wjMTesZ2^RX-=qKF_A8k&|t^`WYLFcT!wwONwZLe9l4012N~hg8o2^-^&wR6 z8rB6ad5gkcH&G7^}x^erHN#4V(Y+6iVCL6^f)e8xrp9=H*voL&)8;)=qNH6p><#mx8CrtlkxHR zk!N`-xc+phZzJ7-GV{i1A6~`aMoxN`t80(NIFyJm0a$w6sei)sI$ix_x64F7O{S@T zOa^4<)oMF6DF(hfZ~j>;s}EWGQtoKo;am4@}D7--G@ zxOrErQ8E+9(TMOI%kY5sAb2^l+Kh_{L0Tp0bM3d>AoB5u_pp{2fX#$PhCOinE|DN$ z8H*w&Lm=U*>v#)=%J{C2bB2;P*Rho~y(Ak>3KUxBF!mt6o&RUx(*4 z*$x`E+l$$zX_727r)8~e1s&S$&DrOh7Av<(at%T|@J;fNpWOP7&;NN?WjO4zzFf8}@l+N^2zq3}}w<#1$Gi7L*^g3e-W6 zdeYbPC$=tzFGsRbak0U0baW_YE^DJbSP^)QX$|&crnKt}rzm+h|0B;@iJls0@AH51 zeEutM4$qiv5Zd^^y3wqs9)n|sK46VV*aU`5gXe~BT12MqYW?%W9D7)b09;*~ol0cv zOx{zL-yktV);%mi?yt;gt3W4^xuMt3I%ClINDXZeq`%)-?!^4-=lcN7?@QdRl5@sv z2)sWVy(~2AKMo;gL$*CEI_&8P{(H?%q|UTp9h!9@=y1zrlV@I1+)`C=kWOf18oMj1 zi4PR|`GfdeQ<}I0b!STHlS>um5kr1XsCW#YU&4UYWBNl|+RTcRn<%;}7?^slOyeuc z^{8Szx2;3~h}Q$-X?{I)379OUZky-Bge$T~#Z4-_M}=6Ik7Q+}woHPAb!J5s(OdJ# zxiDRo+2>y3M?OAtcTr7ht!un$Az~7mQLs6I1phWn`kuIbjdozM`%sjVGqfyTV zx-R=Qf70_z@xtds2SXQI_qJp;;ir$xHiVXJP71H+LL6FFycsX-ij?KBatMbpZU%`{ z+ZR;R{yW#Zf74PiLT}6Bh0fS6pwe?%_|kP|T*0XqrKS>e!lfbUY$P6|`vmwiRj-|X zzkXfz$Q-tXHTEF2J+&*J`x936T8XQ~tiHWaLs~rPIT0rlE1eN&xD4>ftQH}OjHa8$ z$U#KZ{FD8kpT;#;-`OrokkO^RT5$Fy?g#Z9&jT~ z$m~n@$ZU^VL;ovpeRrsI>!S3(^01(E_)xQ0$=Tln<^!_)z@s3fPcy1%DqYLh1P*`NnyuF-*+HXHBL8DcC#D>t;Hm zI;bdG8@VG79O6f5nTgp97>zc8b5A)vs4t)fxh6R;WDbfM5Bb-3w=@jm{GQlEj0Qx+ z8cJ1l-f8T=fE@dbm3#W^6G^lLBoyb;0}(j>n@WaQ^S8*^#yFR3c<1U8Fc^ zB;A5q4w8xx{`p@zw8g7!7tGa+Ig0+zk=e2O^SubCHO3ESwwaT2Tv>k(69}be3?uG8 zQNNk0Bp?~WW>08%5VmPMAss;@bx++^eRKH{@^Tx(JVMq_ah=U0{gOWLLpLd_cK|5U zzK+>ZGk0|KqJpM-0duEEXOP({Lp}LycHHzseO5!Kt&}`>n*VU;=w;BHYsU%LXoR$8 ztlr`0=actMbWNx9be1~yFha&d6Ov|JQzvS^>a-%UFy7Hlr@#IQS9V|1UZ0-T)yWPY zgj}X{(OZT9ReX&r#&x#w$s*nvblP;E`xOvx5wGID4(|)!7I3Rmm@g+KY9=vtF!Dwn-k{5M8AV>J7qh5*67&ol33`g9Q;7J8sgFu(HB07W zIDz?#Z6=|&+3dQLDX^d5EmE8Q76af@Okj=#@9nnL-(ObBrH98!{zbK-BT z0CDHbVzp+%n*EjIjqmRCD;I2cQV+hdmTq@r28p`T3a5USw z8vM2K)ce#jo3V*(EPo#oGHj|%{?xP_mI)vuq0XvFA;#~43Sz=z4+@m8Z&y{d%6x7= z$C0@2&MI0=MVOXd5NZH+ zRE=Nd%DKJcShs7oA<&D{E5{)jUvMUjV2sYE^ijDR`>viP0Hru22hTB1p?u%}U}gFS zspzNdbTnP4en3SiyR-1Zuk8a3G8w7v6wOe(lKH2Y7|A^0Ly$sc;rkop`V!; z1<=0?P=QI9mMm4DXPXga4LXkQ`{&Lb=m5AC`Zd4#o!-7$D;J7nFJ&n6{+u`3?QvvB z%krT}&(UgDe^AuHi$`5P@)pnJo5aKK&v(DxiySIj9hau=vfI(Lcz?uHr`M5S<=9^A z{9O2oiF{(h5l1H(1S2}3W&N0S{^#N1qHO55n?jJPeN+y6DSOK%mja6?rqMpJHv08O zDLVkxB7WA&{?CR~*|vvQKha01I*_J)53JE4qwBB@lu$qdcG;Gp63Z~w6h7g+Fnj=^&bAIe4qn|dne*whiE z2LCnV!j`uoLluSA>_ITz{^xkM~SVjcGczcPLEUu$jvWxQU**e?Pix{qrMo&fNAjzbQc zkNS!)j91r{-Q%_TX^~g$rvckf-ulI^Iv3xZZr$;vttq~2#TDrq=ukqsP8cR=t1jYG z4SX;m_3XK9>r7t{TiDd9EqD26J?sbq*CKCFFJ$^ zfMzMC^z3l zfY}v{^}*r6H0E5VDjgJ)T=43gI(_+~f=a9m01z~Ks{ z_ee@`6tW>LAL*Kn(5M0Bl(H(d2gw(IHVF_p+WJ-}2mGhr*OIxNZ!>8VC-f zWx+^X@G>{LzagP~WGTYEhSsRU5SEZ#Ekh7MwLxN+jR>w`JQFX3&$;<9A?88@#>)>-)NBMby5h~dHvHQ0X zr6Bo@Z>MqaIm-aIHO!j0O?iz!GrEgT zGup#*;~Eeac_@$oOBic}K%Q%mB#04pjcl6HNJ$Ik3Nad?dv6Zr(0ztl5@fVmQ}&R) zk&@un2`nW-0QA|bX~v<9S_U;7U{14;E7^Rj8yOazHB|>n(}PX!g|@Fe={Q=^cqIt^ zHBd!O;+*~a!m&V_jfnR$Rt;+POHh(l?5)^_)a8xM`F;=U^GN%iF^Nh`;zl(nBORJl zJs`*|DEh5MukZwZYHq-h`MvMk^gQTGVgRls)xyt4tLios#QbcXls3#PiBC`aUrzXnVUGnV zLSmWAgL&5>g}jrtA+-bu@dJGvO|{%{IT7+EG9$B3meD)5kf#kfv&29X|F&Q1Lu zSP#wJweprae;Kw5xgK7|g`)&mn>>fn#Jg z*mO_ip8|0}w_DiXw=1}l>L6+E-(VXKm>vZD@p^W6d}*^LvAff6uWz~7Y9PA>MCsa8_(H^!+5D;E!*HC*2J2GCs|Y>*^jZGb~CPS@890dclx{=apcWh|ze$65uwqnF;m*B=8hH1l5<_;7T~{POw4cF!D?uAgBOzDl$dY3X9qa299Gw zQhmuTq#_T`{;p;05o{&s4G|h9eVw)%`_UVz^$I9s2`V6j3h(yq1i!(W8KfCx-=Zu# zesf6mez!o;VmEjXvt#u9jvci-$-8mQD*d_F`w3fmfw1vU>(u4JD38H16_iYfO|fg; zW#fWdqxb98sR-@(e*tkT{xlVNgj3ChDvY zW&~9WRHOq$VqVZ!?KgN7y?XavM2&oV)`G|<)o71KCN#TPS0=Re=thJanV!rHqkOxr zaTz4jbMv$al^3HlIa6Mu6^oSiSBFa)LIh(M zH}{ElGEj%h8$?E-f;*vl?n2I#7Od(Zs|;wD!{e=I{K;8R^{VZ#BZpLB*yUKyOC;)U zIRh`~p;fby<+=C7Agj?i?-s3_*x4be_EdGA=Y$F+hNKLrBg3o%h_mUUud^BMG-x@8 z2b#%%7FE1Bm#quCeDIG(KMUG=JGL(tI5X7CV%?Cw^#xBFx%RtihCigUa+k1mYaCBH zsaY{!S&J{^wYK)B^1|+wujYL-{`B8Jk@Z=+H%1azOPw=2`m@pWmU{X^_Ey%uEU56N zYe{c6D+a>HJiJc!fFDs)@ArB{VMBk6CKTS>OB?dM#iWNO^jg}(e?39!7<{_(4oz41 zyB>@RW&g=~tBU!4iaa0>Vn0tpO+LRT8*w2HtEZ`StQl8_l;;cMz0uldJ*5%6mcGTx z!9M0JaEe+4zcXoDTKt)hHcfc{D`OC%td=*q0qs8MPzmF-c`>$Crv2z}0`8b#Kg|jJ zskh8QWk*XpM-ljVx4gUjFAF{CkTJ}x#27SKRy!g(adn4cRFapTtT^-;EBbX&@fZ=f ztQz>P5j}ts8}PCgl-+ZT!6j>6OdU;6-ueYDKXzcoQp#IvNj)_#j#)w>s_`xUcU)A7 z9#CKZ3k^f8TA<2$Y2Sf7g&Wz)>RZbZ<^RaI=7K8KwiE@3f8{U|5_A5%^v?Hhw zGc8ehhkMH$G6LcdEgoBJ%t&7CskM8okF$gWYDfF;1UEuoi;N-Ac>uUQ)UM+5A3O@i z2ynVV0#;Vo1kt=+f@N&He-Uz?&q~;e3_GWZsc3r2K2ibS%)Rkq>$7_2rX0q-OY)-YACoD^tPLb9uKFC?JXl7_-{WYT2~-aCZ3v{^f1PYwZ`RQPfKf z0G74xrxI?s*u<3VNAT4$0Br3zGi$Y?Bk(k6oMPIZVFqiRfXskF0HVQ#!0iA~zYFLe zRw#x4u(yY9aqO-@=c0K^;~zIW^5bJUN_zu}5L#=X{CoC~m&3zKEAQobt7a@@%7TFa4PC5s*bNo?0RuDN9M*jSz*1-bM$?l9c~;&g1&;+&QUaP0h6 z2^-X9&MV)42V+4MR3LB!4jF0DNv5B?DnGew?j>}=_0pR8O&};AK(TTH_CdCsLLV=^ z)LsN2uW=3-Wu#?LD8dq905{}*g!CL^zR~nKx>1dU9CglQ6FvJS34qWP!1%e^a~o^D z2D``U42&2d1*ppBjy`dHUfP-mf0~7G>;jF4gudYX_UFf{PI|>Q&L=O40(zo#$RWfP zNjf)C@7FfI{Nrv!S)cW_TuNwlA0qv}_O)X$N!P)rp-;PVUv0idpeS~`pJqU1@4jNg z;+tlcL*A1>wvnD1*1LbBb@8u3_1kr3j=#Q?qo_zlQCYfQ7@z1Q3$(Mme3Qo?UBl6?mK8G#v#lDVMn;EsZ~qJQ-X{Ok)J=1fIhhwkM+lWJFa3D=j>F48 z0e7-03LzN-3S4sz-`h9_e_FEo9c?3Zy|ICcWZR#5v1ro?YfBfGhN%(^ChPs+;E29u z3Y!DXXs^8@r&g{6EB^#p5JKC^<=pVatL@_Z*5>*s3<>Z+F@&|4j`_wajuR8|I3)=n zX~No&9TMX)e(YJ+93o;%CE#`fK|I8u?6z->fFid0p&VhSv}xCJ-nV8(Bx@Hyc>Z!i zl9#N2oH~jF{tQ`xFhst5>JVi-_9;YEj2e<25vUJ`J47O;IQhJ%suj7irDnKGsqbfMix^XSAxtT=0M3w;qpR+4(9JCs9z|D*;+Kl2X zzP#xW}}=({EHp|h0k4|Xym zMv5Jxm!#Ay$I*#xkt}&+;%G%MT6ZKHod?f}WU+kYXoV@N7*Fo87h?R%)aD|QJ2~IUy zC4%D^BPd)5@X-ZZ@LGu`IDln(5Cwsu5CT)UcM8Ub`-p_?RSr~Q6(mZaF68{z6og-c!WX{A-;9SfjVBlkN7p!q!4z$kk8aD`J5KSfpnx-hwwvDq#1% zX7TgO3?Gx^|Hh7~NUke4FbVaryJBkbKA2&%Cx5YI3M^9IJ@&o1{(RH9ynORx`o=Ii zkAYrwD!K}Cu3$P%^(^7%b1of&!Gn`uUoD)NSgEIysvse#(5dtF_X$mLCT*^lx=6zx z{3{|n0Qw&H>5wzN1>X}4)tC$M3T?DxT}~H=QdM1qXuw;0wL2g zbg^^C{XA|PLC%d(kOhYG8?VM1&i7koe*4H2_d^_8`{P@oLsl>ooU|&2)5iFOv;%F= z4gYhF@*T_!Sk2F(lYzgwFcS1+o(Ize!Q+W#`5QKQIUMxF|9VRxHiT;nzr!i76c>|P zf91uCz_MbP=XmHM>TuK_4+DEh2_hy3C9*keOFzqm*-z3cXaAd@)=}SLJXA7z4O%*o0ax>;?snN${TJqQ_2*fH~1EfF8{^{fy-!mwVg4_&V!G zzS0zpHa#Y1_%#lH>ox(fdPaOv>lBLfkLzDArZ!?UdeS2M) znQn$*)9jXUv_in96l<6-sCk>yi}@)aDFWK53)4ychtvz#*w4{XvLUD&!D2yYWZ{qZ z)z5}RAWZ1}Gs6ODBEYLID0erQ2}4b5Whgp|5UAxVk~=Pe&h!dr z5zzkvl)b~r9*eEwO@I2nM3BtQ{5@$h+l-!`9Xp)4F-nc#kmC8?m(AY|k(vbwB~{yy z38Z?)hfT87%OuezHqNrw?0hiB0l1R$FQt?h$j@;f-Yhp3)eR1Rv^QmoS z4FbCay%B1{guq!@$PsLE?aq27aDJ_IjvEGjWa2{4^g(v}!9$gmzi8MExJQsr+WhpBVpeqo*dlH01UQXYZzOo^2~i1HErUBD zXqdSZi@%W0hlBuP-^Mu*n4~(d>dx&NNR#^t5Jc$l-Jy=$nrZkvzeG|CFsx?q^aJbS z^+)Sy`PM9YEI7S7XTA892mhf!!IrcY)*!>%D8-o@Dja7V9yMk$;5C!>Tv~%|^lR#c zq%v1{GfjLE;538z)VI;#jRQBDIw!!H8R)F#^&`}ftHYut=&!U@y_uFctG2v4AV-Z@ zNVtP>H&t_l$X?aFPB5s9nf*MYmo1_C1(jf_3elvQXCJjdVI{iPraR}r%5(maG zyqGvP2pcQ^`A+lI7ndv`BAp9hpOR~>dGv%+;|Yg=l^%d*0#``%&G!>B=#}?t>h)^k z{(Zrk&<%p6ND+dQ)c1wXKPnl^koKGM(^;{lG3yv+Ex0knJi*Sq?l7FaZm2t*6qp4k zTl_||wckSKq(velO9;ok#2L(zqfKL#W|T-~%5$bLs%f63N{k_*D;?#|Oh@Nv%LT=9Y_*TFD8T4#oN6871dp35|N+*%ngWVk5?a%=lS|sbyz`_ z%?Prt5>3Xc4bahih9qnF^`kE@mLG->(Dl(AmLe0lfW;A z)Fdi*zA|Bcl(pV|dUxnf>gge&Q)Nay#lhVQP;Q}b{%|d>H}G~ZL@(Z&%>zi`z{RiR zDe)&%YDy)Oam?xV4nsUPr}5MSA(ZWVOGnTfSa zJkjEW2V>OnCUu)hHPQK2CFn5HL|^NBK7h8C;6bz9U7Kw7dGe)`gMig*VoOOxU>Ecx z{REqOi2qD9oCC8zWaG^*B!Uop6GT3iYbtGc2q3b{(4uDDtj5)P{>v1H(KX=g zeDE}LS#;aF&7CMIlXIa3=oJjx zu5p5|6^9%y^Sk%J({o?ciWPs~LBDQ#o(I4^GCTA6FtqvULkcleyhIXE#AGBsF0nsk z#dyYn1BB6)DM;(^;qUwD*QiKR0rD`^HLNogeV-7zv0Le^( z?@Ld@L93n?0}irGgK}T%)P`uh(AjG!M(}0GzI^#*@fBbylQR#ZN$(|HNKV>Y#{*Pj znZDhdl8aI$sN54tdR4ru15i`fhP`?3%r{IHs8Cdp4KB?VPBM7#Ku@sL%4aCH0F*L! ziyXw*3XR)@WB|=FJG!@TFMi&kvw-^N7|@cLbpvL6_w)i z+Z26Gtz?b*O1tyyeGcO9qzB8>Q#txZ3xmW)szJp|CBszqWp`3{htv;bHc7--&Ipfh zi6%v7cMN8oospIo7Y|oI9s`r2MibD8L}mc}6T~TEpQRneaD~_n1cVBV>T-Z38Ti() z)?4D`t{HC9O0mpupc++KyNIbf5obBV)MfX(`^&Gv0>A%U{Mq)w^#=x+tKw}@Q|7}A z=6&OgeR<4w!O>P)lRMeM&Ip}vW0$EpTKuj>$C<|Z_R12XzotI+i5q6=bpF=h30$NM z`aLi^Go^l8peJ=#*yx&Dh1}DhCQe%2uy}toMe$7^y+u`ImgV-LcvE?mBK3gLc=Qu5 z&9Zc9g1fg#0zv9i$YcBF3uU^O3zUS*(z$C%&!Bgsvj3G8LS`qGp*u z=n)zGedqS!)Tgz`n7!wPcp@QgGY-#^Ssxm;w{x-T%IF$39`+KTZH_pHq&*nxCDsY~ z`m3IMj5jKtnqkiYV`W^)yE5lbSrXrNd{bR1`0}1!WRU9`++?=rb$kE&SrSP3A+c4K z#T_7bi1k}9ai+8;xpDG`|6(`$s*wLoERZ6)#102zId4%T(=veF#XLO#Fc+CIKlGrh zn)P!oQs$7G3V}s|A|x9O|NaNHyCa(5lUKn!fOV5*UA85b{)&-r@Ie4Wy2HG4+}rKr z$0|=E#t6SbE9y^HO@Diy_CC|n?;o?*MsHqMXv zh~9hez5ec$@9)32v+uo`eKYf!=Vr|cd_1Ev{icm?$*U`!x%1(Do4(; zd}`w4#{qHPBmIqa z9Q}>;P)5rgObSNKedPC(M_AwI`_aYdz;Op-(NBN!_$&5Fy%PC@n5v@yFf=eR%(bwL zg0zkjgkg9Ha1cHdtNd2>d?sSlnv;&c7O% zMqs)pB5-;L`3Dt5yy(-Ide%{UfEc>c*-~Q4WD^m2KM{;T zR5+T?#(VwI!+9nzF$b@4TG=0XCPSPj`6-Z(b}~D0HG9B&q(o0Fx1}X2mmXm*Qfr%EsnnK@}FI7x83)9TY&n+3r=x+vRoP>x5m! z7iKNVZs0Ij*nl=#{H1A)gEM+O;8l8N!hJDWxXJ;U_?d45PkTVFRJOevSb-fezk7@W_hpQP@l_NhKHN_zLuP5(h0Iv{@dLkwVZe5P43mAq^2i)*J; z4=v|FgHzWvr()JG7F%d$l$Ju40r;Kl3WjQ`nhy>159xtuVsWppjB`rthwr*+?xVp; z(DF4lZ!ttRn0*By(5Geu(cjM8naTn~ee&0pPxiyai^HK0bc7UGsfmzk6@;Nr zj@6gztdB^Pn|V7R0{ovuo!&>D^`O{^lgVV*TK_fYKu3r)+Jl8@5;5~0EK#&0`2{n7 z!miFMxygu<%>Z&o4L#dA;pt5BfKiGfxO5Hj`IkYh(c!@QRJBpIiHfr?0CgA%3>Kl1G612}81; zw}ony7!*e53Nz5&plCMxjsvZ=d&Et!@kvbe-IUS6()pBS^4^f%IDM`%;p+E5ocy*Q z#$R@-iFJNflvRslF8h5+(|KA5iA0`S#t#w@{%8HGV^}M>!gzJ_AxJ>(7{-$tQv4oSOElI~cD{)C=~%>0;duefvDWfdU_T@grJdLvT9FSmses8l@z* zD#uaovqk&4mS{XltdB1hYPhR{RRi^yV}X`%`HHoY%-bU0x`00UUdt2SfgrkWa{xBG zRDPQ2q515MlWsDoH{$-vUdBu)tT&i$*c|RS+iH1{JCqV@gmplLpmQzUuCXc}V9)0V z$H*mF6EDWC@tbJon*;8%t;Ie_J*^j+k9+X}2G_!s6r^JC*g`DmIO4uLNx7M?StGGR zuMikpKFQh?We6LEMcuz)254iKb;q)&D+)M=jDuYee|qfbX`gAbE*(NKRnx6-`_8p zsJ^K5UNJd7pYbYHEg!@;W%|Cu{|2$Wq8%pZu|)3jcNru=QhCM`au1;a_<%>p)wb8RKlSOSDjE%T&vUhZm#>faYw&*Yj3*poSN&n z+<13_h9S?e5w`N-6`VIx0G>AxkS7s!!WG6z@%@hQ8^rQT;6Lv-Gs_2enBJHr3&C$M zVz|`qZ#+p|90vL%H8X25H|m=g{2Y7=Qh6exIJ*Jq?$2=_N(8?`cXIaAt<-)#c!cxm zUvMi>iag^)=By)f;%CUCoWYgHNd%GjlJ}H8D7F8vgLxku$vY4J_4EZ~Ons)hvB2F} zYF~-|){gE@WmfUX39;W+Ik=s?Ui0fcs5oTvaMf;ippPD4J;%j{FfTOAaEIU)D+ z-YZ}l63m0Oz+yV(5?TbVLTalv_wM@Q!)cW;gerx!i=@U5zH*u#^ zNP?XR2@+gz;@R%;K%Wk>!||+h$c#SUwMM=#&ESMbQCPKJb)hd-xAlkuffijdRkfGI zJzG`Qinc#0FubG2{u<^~>CT7aqlL&A8E*Hjg2-@phLwfo)x^#l*k07>7xcJAvsYv{ zq4|w5Jp)z_s1-lGEqx~w`B)caz~!r7ihT2P=sqFzptxr&DC4b2iP3{hN;w+Dm`=l3 za;r_ukbg7CZ=%2H_U;HV8DF?c0yxG;2To;uA+Q^mOCU)&it}5D&%$J=qk_;7J6Ba3 z8akZcCP9a|KWgk@Trrm{6*Aj$HyJ@BAtPb#V_G+=X`QJSJm)B|#e1tX5>@ts z-#iV^m6Qi)g;zlIQ}c1RGz|O>^W7_Jj;sVnJ)AH8F{=1z3;nPqLSRU6%NRZN`{!pr zAyYo$`L-uO>~jM+ymp|1syO3?T7YKXo#lp)w4#z!9#uX@JZ3RvGb|<|RHr$rs~E;I z29PXi;V__blicNw!Z`bZn~eU$Qg%%9NC!;$0NaEisA{e0gFgxKYAK=!p;NwvLob#O zGY2ImEo!L+eGnR_`5tg-J@XF5`y%^aMLpR~gCE)zyj_4dE@=VAA(njiL?2>}ql`CI$r-YqYp3whqta+hN_JI5o3qmjL_JvFs5=i-?m z`KHMC09S(|)|x=>ZMh36Ok$~5*@ROad271IjfYTWC^aBJOpzrCS^jr1u(xjI%h#A@ zKv;|e_a|~B3L_F4mVK=qJUkgEpB`@461@e%ixek9gw0Gj7L~ z`EBSYpf8;5-dyXPylG7O*2y_J`P|wPk52u;wqe8*q|LW_jeQz?gK)9fM`&Jx*3LDx zCFFVrj~E`=a*B^ljMJYJ*F35@?%8W+{Tu+t3vH_~oSH5@*eN{WZi=XBsCV<#+()IDb=Qhh_e4YgxpMY+8#KGJe%n z!eQS7(@JhbtL%KNx1*We6Ngd_m?phU8qab?3u_MDhN3Ykz0~ltXFs*cAx@fd@Ed8o z>T@+e!WBb;2pUHm9Bgq{WR9mWv#n;e2BLHMY#a(c!C@-*5oJx<=}t8n>i&D($Hhbj z!PB@2P6%|DUUgPtPKi^g4tV_1I*q!KW)urzD80-f@A-~w)j<48^>0>OIe82`9BBEE zX$Pz3XJb$SQd@T~{+GWB=7aG>2wRb(qACG3WWht;QtX=I?m4fUdl=8Hp{Oy&o?>F*SX z&?nRX8mlda9tQ}%?8h*b>#qHA{|{VEC1btM*S^u1N_m?DtSqLPa12%YwKWvaySeQ9 zh`D?k5{~b(r@OVh#Mj~W+>6gAE&l=7OiZ)g?d)|zkRHRQhxhJ9&=VnY7XtN?HJ0%S zP8dfm1{G|H3Pqe9CxGX7}MSbH}e+9}CeSk8`2# zzeIhW5AE$qchg<-^X^S1cv`v@nu<=S)dBmEZK*qzX;^e764mBd6o-YO^-`*K3uLc0 zI4A3-B*mlEu>;Sx6rDs*&6UB_B&5Dq?6;G~m_^O{s`ibRG2z^1Yby(HmTOe<%f{Kf zsVKo(54s@1FT0fO6?;|m4BQ{QMYs&)?7^lQJ6>X0$I=I6jYq;1U_5!IcK7w{asKQx zcIY|HcX`gH%8t&T#ziNu&IIf37SH|N85hZF65oodV{SZU>cAo9oBZSG2|e*^S}v(n zlD*9=D8w0s$zGWh)92;ZUwGCk?Drm^>E1|K@bZ(du_j^0!2&>IrK;V7&!QHVAjnr02PuAbe?HPsT>>Y!R>X19q+?$8cmF6Zn;D zQEP$iR)^cYX4kg_hIP+uR^um#yml;CU=xMPChwn6kwy?0#&x)MdeZbUZcMdeLhI&5 z7pmtX`|KriKuUQ+8)dd-3F->CPQZaxG55jR@$&Ti@z&C`9k#Di8@mq`I@QD)1KG5M zHYws>(EtVgImSeL*Nle*;y7+C)LTcEocCNeirxhACvN0nbY^Fb!ik3Pr6$EPshpnwAL55g-&tn!X#!c|W8+LzD0~ z0jbzn!DjmxZbuITyqVAWxOIBt2}<9r6vHa=vslZSuB*&x!$F#!$rK~_!?fN}Oc-e1x zoCE5A79LKD{1B-sFn-<>{0$teJ>`O+5446JdK_w%>nYW)M{S$_k0$w-s?n@B6cx_} z-cmk8I-@guu4i(z^AUYtIqFNzYpha@W>^kKM|b@lT0QSc>1aZ(oLpn=e{RH{Gxr&| zr5Mi|(3JaLT1NjH6|+b4T&VBTE})5K#xgV=(|xO%X_P)b*Woya%#dpCvTC4@=46%X zd=SmOKVU8XfUmkBmMI(yKQD`NJQ+i{o_f-otVzH5EvxPKTVG%Ot}ke!&u^-ZCP_Lzt_%}9Z65#SvC4$;gj#DF#ITm( z@oRIYI%iL&UQuDA@&K-||NHUM%tZ`mS>@t`WO}Z_>S%KD^9);b$MI0*cQe^yli8j& zasy@L@7?5xT!1fYY>!UVAyJ%9i8z+{fZ}0I>%C2hL=4Mq85*ty4s3YXz8b7VC7l20 z9lxEtERbNA$+<18Hav?r+_29LC&TPda*zkoEfH=BfOrXX)p`1zAgJbF+Cy^0IQz-BAQ z$smvNIZtFkc(5iBKoLXcB!N1x1k@C&1?bcp%kDR zIxCxh=9L~R4}?LOA{zMnpOhD3zT+l|Rq}(9ME3vbq(1)Y@=l%<^j7rcsEb{`nQRN4 zh}OSdrn#*aN@X5#P_fv{lTwu~rAM3UbKHIRo*`5K0(o+3E{;hmrql4D5hxh|s2Y^L ze4uV^I2S&H4*05rFgfolCbQRtrI*e?9s_)LzJi+teK%9TPACA3@-HQ!9*_G;of?g0 z0e_i$aBvzm7Nj=Qit0TBhdaSzG)k$T-0#Q}qGgD^e#M=(@)~Zc*(@LaU?2KZkiQzS zb8LnvTzHl{_=h|6bq@@X$2@h)JzPWn>Dio|J@hHyY3_sismT!h(ivU5=8WFxXkBj6 zC@>4pq&2K@C`m>|+kW*qFZhu_*IKu5Vw%CjXwvXpMnu(oD~`4Kmo-Hh{R9UiP_bZF zrvzE!(x8~vu1`ZrW7$_5La^A9eWQNpsP!9TmY;OT8EF?p4x?Au;1Oyjd`g92O}b$Q0Q8?AbkE|3|6E z!Av6Hv81JW@@stF8oZEVFdh1sCJp3Kn6%Rh z+l~n9J{w!#LgJqR>+hYcV2+2GLg&xGZ(nuIC6UOOsq)L=+3MVdjlSMX!JI60tPfq; zW9#>riv2e02R7n29Cn;}{wi$=sWZSihF3=8FRfaZk5u&uXf3gA3T<$rBm8{uN_c zyi`07Vsx%k&fJyMn$u=a^?EuFlTV=4a!j>e^G&tnphzu37#}=U5MS=XXW~-@yZfO& zyiL&Xof})FC`7HjaaKAHlWQH5O08N}fBdH`H#lnM9DlN&ATm!smHJtlU7qZVpS3T* zo73sFLADa^$iMsBHzV{h5w zsb(FKd=Fy>p2Z*Xc~GuPpRc+?Ak0-r6`g2M!*}+_z~Q<;){E>nshLKgdaLlSudUQm zx0>vKKKQth2`bg8K+Lal$Vn;NogDjnb5DlZIARY8e@nU86GJ~{2ve!xtT|uU^sYJ% z%W50>Qe#8%jdGfvSpg66q$ap+yoI1XcQGHn@;gZLO_Xg6a{nzFuPM~$;r;Fs_|fmk zqa5rz_}gxB+;XFVW=R#uE zJ`O}N@sy&*DtahR2ta>@gPu@jreBanP7B9Nc#l_Z7Dw=%4j@ZVSDoPJU8QUQb=N9< zJjspyUSkd{LP-}F=wPanXLfE8NlKK26sE2)K{$u2GBM* z5;9p({K~MTcqiAXwZIr@Hp@ZgYt|x!RPSxK7u!prLeO}k%5yVq(<~@}@kJ4oPhCRN zeLSiKev}$Jl*(@K0`L05UZ~~T=9`(WikJoFrtE}JK$@0u^D$^Lq#8!7P}IKPcO^;{ zWtNu3Nb$gZ4?Cq;I)0g1VM2+K+L>D1J%OoI2&u=Yn*HbmW#PW>Q5x(bZ-W6WoJ3#w zI3WS%&}*^B)dU?Pzldx~a3>A(wUxs-C5Vs|6#&K2@+>LdOApc=q$dd=S{4UFrZ>@L zR{VA15nOr5$9`w_gYxuAFa9KB0kVO-_6ADWIM;Ih{PB(0b|b#v6&O$fus&Y4!znu% zAS6Xag_+Zjfe#FkUquBae?^KEA!UqufCzd)G+GuXRGo`<$o3%%znP61dHb3HSPF^0 z*m0)Y*n`A82q{(6ja67Qh%ge>@V~3&S@_WKK4kb}jd-VTMAMU^8`0$dqWO)ONhS6c z8II-aMqIf?HbTsb2~5l4jA}(G9_WcWfYpj>zo|VwDLF#Q9#!@1@ViWaFj537w!fWC zrc-T$ru3>;gT?T;9>RTRQGGDD7xDrbLX@MXT-#vWNyai5@e5uatdCcs0bS`n;G415gR?C7axfsTu|D?Ch6vq*D5M8O+khH;O-U1!H^oVM&U3)E;R!Dm zlz8N_{RZt`Tc?#~s0WZKmVg0R1t_uLKuzvSC6lUxz5(g(GSKD?^plLaFk&g*j6r_U zD_McJBNzFdYb-mI9hKAQBIpW2MXRMf;7dD8S=8vbBooY^Rrs(RaWlOwwY_ zin6Xmq&q!~Yj^l!{aBEI4P?x-2iv3rX^^#Fh7lGHK*PI{(TFwa;t04Q%mI(gHGHh?3z$T zEx8p>uen6N6F@G99Q;T-jv6_tHaYyO~FG*u25m$csX;yL}s}GT^xf~@6$5fYuP~hFi zUrM{=0tr7ea&O=o5sI@F+^pAV;fW==j{yxIMRoic3S#TD(%tL@&QI1aKJ3Jy+%@G6 zsN4|8o>NWr8e&xT=j=C51}pAF#Vs((qSWUHH)4~nWK?hMiehnJqD3XCzKBTudc_J~ zp^3UydPC7#Ri@AdBTf}YO{r!SV<<1_#ncri|K9~YKXaA&4sAnbrxbbzkNdiF@6`no zjLJnua0i-qxCKN61 zm{CY*t)Wm>cousj7MMSSz#hcRtQ@t?^7CR6Ty3}Urz(=iG zm+sml%}4HCBPli&l@cIHkQQscU@VG|(nTTHdi;hfsXMY)q)5|x-G#Mv4W1s?ROH-7 zWjw)`Pl_4}(L=v6u6b27K=QT^6<8#^6pD=d(G`&Mq!cy3@W<4UltL~NmQC}>bcGR> zuJ9t4{Ts2lng)!J^0>tQ@PB+U@A`a0CXC}T?5h;!auvSh3wz zdPJniya74R_=*DQe5J4Rlfk`Xl1av|sCqkcu|8fS1^FBMO|qlqbk2UmpZ&fY-Dc#1 zbWN1huKyAv3{=mdV213uYHp;}WzP)>b=|<_8#20Am@#u-zA8jAQt;j-Hqbi~zD1iTLiPONdQC_sclG1YR7^?ELTS$a z?56eow!hvbnNqpQUAU9T|KlL_XDz%CS*H>bRSn8hH2M*SBS~-nL3A+=-FtMDDB1id z@9ebHy|lElqQcwl3{0MUHeld=XnBr*ynTKG z9EEA1P2$8J>=%!7Mz4|;p;dZDSBgPvZaF_avf{PPf4M723Ej}L->~xXjsM3u@aZ-G zMw~>3NOE(K}7QFLdm#xEF`4$VA<-Mj8w}Y&cJ{mwesSd$;X{={ROhwX}}M zJ7j(L$BU9mYBUr+9`I-NxS4GIizz|mZ6SoPq%magVo2{aA*(9TH{_q9LF-koE+`ap?oXIpUie!Pb~FH_Z7o8RP5D|ma?x49bWZX^c5YazQ1&-UA_vOY#oPHEyJcs9` zz@?&vq^b{K3{(1U<+eSB?Q7JXK~q45Et^w~m%!}K3wNPVd;LdA>&U$;?>DZ)ZIzt< zydsF*5p9B_G}Vn4S9{CbJk|0fQOcA6^5k3M@M#`5Qg81<>fXllXU37tBye@U< zvTaxF=7TCR9Z@}cr?3M-?=`>pjrAJp+16F-YOolcch&>? zVrwNfnsvCyA@%Bk07(Ty$$^~rTSVzdpxuQ#JAUPD?|4Ub&pAPc3>-{` zTQ)hQlEejCh_w$EQbj|-V8BO)`cD^&G~O_IMw|V%03ib7E%j{=iPv>LKb3q&BuVV~ z#L4ZS|Jnrjq4HnKpszxmAaVaFrKtV!UEuhoITVY6bB6nt`56qZh>Z%2)AJX}$LW!q zRS%iPXBy1t^AB^)wluX#t99#*_`W8pDCSzqWK z2Rd2qvR4|`57~M<_1+i6m4BYyoVNCI%lak2yi&q_Z_W2u;NWNQ5{@#240y64$ir)M z=oOtR&uLo`sw5k$3&<{EiLF z!;KHZKx7?1oz_ZxjWe(Bv*kL+r;CZ0KYrR@>V*8}*tvc9j~LcE1qn>Vvy;I1q{yYy zK+c1o1^&F~2uJ@Wr=3>t-iA)w>IS*7Ge{co(La!kv-7jI^O$KC!GVbqg4=^6f9k7o z{Xg7?7k(GR@zG&)P0k<4$mI}v{!hzYdIM;40!=?2PgkLt)FqLj$sv;c10@XV!%oot zHJ+$+j{6gX4(NWapByGvw(6lD>k-DRKj$5b0<9cHlQYOjAl0?Jz52-;-G^k=R01tQ zY7Er3{S!ZJ1TcrPh)KbdiTEWhrXlmny*=;pv&J@6j(4DUjExWEWD(z*we?Dd%|pyp zz1ei6MzfWBJeoldaPk)l`WZqqiz1jKhLl)9n4#S!OU$^PkCIWbgmRjpDH7TtBz_O*3ZAsR@g_pYccw_E~> zi!)y+`*7!(AD+U?I?Z^$Je|~_A($Y_uW>ENF|_sCdh|J+-tZ)F+12AMF-Cr~YaRVY zY{Pqsyppy9l^Pycw{E?)0e451a_kFq5p0Z@IbGg1GQ1I%9{9c1mrsXvN3CtRW%+n5 zFEEWjDoIvJx`Pe=c9EN^Ee2Ppjv6 zO$OJI<&|lzN_}>HH{GZ1Q|sz?zk2Yk31ss{*9=*Pu=pE%mC!q@qT+TxXQE8&$@1jR z6>wGNmp-Z`CI#chPSdf7z2Dhw_|fqu2#*Cvv{y&=RYs_>{;qMne_LEZjoU+Vn=ohF5FHZF-Q^S2Me0&my z*eJ-I$;TLp0TCsgt(JX*g6N}u-}FW&*3e7}Ne0;mv_q3WtlrfZ4E~WVr{s`Cp1&ol zj7C_35m4}|AysIJ3onHTCcc>ZLj0M8GjPg9wnAJ&%ve0;TL4i`81}$3BxKg~PiPQx zas3yr`|*c-Zg{Q~+>0{S>la0_M5w zYvoj4T%@I%#@pg`lc=RKXDPoBEN;#)#<;!22f}X<=p2pt@pk56-@8mw*Z61Zgeg=` z#o}fWC4V<1bIt2WA&2u@6If*%{vKYLE%bXFd@INUYXJu4 zs_Qp=e^{ouU&on(=d9A4f+tC3ozq~9(VVhm7cet}VC{DRB=Y&aHk15~vRs4qiz7eD zGBto42m!oeOp2d*V(ivOR(QRS`jjJ8r3{6R(OPQvogyE9EtvAZr4B%A`zP!02;+L^ zM$UIqbMBqmdpg@yR8)Celo14VnvgEp0ZjQ(0fyTCd*jvvVYhuXoS&6wm&Js<_5l1Q zEgf&g|64NtNcK5s8e4g~4$N2ktoJyUk>>1rE=zNKhE1+4b{h=W)xjNn6U)O0C z_D5Y*l0}!(*(EP@^h1YlFwh2#XLGjr3GX=lBSsXCDpFF$!k zG!bLM^$uB`dcM00+zkhL+p+Eqh90}>lPJU1jUB>X$Fd4c^q=my`1?pF9(lDf4!0&J|wDruShGTC*BTo zLhGY8>ljQ+~p3J8;OPz9l2`&X0ei WKJQ}3_F`k=T!PJ{k!5^^2>2iV*B5&L literal 0 HcmV?d00001 diff --git a/tests/test_trace_analysis.py b/tests/test_trace_analysis.py index d13edcd..57b3b72 100644 --- a/tests/test_trace_analysis.py +++ b/tests/test_trace_analysis.py @@ -18,6 +18,7 @@ class TraceAnalysisTestCase(unittest.TestCase): vision_transformer_t: TraceAnalysis inference_t: TraceAnalysis df_index_resolver_t: TraceAnalysis + rank_non_gpu_t: TraceAnalysis @classmethod def setUpClass(cls): @@ -25,9 +26,11 @@ def setUpClass(cls): vision_transformer_trace_dir: str = "tests/data/vision_transformer" inference_trace_dir: str = "tests/data/inference_single_rank" df_index_resolver_trace_dir: str = "tests/data/df_index_resolver" + rank_non_gpu_trace_dir: str = "tests/data/rank_non_gpu/" cls.vision_transformer_t = TraceAnalysis(trace_dir=vision_transformer_trace_dir) cls.inference_t = TraceAnalysis(trace_dir=inference_trace_dir) cls.df_index_resolver_t = TraceAnalysis(trace_dir=df_index_resolver_trace_dir) + cls.rank_non_gpu_t = TraceAnalysis(trace_dir=rank_non_gpu_trace_dir) def setUp(self): self.overlaid_trace_dir = "tests/data" @@ -225,6 +228,16 @@ def test_generate_trace_with_counters(self, mock_write_trace): # 2 ranks x 6 streams self.assertEqual(len(queue_len_summary_df), 12) + # Test traces without GPU kernels, these should return empty dicts or dataframes + queue_len_ts_dict = self.rank_non_gpu_t.get_queue_length_time_series() + self.assertEqual(len(queue_len_ts_dict), 0) + + queue_len_summary_df = self.rank_non_gpu_t.get_queue_length_summary(ranks=[0]) + self.assertIsNone(queue_len_summary_df) + + #mem_bw_summary_df = self.rank_non_gpu_t.get_memory_bw_summary(ranks=[0]) + #self.assertIsNone(queue_len_summary_df) + def test_get_idle_time_breakdown(self): (idle_time_df, idle_interval_df,) = self.vision_transformer_t.get_idle_time_breakdown( ranks=[0, 1], visualize=False, show_idle_interval_stats=True From 61de64321afb769e0239ecd0db6876374f2fda20 Mon Sep 17 00:00:00 2001 From: Brian Coutinho Date: Fri, 3 Feb 2023 16:32:12 -0800 Subject: [PATCH 4/8] handle memory bandwidth series without gpu kernels too --- hta/analyzers/trace_counters.py | 18 ++++++++++++------ tests/test_trace_analysis.py | 4 ++-- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/hta/analyzers/trace_counters.py b/hta/analyzers/trace_counters.py index 52cd1ac..34b6bf3 100644 --- a/hta/analyzers/trace_counters.py +++ b/hta/analyzers/trace_counters.py @@ -16,7 +16,8 @@ def __init__(self): pass @classmethod - def _get_queue_length_time_series_for_rank(cls, t: "Trace", rank: int) -> Optional[pd.DataFrame]: + def _get_queue_length_time_series_for_rank( + cls, t: "Trace", rank: int) -> Optional[pd.DataFrame]: """ Returns a dataframe (optional) with time series for the queue length on a CUDA streams within requested rank. @@ -163,7 +164,7 @@ def get_queue_length_summary( return pd.concat(results_list) if len(results_list) > 0 else None @classmethod - def _get_memory_bw_time_series_for_rank(cls, t: "Trace", rank: int) -> pd.DataFrame: + def _get_memory_bw_time_series_for_rank(cls, t: "Trace", rank: int) -> Optional[pd.DataFrame]: """ Returns time series for the memory bandwidth of memory copy and memory set operations for specified rank. @@ -173,8 +174,8 @@ def _get_memory_bw_time_series_for_rank(cls, t: "Trace", rank: int) -> pd.DataFr rank (int): rank to generate the time series for. Returns: - pd.DataFrame - Returns time series for the memory bandwidth. + Optional[pd.DataFrame] + Returns dataframe (optional) with time series for the memory bandwidth. The dataframe returned contains time series points with columns - ts (timestamp), pid (of corresponding GPU), name of memory copy type and memory_bw_gbps - memory bandwidth in GB/sec @@ -216,6 +217,10 @@ def _get_memory_bw_time_series_for_rank(cls, t: "Trace", rank: int) -> pd.DataFr for _, membw_df in membw_time_series.groupby("name"): membw_df.memory_bw_gbps = membw_df.memory_bw_gbps.cumsum() result_df_list.append(membw_df) + + if len(result_df_list) == 0: + return None + result_df = pd.concat(result_df_list)[["ts", "pid", "name", "memory_bw_gbps"]] result_df["tid"] = 0 return result_df @@ -248,7 +253,8 @@ def get_memory_bw_time_series( "when the value changes. Once a values is observed the time series " "stays constant until the next update." ) - return {rank: TraceCounters._get_memory_bw_time_series_for_rank(t, rank) for rank in ranks} + result = {rank: TraceCounters._get_memory_bw_time_series_for_rank(t, rank) for rank in ranks} + return dict(filter(lambda x: x[1] is not None, result.items())) @classmethod def get_memory_bw_summary( @@ -282,4 +288,4 @@ def get_memory_bw_summary( result = rank_df[["rank", "name", "memory_bw_gbps"]].groupby(["rank", "name"]).describe() results_list.append(result) - return pd.concat(results_list) + return pd.concat(results_list) if len(results_list) > 0 else None diff --git a/tests/test_trace_analysis.py b/tests/test_trace_analysis.py index 57b3b72..191fb95 100644 --- a/tests/test_trace_analysis.py +++ b/tests/test_trace_analysis.py @@ -235,8 +235,8 @@ def test_generate_trace_with_counters(self, mock_write_trace): queue_len_summary_df = self.rank_non_gpu_t.get_queue_length_summary(ranks=[0]) self.assertIsNone(queue_len_summary_df) - #mem_bw_summary_df = self.rank_non_gpu_t.get_memory_bw_summary(ranks=[0]) - #self.assertIsNone(queue_len_summary_df) + mem_bw_summary_df = self.rank_non_gpu_t.get_memory_bw_summary(ranks=[0]) + self.assertIsNone(mem_bw_summary_df) def test_get_idle_time_breakdown(self): (idle_time_df, idle_interval_df,) = self.vision_transformer_t.get_idle_time_breakdown( From 31f4ea2f71c93d17f8c7fb60713e773900348092 Mon Sep 17 00:00:00 2001 From: Brian Coutinho Date: Fri, 3 Feb 2023 16:38:11 -0800 Subject: [PATCH 5/8] pre-commit hook fixes --- hta/analyzers/trace_counters.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hta/analyzers/trace_counters.py b/hta/analyzers/trace_counters.py index 34b6bf3..1164f85 100644 --- a/hta/analyzers/trace_counters.py +++ b/hta/analyzers/trace_counters.py @@ -16,8 +16,7 @@ def __init__(self): pass @classmethod - def _get_queue_length_time_series_for_rank( - cls, t: "Trace", rank: int) -> Optional[pd.DataFrame]: + def _get_queue_length_time_series_for_rank(cls, t: "Trace", rank: int) -> Optional[pd.DataFrame]: """ Returns a dataframe (optional) with time series for the queue length on a CUDA streams within requested rank. From 31aacab6bdef88fe8fc33e23a70a48ebb9960e90 Mon Sep 17 00:00:00 2001 From: Anupam Bhatnagar Date: Wed, 8 Feb 2023 11:04:23 -0800 Subject: [PATCH 6/8] update docstrings in trace_counters.py --- hta/analyzers/trace_counters.py | 63 ++++++++++++++++----------------- hta/trace_analysis.py | 8 +++-- 2 files changed, 35 insertions(+), 36 deletions(-) diff --git a/hta/analyzers/trace_counters.py b/hta/analyzers/trace_counters.py index 1164f85..9553db2 100644 --- a/hta/analyzers/trace_counters.py +++ b/hta/analyzers/trace_counters.py @@ -18,13 +18,13 @@ def __init__(self): @classmethod def _get_queue_length_time_series_for_rank(cls, t: "Trace", rank: int) -> Optional[pd.DataFrame]: """ - Returns a dataframe (optional) with time series for the queue length + Returns an (optional) dataframe with time series for the queue length on a CUDA streams within requested rank. Queue length is defined as the number of outstanding CUDA operations on a stream The value of the queue length is: 1. Incremented when a CUDA runtime operation enqueues a kernel on a stream. - 3. Decremented when a CUDA kernel/memcopy operation executes on a stream. + 2. Decremented when a CUDA kernel/memcopy operation executes on a stream. Args: t (Trace): Input trace data structure. @@ -32,12 +32,13 @@ def _get_queue_length_time_series_for_rank(cls, t: "Trace", rank: int) -> Option Returns: Optional[pd.DataFrame] - Returns an optional dataframe containing time series points with columns - - ts (timestamp), pid, tid (of corresponding GPU,stream), stream and queue_length. + Returns an (optional) dataframe containing time series with the following + columns: ts (timestamp), pid, tid (of corresponding GPU, stream), stream and + queue_length. - Note that each row or time point shows a changes in the value of the - time series. The value remains constant until the next time point. - In essence, you can think of it like a step function changes. + Note that each row or timestamp denotes a change in the value of the + time series. The value remains constant until the next timestamp. + In essence, it can be thought of as a step function. """ # get trace for a rank trace_df: pd.DataFrame = t.get_trace(rank) @@ -102,7 +103,7 @@ def get_queue_length_time_series( Queue length is defined as the number of outstanding CUDA operations on a stream The value of the queue length is: 1. Incremented when a CUDA runtime operation enqueues a kernel on a stream. - 3. Decremented when a CUDA kernel/memcopy operation executes on a stream. + 2. Decremented when a CUDA kernel/memcopy operation executes on a stream. Args: t (Trace): Input trace data structure. @@ -111,12 +112,12 @@ def get_queue_length_time_series( Returns: Dict[int, pd.DataFrame]: A dictionary of rank -> time series with the queue length of each CUDA stream. - Each dataframe contains time series points with columns - - ts (timestamp), pid, tid (of corresponding GPU,stream), stream and queue_length. + Each dataframe contains a time series consisting of the following columns: + ts (timestamp), pid, tid (of corresponding GPU, stream), stream and queue_length. - Note that each row or time point shows a changes in the value of the - time series. The value remains constant until the next time point. - In essence, you can think of it like a step function changes. + Note that each row or timestamp shows a change in the value of the + time series. The value remains constant until the next timestamp. + In essence, it can be thought of as a step function. """ if ranks is None or len(ranks) == 0: ranks = [0] @@ -137,19 +138,16 @@ def get_queue_length_summary( ranks: Optional[List[int]] = None, ) -> Optional[pd.DataFrame]: """ - Returns a dataframe (optional) with queue length statistics per CUDA stream - and rank. + Returns an (optional) dataframe with queue length statistics per CUDA stream and rank. Args: t (Trace): Input trace data structure. - ranks (list of int): ranks to perform this analysis for. + ranks (list of int): ranks to perform this analysis. Returns: Optional[pd.DataFrame] - A dataframe (optional) summarizing queue length per stream and rank - using- - count, min, max, std-deviation, 25th, 50th and 75th percentiles. - This summary uses the pandas describe() function. + An (optional) dataframe containing the summary statistics of queue length per + stream and rank. """ if ranks is None or len(ranks) == 0: ranks = [0] @@ -174,10 +172,10 @@ def _get_memory_bw_time_series_for_rank(cls, t: "Trace", rank: int) -> Optional[ Returns: Optional[pd.DataFrame] - Returns dataframe (optional) with time series for the memory bandwidth. - The dataframe returned contains time series points with columns - - ts (timestamp), pid (of corresponding GPU), name of memory copy type - and memory_bw_gbps - memory bandwidth in GB/sec + Returns an (optional) dataframe with time series for the memory bandwidth. + The dataframe returned contains time series with columns: + ts (timestamp), pid (of corresponding GPU), name of memory copy type + and memory_bw_gbps (memory bandwidth in GB/sec). """ # get trace for a rank trace_df: pd.DataFrame = t.get_trace(rank) @@ -240,9 +238,9 @@ def get_memory_bw_time_series( Returns: Dict[int, pd.DataFrame] Returns a dictionary of rank -> time series for the memory bandwidth. - The dataframe returned contains time series points with columns - - ts (timestamp), pid (of corresponding GPU), name of memory copy type - and memory_bw_gbps - memory bandwidth in GB/sec + The dataframe returned contains time series along with the following columns: + ts (timestamp), pid (of corresponding GPU), name of memory copy type + and memory_bw_gbps (memory bandwidth in GB/sec). """ if ranks is None or len(ranks) == 0: ranks = [0] @@ -260,9 +258,10 @@ def get_memory_bw_summary( cls, t: "Trace", ranks: Optional[List[int]] = None, - ) -> pd.DataFrame: + ) -> Optional[pd.DataFrame]: """ - Returns a dataframe with memory copy bandwidth statistic per rank and memory copy/memset type. + Returns an (optional) dataframe containing the summary statistics of memory ops. The + tracked memory ops are MemcpyDtoH, MemcpyHtoD, MemcpyDtoD and MemSet. Args: t (Trace): Input trace data structure. @@ -270,10 +269,8 @@ def get_memory_bw_summary( Returns: Optional[pd.DataFrame] - A dataframe (optional) summarizing memory bandwidth per stream and - memory/memset copy type using - - count, min, max, std-deviation, 25th, 50th and 75th percentiles. - This summary uses the pandas describe() function. + An (optional) dataframe containing the summary statistics of the following memory ops: + MemcpyDtoH, MemcpyHtoD, MemcpyDtoD, MemSet. """ if ranks is None or len(ranks) == 0: ranks = [0] diff --git a/hta/trace_analysis.py b/hta/trace_analysis.py index f4509ea..df52f04 100644 --- a/hta/trace_analysis.py +++ b/hta/trace_analysis.py @@ -324,7 +324,7 @@ def add_time_series(series_dict: Dict[int, pd.DataFrame], counter_name: str, cou def get_queue_length_summary( self, ranks: Optional[List[int]] = None, - ) -> pd.DataFrame: + ) -> Optional[pd.DataFrame]: r""" Queue length is defined as the number of outstanding CUDA operations on a stream. This functions calculates the summary statistics for the queue length on each CUDA stream for @@ -334,9 +334,10 @@ def get_queue_length_summary( ranks (List[int]): List of ranks for which to queue length summary is calculated. Default = [0]. Returns: - pd.DataFrame + pd.DataFrame or None A dataframe summarizing the queue length statistics. The dataframe contains count, min, max, standard deviation, 25th, 50th and 75th percentiles. + The function returns None when the dataframe is empty. """ return TraceCounters.get_queue_length_summary(self.t, ranks) @@ -373,9 +374,10 @@ def get_memory_bw_summary( ranks (List[int]): List of ranks for which memory bandwidth is calculated. Default = [0]. Returns: - pd.DataFrame + pd.DataFrame or None A dataframe containing the summary statistics. The dataframe includes count, min, max, standard deviation, 25th, 50th and 75th percentiles of memory copy/memset operations. + The function returns None when the dataframe is empty. """ return TraceCounters.get_memory_bw_summary(self.t, ranks) From 23db9000efbe607554682ec14e53204b0c08572a Mon Sep 17 00:00:00 2001 From: Anupam Bhatnagar Date: Wed, 8 Feb 2023 11:15:19 -0800 Subject: [PATCH 7/8] remove tid from memory bandwidth time series --- hta/analyzers/trace_counters.py | 1 - 1 file changed, 1 deletion(-) diff --git a/hta/analyzers/trace_counters.py b/hta/analyzers/trace_counters.py index 9553db2..edd98ad 100644 --- a/hta/analyzers/trace_counters.py +++ b/hta/analyzers/trace_counters.py @@ -219,7 +219,6 @@ def _get_memory_bw_time_series_for_rank(cls, t: "Trace", rank: int) -> Optional[ return None result_df = pd.concat(result_df_list)[["ts", "pid", "name", "memory_bw_gbps"]] - result_df["tid"] = 0 return result_df @classmethod From d3a3c7861233528471048b2dd0e687253cdfb426 Mon Sep 17 00:00:00 2001 From: Anupam Bhatnagar Date: Wed, 8 Feb 2023 18:29:56 -0800 Subject: [PATCH 8/8] fix broken test --- hta/common/trace.py | 4 ++-- tests/test_trace_analysis.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/hta/common/trace.py b/hta/common/trace.py index 0505183..c00d47f 100644 --- a/hta/common/trace.py +++ b/hta/common/trace.py @@ -637,10 +637,10 @@ def convert_time_series_to_events( Returns a list of json events that can be appended to the trace. """ - required_columns = ["tid", "pid", "ts", counter_col] + required_columns = ["pid", "ts", counter_col] if not set(required_columns).issubset(series.columns): logger.warning( - "Time seried dataframe does NOT contain required columns " + "Time series dataframe does NOT contain required columns " f"{required_columns}, columns contained = {series.columns}" ) return [] diff --git a/tests/test_trace_analysis.py b/tests/test_trace_analysis.py index 191fb95..78b9364 100644 --- a/tests/test_trace_analysis.py +++ b/tests/test_trace_analysis.py @@ -206,7 +206,7 @@ def test_generate_trace_with_counters(self, mock_write_trace): counter_events = [ev for ev in trace_json["traceEvents"] if ev["ph"] == PHASE_COUNTER] print(f"Trace has {len(counter_events)} counter events") - self.assertGreaterEqual(len(counter_events), 23000) + self.assertGreaterEqual(len(counter_events), 21000) counter_names = {ev["name"] for ev in counter_events} self.assertEqual(