From 265b813dbe048e5a55b78e7d84729b571ded6294 Mon Sep 17 00:00:00 2001 From: Benjamin Jee Date: Tue, 4 Feb 2025 14:05:19 -0800 Subject: [PATCH 01/12] Add leader election to allow data plane pods to only connect to the leader --- .../control-data-plane-split/graph-conns.png | Bin 22038 -> 23782 bytes internal/framework/events/event.go | 4 ++ internal/framework/runnables/runnables.go | 21 ++++++----- .../framework/runnables/runnables_test.go | 16 +++++--- internal/mode/static/handler.go | 21 ++++++++++- internal/mode/static/handler_test.go | 35 +++++++++++++++++- internal/mode/static/health.go | 23 ++++++++++-- internal/mode/static/health_test.go | 25 ++++++++++++- internal/mode/static/manager.go | 10 ++++- 9 files changed, 132 insertions(+), 23 deletions(-) diff --git a/docs/proposals/control-data-plane-split/graph-conns.png b/docs/proposals/control-data-plane-split/graph-conns.png index bb41cd488e53fa54f625eef29454e2f44d8c5a5b..b383363917f44f6cd478d8d2fdb5175008fa693c 100644 GIT binary patch literal 23782 zcmeFZXH-<(vMAaFL8Ji%$p{$8k~0VhiqH*B4nh+pHlbIZ0|V*b0(! zY*K^dq(sTD`TX`i``mZly=T1pe%v?4W(>w!tLB_Ft7=x&tXZp8ztd1tBD=wG0{{S! zfs~(W0RRLD0D#c-Ispzsj8Eu`yRcd)Ye4`2FLnSR;57hnf&&Gt0sw9T0KmE#03ekB z0MH^*>NOwX8jdwobrdfzFMs@4iit~lB=?N-p|H6ntg@ zzqPgfIwVvVYEe*JuB@h?oSv-&(MwFp2#t&h3Hx|(a2OUDtNGm2+TPjd1uXnyoR!^6 znWqqaV@prpU^^#wcdtNuxQmmshn=G{!r7yvv!}AUZsEsLZb6Ap;9K9Ix0N+@1A`+i zZC^_(stwI-O3SNNG!4Ie?T}Vbv2k?W+uKi0&k7C+|JL3a_$Jgl0BL6BsHS5qD*c3u zSEQuuGsvr76ek3{7b;3m0dKF}QN}%xAe0SU006q=tA9L@TW|~xNbCxNC=##XU!x+Y zL}T#v0RY1a&{KIG&#A2@lJp-EH8UMu?+c4d?~x$z#7BmNhtKPqY)@eApC1~SJecf{ z`=PjdbScn&js9a>&%{mAmw0KHWzsw}M4o`zCjqPgb`sni|6l*j2YaaRh(A33`73qj zGv55H&;C}C_M>CIJAm!(Ve7rKwL40FUS6{fvL}2jfbGtP-sL^oxqA`2uPWt>D8GK~ z=&f`j;rP89_KfZAdy2;UC&SB4P9%WokG;zrbChc4hK)XqjQ|Ps*}XMV7heFXP3BYG z20<&yHeRnW3qTD)VY__-Km>U|b{QuQpqeH~YLEq_%rg&6AK=Xs9tA9yQUEwm#3CNP z0EvGLT?tUDH2R*|2zP>+$QZH!QeKj+8Wud`9Nm^KjitSi&;~zx&H^ZY30yUN_vLJu z3#xq({L5G}AFjMQx5Y8cvL`miUd+%D0M~K4td6sb(RwA6(89eYU6NE> z9Q(_n)Na>O%QnZ=O4V+bU5l*UPk`yUI2_No@Ng+yBeN{5jfkz&(s72{Kx;Hl*QPiz zKD^K?<3wvXGO7K72uV=X&ikpPR`toC*=)k4H}qQ~MW<~64G6s$q{(G`PvWe zRwh2nDsPI@j{clnGoUs4ZWYIvZBin&uR)Q+$sHi^Sh&UGJn)iYdUbBDN~{|3-aSaC z!z*nLt(6gJ7(=91QQJf@uYHN(aLq^|y*~vy&~1@nfMAAH1H6BzReC48!USHrN>pDy z%r$CFX1tM@n^%!IsivVQF6$p$G_W}VIs;iHy2=f`(6>h9>F|VT`UWTd+{znOdp|7?vBJhwtRSNU~-^x zjJWm1{~PW3pDLai^MGf&_*>FtymPn8;{_-I;AruDIoXLM&vVM>kQXmNdbjG8wb2`& zS2#N$3Wgp4OHZ6XyUFlu!^f(3}Ow+`|j@KJp0Vwe~ zxpU7Q%yZ&b7s$g00FDZZI{?5#LLhN#P%8rP@cRGXZ|?0AWUQ+1KuH0E(S>&koOl6W z`}?6S=bKOx0MEQS6n)-VO9{|>Tp?-KWQ)lmXX{ z-<%zWI#C0PUz>I!T2A!vTEUzHnS^HYe)72HrQg)NU_e}!Ug05b4MsJ%bx+0stzN%F z=6rYnW{M-Wn)qhN7hA8W2zLB^PG&p|@DUMZ&Ed}KX+H7nfV|_H$wzYM`+WGa=bkN- zJ_p8WfRu$(>pg6!)2*Y0?vUQ)RHs`wC$zXM$1cHc7Mo-tZPJu`ut;IS;#s24^|x;dsK!t@2Cct z9HJeG1qyStAtAzw;@n#eiC;koF7$;s^NI4?-vY-U;X6UMzd+K&G#0U!m;9YhoD8 zotFe+%M=~!>g{(huC_r1iN!Aaj_-pe6BxS2v>ffp#uL(qopM0pFk6n?XTycHZYDX} zCyFpXb4$7vGFz>nP-Qmh7%3|W=WxNq0TSIu(O6_BD0f+Kj)MK8R5i^&mqf)wokG>h zGUXa%Q7=tXK~4J+#A0^Y!ca$8D`>G_S~seJ)Itmd#ejEv`FrwgfO$UgYg^ORbMDnY z3`K(&^T;@;>?I%{`Aiv0^%uV-!G489Yr0016>VzDa=bvTn@`B#QP-1V ztY&Hu(~c9}R7j&267|^(N%}}(`gITSFV%HI(OnW*S-qB7pW_L0s|vHT{XX{i%z3iSOk?EznasCw8`pB6h;qTvw- zUfW=*nl5H&6rKKw^LEFfhD5ee_s-LR!^4G{2)+XQ;cV5l16J=7cHRfouYOpfvU`wG z4jNOvTOHcOtzy*~!YwSDp@pBepBaF`X!eAW;|T!g=F&84hcEjg|Ija5DBH-fE5z@M zf=z8+#S0-{Zw3l%6yT1t0Xav9W>=${^ODHVjh}B4G{j9Bo?1v~&ZM=!{|HC>#V$n|P*5SI*mPd*y}8 z!xIF|pgx0ywhqVUjDqH^HNP`S1}1jVVlTb~IQ~+k`c$zL;`sG79%dnw{mOg)dxC$9 z=zmj-l>8#u|3e|AcHMAc0#@yBP3%v0YG#{`&g-?AqueI^qisiydlw#a4BH*?N6G5V zqSGXQG`rtu-{JUhJO1yBXQ$=Z z$(QGP*k7V<`VGzj&oO*?v31MkzCC}lLcy0>WchRNuhZYVLdzW9*lK>OE8l%v7wbkl zHy{F=5(##?bGKsqZCzq{Xj$i~mp#uX#)otmYL(3m>>mJj*RqTkI8tXYNMneciiyJ>G|+0!v+ z>SMJ_xjkOCrx*1+&8=}@pLftX@U*f={pC8e%gwR@NZ34<&hl9`*sZC=2oj8#l^HMCOTZ&x zWzsQOXnWy^WZTKjCDiYSdgi`XbrsPG6YlSC%%}O1t&4lW(;Vk3nb$&F!g} zjA~8~(WkbM_q5N>TTbF`abMEeo_+#WD%O?Uy%?+w+c;hcd1!k}UMP9;c0D*&nc(dbe1R`({(E`lF^1SGfw!vy za$M4E6%!x$lety3^WSyX+WK{bgAyyQK-*$1uV~9A ztprs(K^p!P|9OsCY4|la!%Yq@YPzEw7j)y<3Z$ywsv{B z(?BenF3Kc)-jZGtE%&*(n8H}j-?s<=o~XLn=Kw z0w9W8*{4DHEL0EG9X!?1x?ez-*RDBHGDXBTHpucn>Op7~2t$EgaYjY;S;?LPMpN9c z_~gy2sCqFm=k%pxE2JsT?9i-6nYb!BbnJppTQbvB5bf?NNQxOPq^C zxN8LsyBb-@#tRJZ$V8i(OWe;cmTIW1-|(h{Oht>7dF?rYPiMEwcCtHd)LFYiqdJ+yIMMTIfgh8$5>uSo$KRIfmTVtC53f-UFCJ3R zGRhfNBe}eczc=B>g9OJSo3hg}79t;34VMlR*6&yMlOZeHG*wSCJ*unNCfs0AT`w4! z7O`cjr z7k5Me41$1rc98*h#*Glw;5)h-=QR9R~$CicX>2M;=E zvU?-5g2(fbY9zU>Yp-*+IEE}gNpS5Dp&f(K?N5{zxG0wCY)3g1l7;6Edkn)u zP()w$&17qht`L;&6e+A|u&iS!IjPAmrek;`cFKW&wBoQ$7bzSfCb|GBlRS*ZZYJK( zY2|hAs20RCN>=F|V|t=kI&K`*LpHvDD3cSbLFB3?cNa}!Pz~O@B_;wYBp)@T;higVFB<-V>y%-hyvhWQ9-P}Xg2&4*^&7z9SG_)|- zUUZ=Iiw?IXs(s`U`1U_I6(jJnM@5Nn{E(WejaAGmdfms^4$_5c9(GyUOJG867%AA* zEodO$c-L&lVmw`}9;J-7kdil)IoPA#-%yL9uJ`Z` zgG;qMl#X#~B9kVXH)i=80$oAc96%QT96gzp>_Em*GpFz5TL4*Z+)K|T4(Z%mwqmFn| zt8_~!2iY;bn|JAbleDtXCQ=2iQgqIbB2B0$Yz}sUESyZWI}&F=C(}^vYC4r~m};OC!l{x=TQ{uiQGzik`@iOku?`m5K8~n!2EKGVi9P=euicrI@sXMs)w?%jVh!@b=2VSeX8R&Sv754m<48;qcF2*;Cc+hyIM9oUCp@&5kdOT= zV=?i?Q*iouw2@vKw2uJ}>{yP)b~Q1;;~BG*shb#dd+jCG@wE)3QfVnS2B%EW;qj>l zA1PacvF@iy^rf>!Sa6p9M|~vhpN==cvl!lR);rPK29cc;i?W3(oG^#xaF z=P%u&B^7*a!{Z6o!uB|0n6Sns(8ezHA|-7+MKopW33!wTJ!y|J=i$kscyREM(L9`v zf7^N-AzGl)A*`5;QL#FWHW9UGsFRt_M+IXyn|Jk*mE#CZnr1yKPr-Z+ocl)UuLZdK zFNX`UY)nm>Hk5N|o1xFB-L>z;6ETKpr_Hoas>8OMbQPRwl}Nb^kkm@m#dVABs$*!uw&2|iSWf2@x+sd__yvWMan=WWbQ%rQRnkc&suQ7sd@%CY+ zDOh_ATnAA}uH2c}EpZ^q7pP*EhV;Z8HS!9;ia~lgwW3D$Es0)S$%7AaxNOXws8(FR ztaD0v!@$qsPM!(RylgNHsdmB`mwX*IhdW$re!kvSxBM%y42%?x4Wdld?yJ>XIXyK7 zj)h1m6)n5@qZ-EQxTvnGIU1k)95c?zAMG}C9I|@~jMs}JiIqqv$f?S+RMy3K!m+E> zcd9g(az5>;u;;e&Hn=NuCt1wyT(^) zBx8rohk<4)8DrPPaWR$x&f*`MGMXY$&T;#$?|B^^h1S#5*p| zSWxKq&^<2`7ZFzW@DZBZu?0zvYPne-TwRZfC*o|lFkfnD?73?dzFSgO-O;fq#b(!t z-wZLroIdx>tNeN~d%KBpvVy0M5hzZiWVJ*#5+SgVm!(N@IQJ;sLFg^Kk@D7UfTY`@ zo(0iR!Ti#|)K6}!;VH&srQ+FlEUb^t9RX`+jO>!mZ(8YCh!W!+4>BC$QO z=6UMhiV7~0wsjg$M439f)l&r=$dk3L8tl;Y9pSW7N?dCZBL%b8{K)$q1!q4TO)JWoFw6@Kv8)M)71PbEw2*h{?uh~i~VqoXGhwn7LJ2~** zz<&$mzo`s_HYol;Vu#g};|&X%@-k2IUWA;Zn5mjmn>4G!c=`i5F*zR)Yg;O$xP0epFvFwaXnuKd#8Pml2o7tFFhL?TKPSVBa`W(y|9K-J4b3MPR;lSk_ z)n2GQ$E6&Z7W0l)l8GB}r&lq#RLnAQr5-}{BzmX6QLU9&!0HTpzk~8@4P(7`VH6q_ zeZg-k=fH<*J>K=(yXeqI=3xg87#n=Ouim4UEHEvHeT!Y$^RvjkjG8(;^`7Hi9Bp~k zzcB3`#c!IP=1V&)pVngR>ei=x8QI-e+Twq*35BJci1(|P8m5ifYN*aK@SIOh}ZO!(L3ryXOs@V-f%zCnfYbe%HKv~ zn*Zg$3+BBms61h6CY>RWb%^>wlKu0ravj zgGEO6oj*Vwq|0ZUs?nA1I}e&RJa!^3_D!pCslrO)p!>_Yib?hdl>$2Zr8LmucZD-h z`{WsEbN@PTkNv#MMumgr!1+bd-#{jg(R)ws+@^2qu>VOf0WN^{l*)kf6{&n0iP8hJ zETu+183t)Ydgs?KcD8Cx_0{6{{e#qq6<{g@$MpkLNTGBZkg0#N<*+0`ID`2}!>uDa za8Z)fePWdIUpgbD!OG!qkl^b@A*wS@_=iB*hhi4 znl}iMdxSUqQvWIKpfyRse~9M^ztw9(ma;aRzr_Au3-CXa`N!T6k8A{{NfczZ|FYQ* zm%li;YPu54y^&Tj-J=@;Shc{`YaY+Co2d~v4KQ>?Fyh3~;Mr9cM4rU|8#-WsgV39%LF|9PE8Tn(u2@l zjZ7qBZUgyJgyj(a4|DuI!YV5H=_8jXFNnXIDprizdQsC23N%VOX1gd zG%&z<4UoQ)yOr==B6Wk?t1%UjA;3RC!aDxR> zgaw#T%s2{)ag0Ejtt9}h&{6>=A`m`;H-Q)MH*aYHH|T<*PnhXo_{?O$T4Esa|8P;I zFT)DR5dX}IZKi$%RPD#x5Cg zV}4G4WJh0elhxQRk$a7^c-wQF0UK)Fi4V@Uy4CwR9z9~kPO)Jx9KXabH_5X8;v=Ep zBa!>0akY!;)}~K5!G}$8$KtO0+F?CW}f>;PSb-7+? zLW29L<5hVlHUAOTOrl)xj}s#>qc<4?#;jSjOhAJX67fGKi%rpKkq~M|4Q`* z#4V|7uU=}quko*!K>cLuCd)9r>$P*|TTxP!0oNz1x%iutxcObWOChi3`AiQx*xVif zZtYfDggTGsK3a)0ji}}{or-B;2NZ|3NYrq8ICiNZVoQ>> zx0RqV?UnwY6iM{D(zy;VDdJ7f+!LXGl4F_EUSc&gOAUEPjL1iFzUK=KwymTixo7AM zMafb1l^F%cF1dB|+&PVT=x&yi&7GC=on}(v%`42>=dQE7-v()xRwut3#;E7=9q&in zWs!3dW6|jN)h2R&v3wXh$LG?`-(eV&WwU6f*^F_JbqQZDOYWtZV`z;qy#tM@@vHyJ z+rsRqY1rRBnt6u8mFr<8(#04{IfALRa2##JCSr8<3i_kD8a<9 zP>WC@&;^bQwFgGe#xhmKlGAYkc3ViV(i~-L#FaRrA2c|O{3>{TX<)<)wm^9_v!%=n z13TWfz0x!EpNxX$QAYGQ2{1{JCNoyE$U2jcbq8SK?1!F~Zdq|JBhgl^h+0bTN4s^J zr^Ca0>;VB1WI&Jm6&?4>7^~A?b!=Sw*$41PE3!m3!bkjGkvD0Qpl6=#|N6f9Y%8zV z8~1XS0ND_}y7+N2e99w(LN*8szklLsNyBQIu=aA9{ABp9w69WC+`$`A6FVWgLh9lu zQpuX$GOr`pFK1*8?^+$@J0^~y6*vL0nZ%A)X1^KYgmkfXm_jIuAL+t`AVAAOp<0YC+FMdX@J$B6V1J$owy+Se?#uO;n<~=-%4ZSxzwYH46A&GyXJa zQ&+H5%)dp1F#%h2fj@Ha6C?LZIZlAPV@_loC3T1PD4pw?;pv0;)UiX>m}k08#$QgB z7~p5$mRb?B2T#-GXHPtk(O`-uR$7B(1HyycAGzczoq7aK_NqDKMdxZ%d8hcAjHOKP zhY9dcW}@Zzeb!eUy=cG!LNXL%Ey7Hq_jLEB5J|FK1@rm$|c*Z%1NjzUe=oF$v zYUQ4g(PRV%x359l)hIoG({7=)WndEHDM9rWAXrkw z^6;-Iv68~UH^IjNK%1%S#_7NlRmau*m2G+>Alxf%$f>lw%5(JbZCavlOSqz~0tTAw<8e^YqbIR=idtO8v?GHeVHVd%1~AXgW)3`6aq^Kp$U`qR!QyX4 zW3{|Poj0#*TGS4ky?3DNYMXtOA)VYi!I86d>HhrpygSP^<#Pho6(71 z#}FgV53~;sD@RJd$xYlUe6g9cc}FSz3!(>GBFBejYhbX~N9!uQbv-NAHHImDN$g%6 za#xz{uf9_qZMkK&5r+P&rz|@0Qkub@J3797;c*ieS2d4IY<$k~liUp|3`{}a|A|3X zd{jXiYG27#!tUu+`n0|xI?e7h^`!|oHJ0HB`+MV^ioX$(QCf99SBTxqN@5tfX-hXg z0CU{=exT4f1lJr=j@C1gxCeiUPF>?*B$vrY`*>ys^ZM>ZpFi~^-hAb`5!sI52Yyy= z7o6c|x#Row0N)D^{s3oByEt~$e{w@}^U_Y@B`K(}?$*6SqUS%!C-#1hv1DLXF`BE# zH{s)@mksN6Ty1?iO?m#Z45cEHaF6tNTMp6aM$zdDlJ_31X(6>2*U82)@@&GvZ_H7C zpALUMr%paJlgnQER7>@~+nh7{g>3eL<%%i?^YX_VC>Rg1CIJOSvCQ*lWev+Y!}HD3V*2^eitN0^SqK;_uFm5g~lX zr@ZlO%}d0nw9sAO2~@aQj1@{EHh-wE6|X5g5mWsD9jhyx7T2@W%ZH?G*d>1+vTNBX z1?({FZ^pWot~p1KmBL_~9N{f-hbp@!9QjK&581h8)Pfy)NOu!oC0aLt!4C0jpLjP* zBKt}qql31L5VdHr*tj0mI5?~uTb=e8Tf%~~b({E4cLK!1YAJCNlM zbEsT7&_lBE9kRGvQ1oE9qU3};tj?~xVh}bvZ_h8AF^^(R|;}CZlT~lJctN?U>o&1 zdr7K2Ft>|AZ9x2>#*V7to!Rt_v-Mp|5}Y&bhoad=d>)PlvRmy9Wjk!zY3X-8{eFwhRq{ZRj#JvT?2vw5*!!F`DM&s=?!;`zlf2Hq#~RHQO#fH@(bK+ui1k6&h4k!wG8DeK1q`t)&O!8Oc=?;H@%D|sDF)8`<>jZocp!s6!viG zuwA^V0I}#zeSfhDIPM!R6Re6))bl=T`z9l&B06zo^RDrqj(w*(0@7jwzvE7;*p(iu#Fq+4VZzc*El z6XBAlkYL}Z7CobC8w*9?2=z}IZri_o3ibbNC2b$}X+=mhF@tXRc!U>gg3hc|jCyB04ZQCf*qflZ6^E|i30ZJCj_Vc1QVR(`0^Z(Ih$LuSxM;K6wi zveJ4Cu`fpTrSzrwcu0YcTo}(%_l5`H5A9Ee)JF|olwoaGA%#=sw$?~SGt>ZXKf@8B zZxJn|LTFGt$ObVILf$`}WM>0flZpRx8OC#HOP)7_KesEhRS?-;P*{4R$q?Sf0)_HSQYqemjb2DtD299L#aa z^K4D5S!t1#7{7adz2Q-tvsc{Je$l;N#-%e$*v=dNU9y6qbku;movm?6L#%M!Wt0&u zT3x{;%COs<5I1e`v_FdESuT#CwTyQm5yPbFH;b_OvKu8>XFJ!t#oqvs1tF0xBO3DL zdfZT@WzfepoDa{&`S7T!B$F7C+oO$9hsU9XDugmqAfr*r5NrY(>R*M-!otHpkJ9r# zPa$)*mRQG~0fGjo;xClR^dlf(nOTX93@MNmnZ7y%OvgtjL%LD6^kh`g4wXIVrmm8q zhO}4liZkTam7bWS7BG_Vq8Zzvp|)C@2&6B<5@jSQs3Nmgel7oy(?-+JbrK>;l__k_ zVOi!VM}wG=Cz8CCPeXx6!3Ef;A=%;ai2KU-n<+YAoV{OAYw=ICBGuiP>t~yzLHwst zaPs}AxP{&y?$Eta)=Twz#Q}AHOAH$E4jmB4vo3+#u;C}V_YF%vXNy4Y@pimJ1Q%9) zT3Ue4U^-vMWoSA^7s-<{$jkS;=v6h60CBH?esR&b{5W{x3?IRXiV)~=x9XD*j8^@F zWql`tOosTOiWe`qi5Ije+Hm2PXPX;w9E}+*G)H~>9CwgRLA(a(pP?#d9K_6jXg7kX z+0RW-<8vEsw$qi;);HLF*y;k>*wk}rJFmeDj~;g?Ohjy}YO;wBeb_S;9kEu4r9Xl- zB-{kjCZ%mq_UQ6eH&vD?T4!dLz-qE}>Z3>KDC;{5H{#+K^yVW+%W}6XQyApXV+f(d zlo7SaWYOfwq2dvZ!eR|yp5b5SJOW0xUZqS8alCRQ`Yl_#-<~gt(c$|?jPt&EO+smS5*_BVBXh=L(4kuso?#e*M*ov*Y+4W6X3amlEb<#pt8RL6iN zWX3rS(UN_^wf4d?TK%b1Rki+LD4OgxeoavlP(RH2HI~oP9+kc0dR_=J2#bG-fk(@~ zZO*(mI&$WaAFmiD9?#UbHOSBx4qht<#;@kqdIaNkWEc54@j~95E){5R8zN$Y%cw+3 zQ-83qyrBNVz5BW}dUUktS~VX}OJLg5^g&$*F5%M%BC?Ur0cxFJ@gfxnZ|3!0K1<{8 zmJ9`U7x|kDP*+u8JZ#i;%Ti*v8Rj;VZ&auze*3mS%6;FWr*1=P)iBqZ!e)tY=yKr{ zopqL?^SvIcB^$LZhzhCJxM8)UtQhIjWbu8}hJr2#)U+I@COUPScLoG+gtShkWS!T&l4iu{J-kzdNXp5R zb5YEzZ7uY>Arav4(+PR}Xly>&teyAKQ#j*P0n9D7j?$G{B>!0w#rjwCn>VP?ojwBN z`(^#L4?~6?_8gq_ANfx2XLFOPpy@9_=OxVc*(zlEk$2b*nX+q<8y(fID%z)xESjd# zdpCwXZFirF#R||q_N7e11~ELW7ic_i5ltys@w?w|Y=?UJrtjB9%mf2pVu}u@orYF= zRKdE}`WzJ~C%T45C#n{N8wD)5Ig>nFaWZCqqGX9M_uObH!`^r$d};0%bWx2EHi8k) zSK+qd&8nQK>^XJla{KGxLT}MkZV)XQC)(ZwIadKig5&!L*MnI3@8%Lqmf3cR1-GwrXUf|Pj z*E;O@y|p+V^bA-SX}m3+&FVoPE*uhyW$Gx6oyj@pY9tVIJwFnj_T@%pUuX+nwpN@8 z}VNJJLrfNd6anCT*juQv2}1~qnfHm-*E zBYg<2nF#@iYU+mjRFvLx0Wy2}&}mDl{^%(^UjZi4t59Ql+k5{I|DboGT6IvQ6MX&S zRPF=F^m@{Re&JMCDUg{2`&GDs!l5Jv>QBCV=7JtA95*N&%2lDHiy*n0F2dt*1=3~d zI`pZ~U3$*RP7Ca^HAcU)W@pq#hP1%Lg_x09*w=Bei!MYCylZAxtH};I6=7T{?Cb*B zi$-Q>qd%CG-*>;ZeZ%SeFMP-i>6{IH2v{0nUD&zlh$r1N<`XqMn)>sRyCY78LvWEz z-F~}AT~qB$d=tEbaW43ry+T+2SSt_`+A!IhCr2}am<`1BdZ}8jJ;SP2Vx=u+NDelY zaS4V7cH~@z@g0Klr$(QmpU7)F)_m|0Lg6gyDt<$;KPR6|e7SJW%5V3GFQ)CypP(NX zvngCM|B01uo*exhS`D-i|H(xF!Typ!i%YmUe{hJ%n*HfZ$yuK|HyG)k%m!W5g*?wF zgN5$ClRQrYsKiy?zG-PWT8Jy36jkR}{qXNzA1-v>WoU8ZLK2FSO!=baH(hI*`EFcW zSyonw=_Xq95B^Lqz6ts|T3a$h@F!Ly7@C7WOxM|Ku}M*Oiv_WA9l?k*zgS+O!a#=F^uohaiXPYp*yTiplI^_&8qcT```&l%Gw8Zd<<5OLMs=YBUM+rVXIE#^OZU zSc)~oy$C8?sm5Mpp@hTjm~ABCOOhN~=s~vc`#vU^ye4R^XM>)gucpzcdL2GmrWur2 zb8>=rM7z485`*+!4G7ew<9p?r2dYcGz9(?5`fdn8Va1D7VfKc8nwItY5qKhbb&hc{ z4yqmqhsDMk6zzVIHBwqfB8%V2>Pz;n+cA7Jb;+q32;Vt8ag%4-uOz_P*-9Z{4y9Hi z_?~_y+)=!EyC+mgP&P(u1?1#2R4^v9RQsgiV*f*i@#i|3@X-A|{69lrWC046?{d8| zJHxS3_5G$CK_?b(j6L34KSiK+UM7qZ_Ap)PoIT~kybO`xrpmjn3OpzXg)*f5lsVk4 zvMBE18!Yo*XIoK{++nv^<1yX>olEDekI16Q3=t8Sf%Eryb3G~nmn#QYYOfVlhID1h z;Y}j&I3#Oo&fU3#^#04-^w}mvpCmH+i}IGhw3|dzve%SS7X+|70!1VVl2`Vq&z2xo zn9*?q7i<*L&oY#bOVN%*FUSRl7(iNkt~^kkP+NWF@Nrjtmh|`F&9%>t%6721LF0|a zMZteTGZ<+s-D8JueU;m!0D73BuKi(fRoL^N%>H-oNhz7On^@(&?#a__f4s6-qYs}J zz5KLNk~J)pjM$qbV)%eZ%2!Np8>0^v&OR*-!eXWNTkZbt-UupZ>awI8pWqt)Wo$<) z%Y6cq%$6a}mGOPbayZN~`JEOWi|N4qRz-KqlBGnd_yWF6q`4@b@Kq9Y1riioD3y~W z%ax;-$;{%zqib$g>fo{UguP6Uegwt%!8n>b$QxAQVurA*k=>%$z=;{^PS5_yMSk|A@eDSl=W zS)0e_-AC(kGa}a=YeB2X2B@-J&`xsi(P=gzdWN?x6`K@{m-9IJIU)-h1KBXcFU5jDdQ)(^=mcwZb>6nHdEo%>tGBl zGx^Ke*DC#v12v@h40{G+-^jh>mAv%dqt0_J7ZVP%a|MdHHx2wa&hkUG#&Chceom9P znYbvNXEirTkVepg z!Zj|r;ij$6;-Z>zv%hQlFS)iB{o6wT21A{`?t8!bzyCfk^D$HnZ(mqy_eeTK@0a>F zJ$LwxRGIZXo5vUsAD^xH9q<7?E(z^NM|OWGJg!?m(M$E*9sZQULP{No1)9#zyt~}@ zpW(@wLvyxJGxkaJA?$J*KFf^sqdM%!JjDHcCtRxG7QD#Q&#UuGRkD>vuOuxwSWZV& zR${r#QS>eOA&7kkoU zaNk}bgNv=^TlboX(zgS!5_^?e6yACpkYbDZ*7*J-iVjk(+i-PK-VhgBxUQ8L1G$_G zITCX}2VNw1toQOQHIV7|?)LEBIdXXu)qov3jG~_lZjzb(DlW59YPlb_hx;Z=;;SJM z`E-N)t4jcXOmh$74jF!^Q%$S!aghw~j5k@HclrB3f}M2co1sqlkEj(V-3~7jH&1l3 zbq|2RmDHVrTT;m&)(uTu`VnklN$^EdA;-`S5 z`fMM&+Q0q^C{NhqG_3ry2v^PaJcv^+4wtV?MYFeu@VG!zD5tFEJq)Gv!bJD91k(C|xq7!++@neYan^S765?L#PuWt%0{iv0rBNrpdIP-M& z@rp>zIk#^2fH?s{3{UGEX@)*@7GBkm2l`_oV;P7xT5IUXqeM+D-4QucVz{EwE28ca zs#f@}jypBSHnP3mUF>{`CJ}maFop0IHL5ScQu1Q>U`36-AhQ-E)jQ+B&l=xQ%U}8s z?h42BGLd&*2OPT{CCK;@Bs}4iOJfpeDq%3FDFcc(Ahq4M3%E`iMRg5E?T({bWAxPA zGPAGoY!$|`xs}|;9lN>f*=V-Er+Xk5#U~#8nV1@^50Mo&Dsbdq(&2S0T+8nk?EEhA zrpe-gRo?3uOcjH9d<~PTVby1aA?`t#jC+!fyW=FzQl9|uhA-#*FHh$#(=W%*j-^{d zQGl*UF5+$UAU0=^%{TtYbw{q|#`FFm5c$BVdDF&Pf8~w}JdB8}YlsE5h#i3;x>SHpItY?5#Q6JflMi zJOzqUYR#Y+0a5*Uc3v5{r+D1wotTHQRTTciu7p*}^;*qLq~`wCep=SypBGU?8CK>> z!qB`}NuWm*dy7oC(oOL3uf+gt68CVDPi(S32YM~HRP~f!_qbblMk}YXGy?<|p_+Dx zHKg5QRD;1*8kY)-823t!K{H$y5-%MXgn{B{-yf|)ueJQnB+O6_Mv%bLGA&X1h_gXE zhF;y_FE?ns+*1k)EuH-CCO=E!Km8Gl-LxE)tzZ_lv2S>D49c8R`D+=7kfGrO$PHN6 zRC%(A9j@p(;oEQEi=@}uDqP;uo#D)bwkZ#!T&h2cTi}x%R4E%IhpR~MRFw<2ykTas zNco6O`d2h64oV9Wk_~sr#-|d+yZj&&Of{sOZ(ph;x=Lxfl~5XQv3;uP&P;gS9!o7H z+IKI{9z_G`3Jt(5bw2h4b`AGy9TlP}b#E~ch8*|2lF}PiN$+#bCHqv&rJ9SIceKnIlzDK7( zLWk4P2x3&v#sj&7uNV>W<-wD$w{@|8&&^QuIO0-_USw$I#fb=pE03o5=#T?FaH&X# z6vG!8R5NQYyAy9t7KqYuAAJlEmlSCD+G;9+h-5oMU#B_ zqu{5Mq`rQod9NvpVHqy3Sk39+L^#ViD#h9ob|ylIbRTF{D{zi3kmToG86hO#X!#~> z7R8V(h}a%=7Xm(Y9Ug&KRD(bkDkm{PFFwh^lJXn9Bx#m?CCM5$H(-yR%g6*LjEQ!} zv@aH_;o}x4up;J3mhIW%tUomB{g z!k2DnC$E*NcJ+>z@+1TucrCcVX` zaZ9>HXUgq0?o~C9p+D#URn3(LHFa+Bgn(2bKwTg#id9(^8X%EK1quqv(y+)T0BVhk(Skv1oZl-BE?R#!WJH9p|j77%Kn-Kg0 z9Lbp7zgEWR%~m?J*l0pcBaKvIF+_~9>b(OLuf>q~H;t~ZSZh|Tp(w3^xmfwf((;mk zmE)Ch$9L<>ao#9}I1_t;#+I?)wNO+vY@oVO3n5met6gc5dK&?R{j5ZZ$$#bc*4nXO zL38ygM#po2l8c>txqZF)d4V&@c(HrL*y;U?Q>+1eut@vSXD`usC>Z}ET&X6<>(S1N zt|;4qM=vv66wem-BpVof%T_!UitVmYCz*Xm%te!4Z64>>*D{Z&6DvDX`6d$e#_)7p z|G^nzVNHvVsebuNz-t%WS-CE%U!av^lgtk0Zq~kW<=^cu4dass1{VIQmF(4crxJ%S ztLXtMx}r8uftSX=7R?(CXSkewdx6W;>}w=4<4mdDg#NgV)^T#n>A|2cwETexi#jj~ z{G!baft}pXsp!lm2hmrnCEGaJz6dErE1V`*oR0})?12p|O*3cvKW)y6uYGcw5U&a2 zq^Bj(zeS^NN|?@^64vtuED9NRgXZUt`BUR59al`3pVlRF$ufJKHG5ZAfxhkvu~4rkQ7z_s=XC$C)KYh?Z|>5`m?}I9x<#U$2oSw$AAlw^mojetwW*{^rrsV?j!OvUpXMTvaryfOyFx0o0b2}sV4Wq#|{Ia zz@z?@Gg7o9pmYahIFOn|NqQ>l2T0!yQ)rQrq?;NsGJob~OS}GC!dLREGDNvRF|bLv z$=I9znZDhkB(4PXlk;QsLgaoYIU8O1Xm#N8NnDXig{vX2Z`pbQ)roHc(|r2?KtDvi zn_<1#_k{NaZG4A(H1gu&b&r-Ox5RfJ78RK~An^f|(L^qJwr7^->sLQyy$K(~wc|~f zMCF?N1oi%po4DE=9e=J#1kkI%r}mw&56AsclR&ixIE?X9#AYZ=0R@&Ymwf;rn*ewv z8h|tb&_5l23G2i?7En)xQ)b&H_C4=}fyFIYPx~IZRe~bGBiV@twV!*v9mLt|{iVBg zj;iW__Ne<+%iCN~&`I@s@*z;q!Z+1$0=6zS<21pL<#(q^vlE|aM1_`bF!0{Uvebe+#?D4X+d)n8brtwfe(#Apob3?Blh{2MPpPw5t2GKx zxg6grY;a3Zy{=qR>b)F#8O6+e|9)D-SL(AOXZGc-cYAo*8BB*!Y?Y!)?#&|t5xcjZ zZRHNZ#(`?U`#fhI0GtZrjpB#SBE(r`zGvTDoeo2-*ST2rzqIbP(20f)3CB!)nvp&> zUyFRVq8Zx|eM==d4%8Ei+(6G&4gdvEE$K7qmXj^dlGR2VK_~M|)$ovXofTz4P_q&dH z_16ehaud{{`R(tGcjn#>#b1BPAyussPE=C|k=kC0*_m^+=qW6U6-mET!u2lI$D!pKe)A+_8quX$v0rJp#;0Z=s}VT+wdA# zKJak=%iAu!{dk@avtt|h_qW{#(VeE@$;P6c`QN)eC~?T1Lx}SGFEFN zu^w4h!i~=bgHpIRq2(lBQr&?XUV#cNUpqKNzUMJh8kB>IE@)s4-918MYvoU;aLr1q zJTS7|9e96tikcxmb>ekxuIth4)lPCiek!rPNv6T!M;Uopj~}YIj?=cDVLY$iK-fig z`i{}8W0#x0)CH!3?)yMoeog>M$x4v*V^SRQZ_^QCkn}x%;%X}aq}&zVF5*~%Ql`F< z1ja<06Z@7vbr^Y%%4BP{5kSQ}xAv4%ijcemcX^J4m}D(LR#zagPeFSJE$K$Yu}KlR zNIus?_>;oTy=U}-pPTsHXWl+ZU(wpiuB)kf;`n-C_)>7rC|9QrO>BX;VY4z6M6w6% z?@7norsyCl3yW-63}?H7&9W2*9%+6Oy|VyUplwI?-r&S%hK`V*IkRi#7!%eb6Ky9e z*NV3wN;+qxij)m&nNrx&IjgI}$a7k9)gJ-Rr|Q2qX;F7)FkR+$a`g8I^C)ymdQqWS znCR)%`hY4#TGR((->Txb4>T0kh6YBoY5Bb=^0{nNAuRARWymAEbI9klg_L{tiXJDb zT4sa#Xx6B4g-sLsOf@+3iOR1Hhq zVe*ksS)f&ghi}ill%C{$KOfuY+S>^$n{zN~+}+KO^FB1#pn_xt_6Pw3nYaF0!M`?$ z#}NeE;GeP%gD2Dox|Q5ltb$>CATTWlzOXgU-Tac%^R0wBv8|<}r0?dySgO1((KP4AS(~^(^%ra+T`=P03 zf(jVaCjiEbk|xI6mVKpxu7c&6^^~?&%UH5M*eegXl2WA3Q}~+43w0)BK<&lx<1h=+ zWGtjD>61V}qpJ*FOb9)YIK95Lz&xWFzxs2;UrlZQyO&9r*zzlziu4x;Z;An~T9Gzh zk(UA@gUy1j1p^O|G29qw05>*(Bi)Qnm>D6=j7*Qh;bw4n=B`QZzYyTAUc!XL{yhQD q)5{ki(EUq7BnB6JJu(1?|2H(K&qAzs?Yu0})y@Wenr!X={XYQYbVcU? literal 22038 zcmeFZcT`i~wm%vJ2-2%`P*D&N5J5mXA|PGqQk7mq=)L5Nf=W}Wbm>y169^>=f=Va! zBoL&RfYJhlz%S@I=bm@({r$#$cii{Kd+#s?WACi;S#!@h*IaYYHTQb;NKb=?l7kWe z0MKYYd|&_okbnUIQeO%ZVhK5rG>mw;V)xKM7XS$62LK|U0|3XwqR3SM!0$Eyu>KSP zkbMULu!6H&AIlRf79Z&tshypjEiNvlq-Dx0so%UM@yy<(zM)yqz!d(eYkC^#;}?v_ zQxJ@6NR3`pPW=w*Nq^?WmUAS>|8-U!8!TG2M4(2l~o-Bi-EzR z!r~8D?AG&`SYr#jvdWr=dZt-gn$v3~>YbczOr8dU%7q1AZ(nm6TO|?;jc-9W#IGaLusponDE$2x{n{TAZ$Ua-~HcmQ}w z#{H1^;}ZDc6CVJ8E$jUMf~H?o3bByfS5sGwe3g=&p5e}f)(=hq!0Ay(&wve-Dy-A6AvoE%20L{yWtCl%G!dABcV2jypf4aOm zuf-S~9SmJMOt^qaYeQngPZ-U+vreGMXC{}f|8l|6n}5F4A^+6i(t}@bn+uh;6`!X8 z;55_Na0-!mX8)ePfX`Hs(*4p&9k(aRBY9CY16I5MYp}FrKo}tK6M7i6e*r@lpz1@c zWc{yH26D;{?A%Ws_xqT(yx+cm0nTotDsA5UVgkL}?YI6}X|O?btC9{d5y;UxqULh8 z*2q~FJ+fTQ;itsEw>|%sz>2p!K72>mzrEh7jDeT@lJu=0$9;@7s%RWc9tj`+Hq423 zG%obu@j*Z0PabvlaOE_>+ZF6NFT}th9t%#ZS^@e>%&ly$oHdVyDW6RGnj1^*v7?dB z%f=NO`NOIXtBErc1R5Z@RD52(Rnz8!5!;BGGaAs#dOCw)1H_>q~5;WRYjwe_9$=?RtNel^ZRdDJmrJ>k7ID zA+LX5YzP?D<4GMUjDKQ!S3STVYer!b`3hTe=^p9|oxDK@N{keejYUvUgwY9U-J&kK!nuE8P=5PLjOA zeV~_25M^o(@ZoCLqf*60-ZT;ST?OmCy4M4*79FITa;ZMYh(tR9bq)#XfcZ$W1qObG?nSo~X);xi3oAl>VDbch|4iiW* z%yR*-M4S7uP!=1-E&uKNu>cHdo6 zBAFRKz`i0ML93Q#t1D-vU_|Om> zdAo?+$Xr30ZIoK5isuUTzqq3J4ZyHK6 zb2B&OCDcus;Rzkg9(RzA6%CKuYn9T?Szm0qhsIfb2ooZ!o>WuqdZry=yrHsYGMzHQ zq>f{4!BcS#TUQ7n2!Xl^fafdrn94KqtXDJ3!h*Dr3YR7`{wTp*$0*s-x0(WKPKBvR zB_F%-dU)!dkwe55@@kp6-2IlvG`myUawp0=5yJ=RnLm2elo zXLkyt<%9C6{QWZcV^{aDl*z;JdO8$TKZ!eNb6W%UhTo6j_6&cmkY72r;iC>M#BEdJ zqMW;~QganSJTFa+k52@AylZWmvNam5J5rCna}S__wqH_ z>mYW}?(?hZqzO`GJqKzSca^RM2-vQY7XW_%s4|&{v-P`XJW+0H>NqCzX4U z?rF_1eHEo9>J1_r?eTYFTJ}K!Cp{VoK0KYcA3C{ajIT!M(&8a?!H7+5$GC ztu>9xNrIQzeVo z^hAH_-*G^_(59AMF6YqC)hOHH|JFG9%BEMko3b5@RMWw{4nDdg6}KaY4Nwpj|Byq+ z;p5FI*=VSR&b1o$Ng%713eRt^%f0!bMV6nofY#Dryusp>*~NMgDvF=mVvFAH3&?|4;zXl6MI3CZNk5Kgl)L@~{V#_6A2dYZl-}?ks=cdmnnO+%plO~vpKfyg1HX@idFt(7Xw#=DzOQ5{2a_d@BDr3VvUT^PjT zceXTpzfIs2s4+g*FQi$%yb{PlIJ%`UC>=zw5)=I2cIwl=8umfbE^6(c|W zr~E27*L_yJZz>c|FQYHD{m#L?Ba9+jAS0t7JoFjXdCogd2+cI~3+(0uO zWY>N-;YrUjG6o)0_b<4A$GfwE2u^bq``}&uS6S ze_v|0qg?nkt!g$MXF$I}6S||;1JU5oJ1lE!i%fW;gThvDG)A_LOST&4fHrRQwY)R< zY{t=S_ZO2FzPuIDYIZ+IPg`q4Ew+VOxY;lt*7WWrcNgR((v*#5+OkH-%_`<{ZOVC3 zAks8Tvf;U~voDkR=lehWsHqA(Y>MlL8LJCGs={Uu8xrPK8nAA@&SaBs*V)LIogQqk z@jO6wIKRmHx{@*?l}kR-BxXMUg0ku<`#*kBph{jfg^f^ZV5D>#u(2jGz-AXBY{O>; zs4kNSzJMhG?vd$?WUT^|f)KyK_`q5GGY4wITo9z>Z@~X&A*$}IY8W|-Sf}WJUT7Up z31(D$KH<%BRCszO+K|F_v|AhY)=KEtx90rR!?%j-rxFb*8((>-UY?joJo|b&6R^E9 ztR;Vx-i16B%TEp5*|o;mcv;^)o-TU$IV{p-r8cbdxycVrrj6i(bf{B`zFXYXgs+ZF zzaU@M0qUfww2!cp=`c6PV+uGuu6P7ZTa|Cszr{<2e}<1>X0#|BU7=EsLa z6Ne%@7YgDI-(MsTPw-Y>8%%n&>mGlHSoXT{m(F2G_W2kR6ZLn)wzr1PJU+vATD85*>p1nw$B5a?%~k52 zbq%C3tDkk^1Eej+66eAxxQk;qk8A9{$9UGRML<`%&$!G>%m^^FzRmrWHLh*NDa8nU zcLFY@5GTHNHbQy&lhF19cM!y@8q4zG4AGouGM$)D;uo#to}5A3J%^RU-xVS#g-(UZ z$cf{Nf08kxs#P@XDbt}lp@0VsqufU0t2k+d{n!u^V$_b z(K%6Yl}-WFzL&%%7b)$r!hCgN*0Yr>;Nk5{0bwX=IdVu;xIT1a!Ex#y1=n84XDeRm zu0a*d`#GkqV<@{nEs&g$3!dy9sPUnfS;akv{R{H{AG{1`d6so;-+bqGZqR*M5inbY zbe?xIoTbr5yJx z5kQqadfYFej+LEUI~^z`rLdfF@dH6#jO#UgOy?2>$KRU+m+SbM1mPwMy2z@92Bl|B zRyk}wNSls&16MRk#RAU+!DXO)UfU(DH{1b*yH$d zW2^jW_S0hxAC(0`aE?c4KCF4-bmU_7<8Nm6%rEjilS(wz?_PJ+F#_q9+`$Ky(Hchb z>yzJ=+Ix?WcTEx%$(@bBPd?d-u5TF3(GX6hGsFa=SEVNxO-TnS%hqSx%@A}XulTW= znUhfj>EcPnw%K7}ixp+M1Vp+@*{`**<_LvFkc0W%y_Gcap%VB=@|_$T)DVd$$7dH` zk;Xb9d~(%u*-2*IRQsaOF4?(mu(F;}m-k{#`z1z09MwtQ9K7Ue5nHo`Zy!j@25grF z+3h`{mvKt)J~Q7lEVLxYE9x2;^Yn|i6B?^(>V|4N*)<)?ch-Xzdds)1y87DmaofgA zz?J9HGl3fmpR#B~eMkK5Zx$m9oT>2J&J`ZJCQcJB8Qhr%AWm~DHaXtayHyYT93ql$ z81LKyt06B(J0|$=yH0jDovE)J=(|@4BLqBfX|2LQ>Gi=WR&7m_*=4f!O#HJY9ydSneo{im8Q1_#%JzGX$rIA_~WZ;iN3 zc~WNjB$`+|w66c0^*tyPs{dk~+3KA4_{I=W!_&IL!u-ZmG=g#fs*xh{US7~s1w*?G z4qw%jAAdSWQW#To+kYUytUe8a%zL!%ruAktN(39UfX#Z?T9L-lDc1e27d|a}(I!k@ zhC#;reLP(iW9lq4{k~|)-Qt$WsHP{?hODleeRHH! zeDe?+sxGrk@P)_s_j$nd_PZjlWxyn<>&sW0#=K=pl?=AE%{XwAy)t4baw(aBKc%3@Lo^qbuZqbGOLFy)oym7{ zXn8v@)2*nUHg3E=+bswdG?of;o5LmaVAK;E>T04W$*LRq5=Fn)2U~kz1Bb*{1AHhf z)iRm1xZ9hO)ha5UPjE_U`k5LA8GBP?BV$+i>6%K)9>{@(gQYpm14D{=2iGGcV?wGn z>7kh%C8(ZSD^>k)@a_}N2KCZZkG8N_)U1F&+AIgGQDvw`+;_HsJ=skQ&kX%lMbq(; zBpnKG36Q==S}k)@GO?&KV<*{~w?=42O0|~3y2lc!-nR#RX#m!{_>id8g_vqPQuEQ8tRH|5a#6RknGhR78@OO>cP3p=QzJQ;{XQ*>hmQV+@6oa&&k z+;@K-QrROky7XSac$Xo;|H9fW^hZ|4^pPHqF5Rr00Z#5t6%3D!+hyqTGzHGNP;?ts zBsSJC8?5+wWE0U5HE7XQU%~~Pyo8F@&c0+>HbA6wF4|{X$)lv$-y*lJEblVd%w(Nk zeO&U>hf-Wo;S0G$)8%+Hzcx_g<9riS9yzFqMq_-q#~{h@$Hw4?QnJnTRDGs`;LK^` z>c|~&u>TnQ;9AhPX?RQ6!?JJ&nPwVk9(#wq$bzs%6Eh1P&VYIkX!KYyBD0_mT~dcn zR%u4Gwa6)!K~*p)wm5cHz1qf;fV9{1!ILdNf)?|K4{Ox~4iqipGG4&*Fqu*ABEjHoZ5W zcYrK1Kk5n3oc?|oBIDnYgF4f_#VRJNswxX%3R@PTquRVHb-h9=hN0T0b9%ejz4VPy)6fe!ooobUg+Ku9cYiZ(+Q2Ez;O%8t z`_`T2l}8(`V7sD%g?)5Z^5GYVTMlpQ7QIX+b{Gc|bMU{}SY(2N{L@gRP+pV$>jTO)T@Np~2)%x%ctUC8yGK?#o zO)WnD?5zm)6@8<~QaV;G?i05D{L2+cVbpl(P;4dk!AQ2OC<{N>zpqBR^u#r$vq@=b zvIfEzbX3L?T~VVjj?^U|2Ea+UMoFg><^bhLuVDnq(|V($NWHF`skQR7_4=I`+&*I&Gt?T7A2sV@4wSfY&sUx+BD}rN>%= zDfAuiGouI2mW8JRbdT)$jhTZlLh>ePZ%PVnV%W(zu(EZDnLdpeXsY;W=GQ0qk3<)G zJr;9#Ds?_t z`+O|?e|4-Yw}@e^e*$j!Y)2k%Lpno#5^480Iyiw@%2%{(?}92@KY6@7ASC_=g8x0p zf0sIwz|Hr+VwLfcyx{(8!-^PkuJiOP06j-n#n>> zIsUCPcUP42+}ZztNX+MJqk6=yoyQvg13?$W#h+gWz|-J>3kBCF-V#$N9y3;W*^Sdk z=~ec`8%slJY6tuOM|d$}0+VzW^0_Tk3L`DDK<;ovbczGV3{jZQff4RZap};e6k1SI6Lg!yM|1YDv zXa75S{gV!xA=6pTLw%Jr>BO03>tR#I*Y0lmk6*=o!;Df|l?l)8H()nfAI3nZh+22c znK%Po-$3c(Mr5POf_Tk8jZ^1rje(I4B`&MwQ^^^_lm+V2`7F2IZ&K+(c`p*a8cq8t zW$|RlX_k+?-w@o7k9j%8UiTR_`DkS5t2v9gN}D7dw!9 z)#vw=s+}L5(yC{2D`0RdTSy5QcpeF#ega?ph3c=y+09A^TrK+)`b+k-7k?2%G8>{6 zv7+#p%ehu*QEMyMS$f7tIsI*OJ_{;~S?=WHQZ=LkHoQdkj6Pyd&to^foeW2fa$X-^ zdXaBPY?HAZL0NUhe2W!I+>7UTO7XWp%TpDGN?$vOCi~>Xin3iO1kZ05 zPi}T7&wen;I{uQ8>Dt#D`vw0v6jJHZkkpY5XZjV+Y;(m)tLnZsCOS~*wb&bppvrWa zR^Fd{fl|n5GpGYz(^$gLd`V_E-0{VW%4gLn&g)V zgAE3)jM;)5ro9yV3A&Zo_x}DQobDVSgN7ga(sfH4he85KATaVDSwbCf806Bt0{yne zFS5Tr{Jz||Xt(IreNJ24ClicYck=_{oeIX9uj9rKsy5C(rQ9fZ`$drpo{xf_wvjk6+>n&&zvCL#}j`ode43 zAM*SzR5AMJ3w!;^o>S28680#x=tqV*#$RN!G@7gN!X_3;(|Hk=$62Z%}lab##@b5U#vje@QWU9vV zgFsY8dY1p~1rDn{LAX!35TE^z+B}00rS)6<6etDyKawEI`27tG&3XEoZNrIJW`{Lw z#r}wqn3Z)_q4=|t%mN6=%0G3>OY5wAsttH5QIxoVXyg@m(m_gpV+z#yS_$WJq<~?3 zaBVh`s0MV?l6)r>m!PC?|0qaT4R2m~q&EGla~}QFJ8212goa*XWH?vU*-O%VmXf@d z9i*=)4(!T#G*#VRB|iHV3O-`AACaGVysG6qJSEW{8BzbcQ^5yb?4h=^V8D+WJM%Mk zbImwOe|4z4aa-u6fja!{s6JbTO67o!WU|no+7YvA)S}@5v?8 zk(YU3f?&Hc?`->7josHS{70Mn&5k#OM##x199B{y%7%pke+v56%`SKG=4aJ2c4{i# zREE+y%u6!;Xy?{e?6mt@%1g{o<78jZ`bJL1bvArVbOviMXb_Ii)HDK(4X3PncH@-) z=sn=^*}f^aISG{_mgl~9VAc?fdouM?{1I$13i#kP`%-r=8h`G7CZf08oM|@Ryq5G$5%np7>ARwya;S&UJGCJd4F)8HB3S6m5C(X+H3i`7KjZM z{0pv$nC|uang1cG2T67{WZg@n>W^zi!z^BU!2TUs@qa9|Ah@mV0;B|M)x3nH?s0 z#%B~U_V$NS{KC7@lsiFpi;WTuweTy(E4bR5q@qxsUt~KIXKXTZ(8h+SI~g8bYt^&c zk}k&Dzv|g5>Ev`@7u*JM{`98*J7rW{*kyf=Li~;c5n9tFrY5P*Y--xxT%P+fc+uJ^ z=qmWr{!6hp&8J>mv@7Z*QMzw~!b|L*A!N4bLg@y=Ee5!$^|2AOLsalh$~ZW)!O`m zA8q1<-lgY;_MJS`FpyVtWDo)uT1}U<%XxaPa+>34{a=jgyFHaNLTZ#&;Tx(;%Qf(D zW~g;QI&bch$JEq*>?PEGcv1c17<)-py-8Ivy)~)AoM8}V!m5Hwmfcn?bMQ$iZ?rwQ zB2mh(_syzG`OInw?9lLf8x1EcTpR+MO~mdMiNoCW*hRsu&XltApH@Ldg|(?7fnqb4 zr$$S4z86iD2wI>Ks($yvd^7ZtzoJ?+RVXdlXOGFqwzQOJbMvlHvq4jg5)sj*xq~>) z|KE8}^zt56WZ)AC*qB$EKZ`D}@w_54_w9BNcLXs|DeU)eOrr%@c zwLP=a5G15OXVQh%yE4^f7z1QJZ`?oIC!m*x3(^@K6iZ*E@{l~M{81B1zbIg!Jw6PCNwW zpRxa$wEw3@WNW;vdCGt5s&dRAbAf0RZa!233MdmH3P+XpN;G5n%QiZ=nT%JPN-BIL z?Ixv&yLoGAmhtQms%ni;i1?a*<72Y0;FjAZZ0c3U`sq2N(I85V$zZqLN~@hGVt?j+ zPm#^Y7g8TeE~^VAMhB%$#ln}S;{0-GEcGXYEpB)JnaQ{L3HPj`@3+855~B>}el&m2yaMBHN zSu4wkm0!B#!Jit-V@_kiRz-R2t4fYmsRV;v}IzP}3R{_r+V^K7a@C@jqB z;|;qRF5;u+!kFJ#ha+~3_bR_K5{?b3!(YT-Oi1S4HhAha9nWZjsK3^?XX^LZ{^Q_v z!L;mwyLN?;g+-!kQBCbhILDk6yqUlq;bZ`B&KzB!~ ze#U@kkNGWF=)7+9wk78KtudQJEQCt|SNiZbzAFSVO{>1$S@yc1IIBY_pi>SlK$df| zvX&D1BYms z&l{F)TFIJ9 z*eR{JC^v=N*ph|Zi2G2PbwnU}l9i_AkgSSp9{pFr{G6^w0*seqFn5PTe^C zh6YfY?U*1B)`c=_XxbS(+5pZ#vFZWX65r)%^hDP_8WCHU-HlC|*6 zW6epnI=a%CF-c7hzimH~pfX78SE6AW)Ex@j=LKrSknK3~KvTlk5@_h9Jk&>oU454u)|jzl1q<;zUZ!6>&nCyiy2<9h!KY4uc*fh-jK%R;X)wY6Im+)LRVb3(D!+jm)} z+DsZK!pP-)aq=39#}F(iY%NZPhD3s0ka{m>6pX$9SVf-I4{mmU^!^ z1xFwAcH|vW(g0Upj2`pu|J=MV5#C0Ye%Z{(|B7C60r%?g#;N4>rNOqwPPXxR=SStc z<=_3pr)&~hjSnFFzhqX=ik!ZljzzeTZl!^#v%`|l`gtUJCljTAi~?~1{63|78LAKx z7Sri*O%2|M-jYTxWmK|RR*imd?YD}Q=71H;Hv64<2^;0e%b`y6GGb$UJZB+;LMCZl*;2r@Q|J6 zbrO!)z}IEntCBR%j* zxNxJR80aHwTvlqO8D-!fWn$G3F}J2UuC&ixn&RTL<$u&WKw3T7m)ND0XRsd8nnA9u zG|M;yOfDISVO52~w$y{BbgGQ9jQ9HEqVds9uDxZ2tX3Gp`%+2xwyh#S&Ws1GqqRQHJS9P3(m zeUtEG;tjBsX&>GJ;NV$o`| zuy~08pBg5G4}}eGwP=M`mjO@<>-*`yEA**iWLd}2G+squScl)W*0Ij}g3 z*mx;`u2@njKuJP&p{@k^df78~?v^B&xOQif(c&Va;zPn={iQxd)%Vg0mO?6&5{R-J{;EDXL->B%+| zKG(VAqNem1)AI?moXj<|7@_r44a+sU_mdN^h74DGWtN0ENUv*5mY3hfBMp$f zuXQT2K^nTeC)KN}bB3O{KA}y+)eCqYT)uvkw5n$adbOiYzZf3BaCp#a>5Q@6SxALFi$`u`F2|s2#3pZYXQ69)FWve4 zR>Gk9(_0>(Mjp0=Iqi7O4)nT5U?YD_>h=)wR@p#9q*EUU3Sls5n=G)!reD)p z`pAQGN(vwB!uLJO5k1tcL6&e%h#N6Q3To3{n`K~+`LV;-EWy`JF6t+zXMJ%H6!fku zIz7wa;K0Q{*TtC?)j+klpHi1ulH$)ftSh)1e{rk&3b^^IC590#W%q0uqU~|Nu1iY2 zEcWq;Vx$&wKmzt^a-_j|bJN{R=9Vz;_om_;{$iv{(pvbV<>{2Vd!oRZNQFif-l%U^fiA!eEH>b@Eaf5Oduj}0e4$3sgcN%W(}IFZ7Co7Z}g zBy)%Zxf>AwJsbk!`PwYg1hiUCOJZg~V|a{0;B2^vIVBIvOG##9BGxRw?@iYy{Ot{B zfmBkb0za1BO8>}JAZO_t;@WGs!Hg^C7D?3Xy{qVxH{=7a$iCBAs`WnCvtFytcbPAf z$dcxUs#ryg-#H_TrpTVV=d9{AH5!=wO>ALA;&hRGP>(Q{(U)54;6G|K&{&~-3BpNG zba1Hc>EXqt`xZ4;GpH>gguT9^}a`I z97KlPW1(sG*Sbgas3c%-^*18dy%IF%QoqxD+mbFMbcp47+eb+3A(FL^eS*K9TIx&= zeuN#I(nQpM3oyS8Y>qP})K6&p;KZrCvtSQP>+LlI?mqUtUL4Bxj`-!6YXdPQGGFXT z@;d_s%|;25U`i2TWQ23gTFgk8eQNZ;SetC;(J6$}Z{+e!$Is#Q!97HWTd^BrZ6e2h zV{qEXzNbO00zC}@t@wE+xkhAQ2zB1a^;b1rp6msFT+_b^&d{sp3y@XCY-v7+@NG2u za^|-qQPhu`U+gk*!+dkEqsS^!O?(UB%1Gfo!C0Z7%h1ovJyxSLgIB7=dcQ4vvBMO4GBD)*iPi`F);$?D0`WrwLRa7c`f@2s(p zT_7eUhWpvieewsIhRi10$v1z*)F?lRAxmjWTk{B|1^GY<`S8pQpQ(?Y8C_d2&N148Km~7l z6gxWYqsT8pb_*o2M{xFL7K0RB(N#;WyJ(nOHb`q)7vjdNe>~`$wTU*-88XtGy}@tj zF}gH#Svha+p1hsx8f+5r1hZs_DZj#(t!2rN`e4*aojuCj{aA2ZXXDb$rK0)*{Z`rS z+{?pLPSl9z13Js7O$wzO!=m7tdT^}E(0U8q1wE{5dGO35&mBn-4vzWEYur0CdSkIk zQYy59?7_y24`J&W_3UGSQ^`x5*YSpxs-m=`(Zwu5-3c$ zfToQg02owJDt8w~!c| z87RDIo>;P%5hLHsGL}P3ysNP>(Q~7@n=?@DyX2UUDBBri*srh?vLz{{O3rerhk}u7 zTS8O^DjvxDD+Y=O*AV=kRnpE>&81=oIUjhodsn+z4smoGnk_D|EK19<)C)^D+N84de8_A%^F~Wo zh4HDkw#9itHjL3cM=4)kL+T|mK<q_O~gSeERssMNVxfi9KI+I8-K^w zB;?KvbOZwZbSlrR76dl^<=y-W1#hvc|MIM6j$9NejY@E}N;O(|$humP`X+ym+TWxo z=9$*p5S!J-MOWXUn!bWQ>JcyG{B5*sKvmy65+}4tGBTHZaJYAI{3^eyzCgYZ$8w#0 ztISfY{u8rp_RLsQDNy@vF(F8nR)aGrY*<1!+Uuc^&-!QKg@=0gYU>AfZ)L-n57X-8 zL=Lg=vTp3dP?Vn7>$Q?uJdKt2z+r>f_ppc6{W`Xpj+u|;=U5vfqN#oHqvvNQjG2V7m&j1ST5GzL~ zf)OK{FqO0hzg&T!xn_N;sYEbyMRnGJ+cblyLL9hkR(~A7u&vb(sRtRBDmgrBPQWK~ z*0zyoyejmFW7;$4^!tIMKrzL@Tb|fBl`anvV=42rS@pi}5!l#ux#Sy9kP}4E*zJ{I1!zFqezVWKy}mJW4%?lNRQi*}T67At38Emtn>t`Hvo z6Sn**Gf%B5V3Tti2eNZ|;dG#ubG&Ov{9dR-H-myJi>xPq1r&YHbL|soW~EeVxC(70 zr~lG)@)f=V8P@qvO@cZ@4(rq}m=aI8nOE1okD0o4n^qw}q92Cd>b*+|js6Hj<2gkr zA$Ij}nLX6NZGYxzg|;)u+)W%|Lm!5Y;T*WhAfreXr#eIcItiq@9|hL$1Ax7b5Uj_@i_b>6z0YHHxLPMn%Fj|JL0et)L5kSV>OSs-6i}pHV%%Co7*#PW@wYM zxP-zIzE;9{{b9xj(%pE~#d zPd^A>HT zTL#RkRCn(YexBeO^5F*sDsL;vaBYVSovll((omWu62Ul;t^R-k`e_L(#$w*fV0+K& z%4Mb30!H5z(hKf1~$jT&a2ZIdw}?0!jAEFSpChI^sg;wa)-dw&K)4@(1*>&*){Y{AETR1;4=1ZOcYV z`|`@=t}9?=(pt8Q5Rz-eE)-n!Dz~ClFp(7h=n_my`;geIA;_7m_f>I3f%S|I=N}hHl#^oDaPQY9Knups7= zh~2*k8k;fN;>D}FNIn)|&Up1}(-wccZ%4uPR0jip47bE^;KNL(#Awt?9(993Wl^Hm zN-yCTdY;o}^OV5Z(KBlfjyk78cN{DhiwVu&Wf4fv=z@p6O>Rjgh!W?_0+Sb7#BZZQ zIDq7AOj2OAXT3KmEzAAX=~@Pwg6ubIoScL2P5CyN<3Oaok3GyOvkNp*b#D=}_=qQ` zKs02IJ86j44GT!ugK4>!6L)-Y*R$#?*L}0>Te%sFY1GO&)wMnp2o~2JamFOw$#Ai{ z3XYV3kfiTPX&vOnEvIRUV}OAUScjA48KP@ICwpz`JP5r1 z!bbp(XI>;Gqc`u8Rf|yIigJ4-Z7CVU{P1EfcY-M_>k{qm%MoWR{M|65P_uwgTDNnxUsr^3!((%O%MXV z8fhmuxJmEoNHHH|mBOcMYuQTR(=8=hyDn7vR)@z7`*aVl24YYQGFh@ZMhTwwx7SL0qR*>Uj&l3DPz(gMA_3r9j<`kRHPtw3a+0(h{Zou z1(H{{9@x!E2NQPBo6JHu#g3+M+%{+O87)KJkVh9~$4^g*5&gV5rxs-)tM*X#m0uE} zvb5inCs}96L0~5BfZF^nW`s|9&_}qjZ#g?4-EjTxtp%sU`?&9`w%I`GCg1kt*WvFa zL{n*=dNyngS??^aPj+Sxq8vGy++)lu_O`@+aLS+$r0fkbvQ=(&rc}OfKca?9{amZC z(l2@$x0#^g(l}4GFK_J4Ht|Zep(_kOWiwE^ME** znFx4#k8ET4 zf*etemtApgt@{D_s5vEc`C$Q(Bz{k}Tv5v(ikF9-Ri%RqVOg%m969=s`UPX+IWg$< z%>T7_eO~1su0=X-@668q1w1ZhaiRI|ZT@ezEZjWz_s35l1CH~~-{qeh{NT+4*T&O# z_)Dad#CKfb&1se0Ib}ENs&#i__uSno_Tszh-TsZm-{qG57TUkB`qfOJPL1Qd@0FgM zn`3g>Rrr^t!50hGR|UWGoXg+){mwi!^X>YlsrBLS*Kd9B$-=hh0ON;g50{_zyYzMY z8w1m*46mF1);=%nrS}}Hwd&h;|5e!Z%WvD{nWtY2mC)J33#@TIoG3gTF48|SZQ{&# ztGrfTVp}G=DruIg-+uWQLX~CvGBQNGzttPE<}JvVx>f#Dr{HeE$JfjXQ_Okp->k`y zdidtrEW=wn=Bj<0{nu=k>~)j3`HBh_d)x;Hv1I+1ZtRU&J(X0%=7MY zzH@HLyJbDA4Bu8*@Lhkh>+GEgp}Su`?LH9dyWD!J4R_4dvZGJe^EdeDnf;Nzo$!w1 zpcIhDGL5Z|Z|~0-vG=cl0m9_Xy!>|3Mc@pn#Rg5Fx)-Ij*KhB)V-VuV{c&ME|AAF4 zX29x^Jt5Rd+V0D1<_%gnQl09Sht5E4tA%cTSQszg5Xw13wZ8|3gDR+%Q2VzXcnGB~ zLxSzU8^FUQe}8)Kw4I?rwDN{+4d?e)@0s~GFswNP+|qD7Aozopr0P=j%fB*mh diff --git a/internal/framework/events/event.go b/internal/framework/events/event.go index 2200c8023f..5dba6f2af2 100644 --- a/internal/framework/events/event.go +++ b/internal/framework/events/event.go @@ -23,3 +23,7 @@ type DeleteEvent struct { // NamespacedName is the namespace & name of the deleted resource. NamespacedName types.NamespacedName } + +// NewLeaderEvent represents an NGF Pod becoming leader. This is used to trigger the event handler to batch process +// events and update nginx conf when no resource has been changed. +type NewLeaderEvent struct{} diff --git a/internal/framework/runnables/runnables.go b/internal/framework/runnables/runnables.go index d960475008..59da358278 100644 --- a/internal/framework/runnables/runnables.go +++ b/internal/framework/runnables/runnables.go @@ -34,29 +34,32 @@ func (r *LeaderOrNonLeader) NeedLeaderElection() bool { return false } -// EnableAfterBecameLeader is a Runnable that will call the enable function when the current instance becomes +// CallFunctionsAfterBecameLeader is a Runnable that will call the given functions when the current instance becomes // the leader. -type EnableAfterBecameLeader struct { +type CallFunctionsAfterBecameLeader struct { enable func(context.Context) + leader func() } var ( - _ manager.LeaderElectionRunnable = &EnableAfterBecameLeader{} - _ manager.Runnable = &EnableAfterBecameLeader{} + _ manager.LeaderElectionRunnable = &CallFunctionsAfterBecameLeader{} + _ manager.Runnable = &CallFunctionsAfterBecameLeader{} ) -// NewEnableAfterBecameLeader creates a new EnableAfterBecameLeader Runnable. -func NewEnableAfterBecameLeader(enable func(context.Context)) *EnableAfterBecameLeader { - return &EnableAfterBecameLeader{ +// NewCallFunctionsAfterBecameLeader creates a new CallFunctionsAfterBecameLeader Runnable. +func NewCallFunctionsAfterBecameLeader(enable func(context.Context), leader func()) *CallFunctionsAfterBecameLeader { + return &CallFunctionsAfterBecameLeader{ enable: enable, + leader: leader, } } -func (j *EnableAfterBecameLeader) Start(ctx context.Context) error { +func (j *CallFunctionsAfterBecameLeader) Start(ctx context.Context) error { j.enable(ctx) + j.leader() return nil } -func (j *EnableAfterBecameLeader) NeedLeaderElection() bool { +func (j *CallFunctionsAfterBecameLeader) NeedLeaderElection() bool { return true } diff --git a/internal/framework/runnables/runnables_test.go b/internal/framework/runnables/runnables_test.go index 9f34d9ccba..fe26f43115 100644 --- a/internal/framework/runnables/runnables_test.go +++ b/internal/framework/runnables/runnables_test.go @@ -23,19 +23,23 @@ func TestLeaderOrNonLeader(t *testing.T) { g.Expect(leaderOrNonLeader.NeedLeaderElection()).To(BeFalse()) } -func TestEnableAfterBecameLeader(t *testing.T) { +func TestCallFunctionsAfterBecameLeader(t *testing.T) { t.Parallel() enabled := false - enableAfterBecameLeader := NewEnableAfterBecameLeader(func(_ context.Context) { - enabled = true - }) + leader := false + + callFunctionsAfterBecameLeader := NewCallFunctionsAfterBecameLeader( + func(_ context.Context) { enabled = true }, + func() { leader = true }, + ) g := NewWithT(t) - g.Expect(enableAfterBecameLeader.NeedLeaderElection()).To(BeTrue()) + g.Expect(callFunctionsAfterBecameLeader.NeedLeaderElection()).To(BeTrue()) g.Expect(enabled).To(BeFalse()) - err := enableAfterBecameLeader.Start(context.Background()) + err := callFunctionsAfterBecameLeader.Start(context.Background()) g.Expect(err).ToNot(HaveOccurred()) g.Expect(enabled).To(BeTrue()) + g.Expect(leader).To(BeTrue()) } diff --git a/internal/mode/static/handler.go b/internal/mode/static/handler.go index 1020f11e03..90578495b9 100644 --- a/internal/mode/static/handler.go +++ b/internal/mode/static/handler.go @@ -161,12 +161,31 @@ func (h *eventHandlerImpl) HandleEventBatch(ctx context.Context, logger logr.Log h.cfg.metricsCollector.ObserveLastEventBatchProcessTime(duration) }() + var newLeader bool for _, event := range batch { - h.parseAndCaptureEvent(ctx, logger, event) + switch event.(type) { + case *events.NewLeaderEvent: + newLeader = true + default: + h.parseAndCaptureEvent(ctx, logger, event) + } } changeType, gr := h.cfg.processor.Process() + // if there is a newLeader event in the EventBatch, we want to generate and update nginx conf, + // so regardless of what came back from Process(), we want to update the nginx conf with the latest graph + if newLeader { + changeType = state.ClusterStateChange + gr = h.cfg.processor.GetLatestGraph() + } + + // if this Pod is not the leader or does not have the leader lease yet, + // the nginx conf should not be updated. + if !h.cfg.graphBuiltHealthChecker.leader { + return + } + // Once we've processed resources on startup and built our first graph, mark the Pod as ready. if !h.cfg.graphBuiltHealthChecker.ready { h.cfg.graphBuiltHealthChecker.setAsReady() diff --git a/internal/mode/static/handler_test.go b/internal/mode/static/handler_test.go index fcde4e7f25..1a6e5812e4 100644 --- a/internal/mode/static/handler_test.go +++ b/internal/mode/static/handler_test.go @@ -127,6 +127,8 @@ var _ = Describe("eventHandler", func() { updateGatewayClassStatus: true, }) Expect(handler.cfg.graphBuiltHealthChecker.ready).To(BeFalse()) + + handler.cfg.graphBuiltHealthChecker.leader = true }) AfterEach(func() { @@ -193,6 +195,17 @@ var _ = Describe("eventHandler", func() { expectReconfig(dcfg, fakeCfgFiles) Expect(helpers.Diff(handler.GetLatestConfiguration(), &dcfg)).To(BeEmpty()) }) + + It("should process a NewLeaderEvent", func() { + e := &events.NewLeaderEvent{} + + batch := []interface{}{e} + + handler.HandleEventBatch(context.Background(), ctlrZap.New(), batch) + + dcfg := dataplane.GetDefaultConfiguration(&graph.Graph{}, 1) + Expect(helpers.Diff(handler.GetLatestConfiguration(), &dcfg)).To(BeEmpty()) + }) }) When("a batch has multiple events", func() { @@ -202,7 +215,8 @@ var _ = Describe("eventHandler", func() { Type: &gatewayv1.HTTPRoute{}, NamespacedName: types.NamespacedName{Namespace: "test", Name: "route"}, } - batch := []interface{}{upsertEvent, deleteEvent} + newLeaderEvent := &events.NewLeaderEvent{} + batch := []interface{}{upsertEvent, deleteEvent, newLeaderEvent} handler.HandleEventBatch(context.Background(), ctlrZap.New(), batch) @@ -502,6 +516,25 @@ var _ = Describe("eventHandler", func() { Expect(handler.cfg.graphBuiltHealthChecker.readyCheck(nil)).To(Succeed()) }) + It("should not update nginx conf if NGF is not leader", func() { + e := &events.UpsertEvent{Resource: &gatewayv1.HTTPRoute{}} + batch := []interface{}{e} + readyChannel := handler.cfg.graphBuiltHealthChecker.getReadyCh() + + fakeProcessor.ProcessReturns(state.ClusterStateChange, &graph.Graph{}) + + handler.cfg.graphBuiltHealthChecker.leader = false + + Expect(handler.cfg.graphBuiltHealthChecker.readyCheck(nil)).ToNot(Succeed()) + handler.HandleEventBatch(context.Background(), ctlrZap.New(), batch) + + Expect(handler.GetLatestConfiguration()).To(BeNil()) + + Expect(readyChannel).ShouldNot(BeClosed()) + + Expect(handler.cfg.graphBuiltHealthChecker.readyCheck(nil)).ToNot(Succeed()) + }) + It("should panic for an unknown event type", func() { e := &struct{}{} diff --git a/internal/mode/static/health.go b/internal/mode/static/health.go index a0fe4e9b59..4a2fbe7724 100644 --- a/internal/mode/static/health.go +++ b/internal/mode/static/health.go @@ -4,6 +4,8 @@ import ( "errors" "net/http" "sync" + + "github.com/nginx/nginx-gateway-fabric/internal/framework/events" ) // newGraphBuiltHealthChecker creates a new graphBuiltHealthChecker. @@ -13,16 +15,18 @@ func newGraphBuiltHealthChecker() *graphBuiltHealthChecker { } } -// graphBuiltHealthChecker is used to check if the initial graph is built and the NGF Pod is ready. +// graphBuiltHealthChecker is used to check if the initial graph is built, if the NGF Pod is leader, and if the +// NGF Pod is ready. type graphBuiltHealthChecker struct { - // readyCh is a channel that is initialized in newGraphBuiltHealthChecker and represents if the NGF Pod is ready. readyCh chan struct{} + eventCh chan interface{} lock sync.RWMutex ready bool + leader bool } // readyCheck returns the ready-state of the Pod. It satisfies the controller-runtime Checker type. -// We are considered ready after the first graph is built. +// We are considered ready after the first graph is built and if the NGF Pod is leader. func (h *graphBuiltHealthChecker) readyCheck(_ *http.Request) error { h.lock.RLock() defer h.lock.RUnlock() @@ -31,6 +35,10 @@ func (h *graphBuiltHealthChecker) readyCheck(_ *http.Request) error { return errors.New("control plane is not yet ready") } + if !h.leader { + return errors.New("this NGF Pod is not currently leader") + } + return nil } @@ -47,3 +55,12 @@ func (h *graphBuiltHealthChecker) setAsReady() { func (h *graphBuiltHealthChecker) getReadyCh() <-chan struct{} { return h.readyCh } + +// setAsLeader marks the health check as leader and sends an empty event to the event channel. +func (h *graphBuiltHealthChecker) setAsLeader() { + h.lock.Lock() + defer h.lock.Unlock() + + h.leader = true + h.eventCh <- &events.NewLeaderEvent{} +} diff --git a/internal/mode/static/health_test.go b/internal/mode/static/health_test.go index 7246283ed9..6faedc44cb 100644 --- a/internal/mode/static/health_test.go +++ b/internal/mode/static/health_test.go @@ -1,6 +1,7 @@ package static import ( + "errors" "testing" . "github.com/onsi/gomega" @@ -10,8 +11,30 @@ func TestReadyCheck(t *testing.T) { t.Parallel() g := NewWithT(t) healthChecker := newGraphBuiltHealthChecker() - g.Expect(healthChecker.readyCheck(nil)).ToNot(Succeed()) + g.Expect(healthChecker.readyCheck(nil)).To(MatchError(errors.New("control plane is not yet ready"))) + + healthChecker.ready = true + g.Expect(healthChecker.readyCheck(nil)).To(MatchError(errors.New("this NGF Pod is not currently leader"))) + + healthChecker.ready = false + healthChecker.leader = true + g.Expect(healthChecker.readyCheck(nil)).To(MatchError(errors.New("control plane is not yet ready"))) healthChecker.ready = true g.Expect(healthChecker.readyCheck(nil)).To(Succeed()) } + +func TestSetAsLeader(t *testing.T) { + t.Parallel() + g := NewWithT(t) + healthChecker := newGraphBuiltHealthChecker() + healthChecker.eventCh = make(chan interface{}, 1) + + g.Expect(healthChecker.leader).To(BeFalse()) + g.Expect(healthChecker.eventCh).ShouldNot(Receive()) + + healthChecker.setAsLeader() + + g.Expect(healthChecker.leader).To(BeTrue()) + g.Expect(healthChecker.eventCh).Should(Receive()) +} diff --git a/internal/mode/static/manager.go b/internal/mode/static/manager.go index e6959e6609..bb5f619617 100644 --- a/internal/mode/static/manager.go +++ b/internal/mode/static/manager.go @@ -245,8 +245,14 @@ func StartManager(cfg config.Config) error { return fmt.Errorf("cannot register event loop: %w", err) } - if err = mgr.Add(runnables.NewEnableAfterBecameLeader(groupStatusUpdater.Enable)); err != nil { - return fmt.Errorf("cannot register status updater: %w", err) + // the healthChecker needs the same eventCh as the event handler so it can send a NewLeaderEvent when + // the pod becomes leader, triggering HandleEventBatch to be called. + healthChecker.eventCh = eventCh + if err = mgr.Add(runnables.NewCallFunctionsAfterBecameLeader( + groupStatusUpdater.Enable, + healthChecker.setAsLeader, + )); err != nil { + return fmt.Errorf("cannot register status updater or set pod as leader: %w", err) } if cfg.ProductTelemetryConfig.Enabled { From 754a98b08523fd96b1f3bdb5e923b551bec85011 Mon Sep 17 00:00:00 2001 From: Benjamin Jee Date: Thu, 6 Feb 2025 12:01:53 -0800 Subject: [PATCH 02/12] Remove dependency on controller-runtime health probe and use ours --- internal/mode/static/health.go | 18 ++++++++--- internal/mode/static/health_test.go | 24 +++++++++++++- internal/mode/static/manager.go | 49 +++++++++++++++++++++++++---- 3 files changed, 80 insertions(+), 11 deletions(-) diff --git a/internal/mode/static/health.go b/internal/mode/static/health.go index 4a2fbe7724..0216968f1b 100644 --- a/internal/mode/static/health.go +++ b/internal/mode/static/health.go @@ -18,27 +18,37 @@ func newGraphBuiltHealthChecker() *graphBuiltHealthChecker { // graphBuiltHealthChecker is used to check if the initial graph is built, if the NGF Pod is leader, and if the // NGF Pod is ready. type graphBuiltHealthChecker struct { + // readyCh is a channel that is initialized in newGraphBuiltHealthChecker and represents if the NGF Pod is ready. readyCh chan struct{} + // eventCh is a channel that a NewLeaderEvent gets sent to when the NGF Pod becomes leader. eventCh chan interface{} lock sync.RWMutex ready bool leader bool } +func (h *graphBuiltHealthChecker) readyHandler(resp http.ResponseWriter, req *http.Request) { + if err := h.readyCheck(req); err != nil { + resp.WriteHeader(http.StatusServiceUnavailable) + } else { + resp.WriteHeader(http.StatusOK) + } +} + // readyCheck returns the ready-state of the Pod. It satisfies the controller-runtime Checker type. // We are considered ready after the first graph is built and if the NGF Pod is leader. func (h *graphBuiltHealthChecker) readyCheck(_ *http.Request) error { h.lock.RLock() defer h.lock.RUnlock() - if !h.ready { - return errors.New("control plane is not yet ready") - } - if !h.leader { return errors.New("this NGF Pod is not currently leader") } + if !h.ready { + return errors.New("control plane is not yet ready") + } + return nil } diff --git a/internal/mode/static/health_test.go b/internal/mode/static/health_test.go index 6faedc44cb..2fcd511a63 100644 --- a/internal/mode/static/health_test.go +++ b/internal/mode/static/health_test.go @@ -2,6 +2,8 @@ package static import ( "errors" + "net/http" + "net/http/httptest" "testing" . "github.com/onsi/gomega" @@ -11,7 +13,8 @@ func TestReadyCheck(t *testing.T) { t.Parallel() g := NewWithT(t) healthChecker := newGraphBuiltHealthChecker() - g.Expect(healthChecker.readyCheck(nil)).To(MatchError(errors.New("control plane is not yet ready"))) + + g.Expect(healthChecker.readyCheck(nil)).To(MatchError(errors.New("this NGF Pod is not currently leader"))) healthChecker.ready = true g.Expect(healthChecker.readyCheck(nil)).To(MatchError(errors.New("this NGF Pod is not currently leader"))) @@ -38,3 +41,22 @@ func TestSetAsLeader(t *testing.T) { g.Expect(healthChecker.leader).To(BeTrue()) g.Expect(healthChecker.eventCh).Should(Receive()) } + +func TestReadyHandler(t *testing.T) { + t.Parallel() + g := NewWithT(t) + healthChecker := newGraphBuiltHealthChecker() + + r := httptest.NewRequest(http.MethodGet, "/readyz", nil) + w := httptest.NewRecorder() + + healthChecker.readyHandler(w, r) + g.Expect(w.Result().StatusCode).To(Equal(http.StatusServiceUnavailable)) + + healthChecker.ready = true + healthChecker.leader = true + + w = httptest.NewRecorder() + healthChecker.readyHandler(w, r) + g.Expect(w.Result().StatusCode).To(Equal(http.StatusOK)) +} diff --git a/internal/mode/static/manager.go b/internal/mode/static/manager.go index bb5f619617..d03d3596a7 100644 --- a/internal/mode/static/manager.go +++ b/internal/mode/static/manager.go @@ -3,6 +3,8 @@ package static import ( "context" "fmt" + "net" + "net/http" "time" "github.com/go-logr/logr" @@ -75,6 +77,8 @@ const ( plusClientCertField = "tls.crt" plusClientKeyField = "tls.key" grpcServerPort = 8443 + // defined in our deployment.yaml. + readinessEndpointName = "/readyz" ) var scheme = runtime.NewScheme() @@ -280,6 +284,7 @@ func StartManager(cfg config.Config) error { } cfg.Logger.Info("Starting manager") + cfg.Logger.Info("Nginx Gateway Fabric Pod will be marked as unready until it has the leader lease") go func() { <-ctx.Done() cfg.Logger.Info("Shutting down") @@ -332,10 +337,6 @@ func createManager(cfg config.Config, healthChecker *graphBuiltHealthChecker) (m }, } - if cfg.HealthConfig.Enabled { - options.HealthProbeBindAddress = fmt.Sprintf(":%d", cfg.HealthConfig.Port) - } - clusterCfg := ctlr.GetConfigOrDie() clusterCfg.Timeout = clusterTimeout @@ -345,8 +346,13 @@ func createManager(cfg config.Config, healthChecker *graphBuiltHealthChecker) (m } if cfg.HealthConfig.Enabled { - if err := mgr.AddReadyzCheck("readyz", healthChecker.readyCheck); err != nil { - return nil, fmt.Errorf("error adding ready check: %w", err) + healthProbeServer, err := createHealthProbe(cfg, healthChecker) + if err != nil { + return nil, fmt.Errorf("error creating health probe: %w", err) + } + + if err := mgr.Add(&healthProbeServer); err != nil { + return nil, fmt.Errorf("error adding health probe: %w", err) } } @@ -809,3 +815,34 @@ func getMetricsOptions(cfg config.MetricsConfig) metricsserver.Options { return metricsOptions } + +// createHealthProbe creates a Server runnable to serve as our health and readiness checker. +func createHealthProbe(cfg config.Config, healthChecker *graphBuiltHealthChecker) (manager.Server, error) { + // we chose to create our own health probe server instead of using the controller-runtime one because + // of an annoying log which would flood our logs on non-ready non-leader NGF Pods. This health probe is pretty + // similar to the controller-runtime's health probe. + + mux := http.NewServeMux() + + // copy of controller-runtime sane defaults for new http.Server + s := &http.Server{ + Handler: mux, + MaxHeaderBytes: 1 << 20, + IdleTimeout: 90 * time.Second, // matches http.DefaultTransport keep-alive timeout + ReadHeaderTimeout: 32 * time.Second, + } + + mux.HandleFunc(readinessEndpointName, healthChecker.readyHandler) + + ln, err := net.Listen("tcp", fmt.Sprintf(":%d", cfg.HealthConfig.Port)) + if err != nil { + return manager.Server{}, + fmt.Errorf("error listening on %s: %w", fmt.Sprintf(":%d", cfg.HealthConfig.Port), err) + } + + return manager.Server{ + Name: "health probe", + Server: s, + Listener: ln, + }, nil +} From ed10855afee9bf0ed232030f74d7aa803630752a Mon Sep 17 00:00:00 2001 From: Benjamin Jee Date: Thu, 6 Feb 2025 12:46:32 -0800 Subject: [PATCH 03/12] Refactor function placement --- internal/mode/static/health.go | 37 +++++++++++++++++++++++++++++++++ internal/mode/static/manager.go | 33 ----------------------------- 2 files changed, 37 insertions(+), 33 deletions(-) diff --git a/internal/mode/static/health.go b/internal/mode/static/health.go index 0216968f1b..311c025092 100644 --- a/internal/mode/static/health.go +++ b/internal/mode/static/health.go @@ -2,10 +2,16 @@ package static import ( "errors" + "fmt" + "net" "net/http" "sync" + "time" + + "sigs.k8s.io/controller-runtime/pkg/manager" "github.com/nginx/nginx-gateway-fabric/internal/framework/events" + "github.com/nginx/nginx-gateway-fabric/internal/mode/static/config" ) // newGraphBuiltHealthChecker creates a new graphBuiltHealthChecker. @@ -27,6 +33,37 @@ type graphBuiltHealthChecker struct { leader bool } +// createHealthProbe creates a Server runnable to serve as our health and readiness checker. +func createHealthProbe(cfg config.Config, healthChecker *graphBuiltHealthChecker) (manager.Server, error) { + // we chose to create our own health probe server instead of using the controller-runtime one because + // of an annoying log which would flood our logs on non-ready non-leader NGF Pods. This health probe is pretty + // similar to the controller-runtime's health probe. + + mux := http.NewServeMux() + + // copy of controller-runtime sane defaults for new http.Server + s := &http.Server{ + Handler: mux, + MaxHeaderBytes: 1 << 20, + IdleTimeout: 90 * time.Second, // matches http.DefaultTransport keep-alive timeout + ReadHeaderTimeout: 32 * time.Second, + } + + mux.HandleFunc(readinessEndpointName, healthChecker.readyHandler) + + ln, err := net.Listen("tcp", fmt.Sprintf(":%d", cfg.HealthConfig.Port)) + if err != nil { + return manager.Server{}, + fmt.Errorf("error listening on %s: %w", fmt.Sprintf(":%d", cfg.HealthConfig.Port), err) + } + + return manager.Server{ + Name: "health probe", + Server: s, + Listener: ln, + }, nil +} + func (h *graphBuiltHealthChecker) readyHandler(resp http.ResponseWriter, req *http.Request) { if err := h.readyCheck(req); err != nil { resp.WriteHeader(http.StatusServiceUnavailable) diff --git a/internal/mode/static/manager.go b/internal/mode/static/manager.go index d03d3596a7..6bcc7fa309 100644 --- a/internal/mode/static/manager.go +++ b/internal/mode/static/manager.go @@ -3,8 +3,6 @@ package static import ( "context" "fmt" - "net" - "net/http" "time" "github.com/go-logr/logr" @@ -815,34 +813,3 @@ func getMetricsOptions(cfg config.MetricsConfig) metricsserver.Options { return metricsOptions } - -// createHealthProbe creates a Server runnable to serve as our health and readiness checker. -func createHealthProbe(cfg config.Config, healthChecker *graphBuiltHealthChecker) (manager.Server, error) { - // we chose to create our own health probe server instead of using the controller-runtime one because - // of an annoying log which would flood our logs on non-ready non-leader NGF Pods. This health probe is pretty - // similar to the controller-runtime's health probe. - - mux := http.NewServeMux() - - // copy of controller-runtime sane defaults for new http.Server - s := &http.Server{ - Handler: mux, - MaxHeaderBytes: 1 << 20, - IdleTimeout: 90 * time.Second, // matches http.DefaultTransport keep-alive timeout - ReadHeaderTimeout: 32 * time.Second, - } - - mux.HandleFunc(readinessEndpointName, healthChecker.readyHandler) - - ln, err := net.Listen("tcp", fmt.Sprintf(":%d", cfg.HealthConfig.Port)) - if err != nil { - return manager.Server{}, - fmt.Errorf("error listening on %s: %w", fmt.Sprintf(":%d", cfg.HealthConfig.Port), err) - } - - return manager.Server{ - Name: "health probe", - Server: s, - Listener: ln, - }, nil -} From 6f63196825cd748ee1af31b146fb2563c048bdb3 Mon Sep 17 00:00:00 2001 From: Benjamin Jee Date: Thu, 6 Feb 2025 13:44:45 -0800 Subject: [PATCH 04/12] Add health probe test --- internal/mode/static/health_test.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/internal/mode/static/health_test.go b/internal/mode/static/health_test.go index 2fcd511a63..5aa06bc769 100644 --- a/internal/mode/static/health_test.go +++ b/internal/mode/static/health_test.go @@ -2,11 +2,14 @@ package static import ( "errors" + "net" "net/http" "net/http/httptest" "testing" . "github.com/onsi/gomega" + + "github.com/nginx/nginx-gateway-fabric/internal/mode/static/config" ) func TestReadyCheck(t *testing.T) { @@ -60,3 +63,21 @@ func TestReadyHandler(t *testing.T) { healthChecker.readyHandler(w, r) g.Expect(w.Result().StatusCode).To(Equal(http.StatusOK)) } + +func TestCreateHealthProbe(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + healthChecker := newGraphBuiltHealthChecker() + + cfg := config.Config{HealthConfig: config.HealthConfig{Port: 8081}} + + hp, err := createHealthProbe(cfg, healthChecker) + g.Expect(err).ToNot(HaveOccurred()) + + addr, ok := (hp.Listener.Addr()).(*net.TCPAddr) + g.Expect(ok).To(BeTrue()) + + g.Expect(addr.Port).To(Equal(cfg.HealthConfig.Port)) + g.Expect(hp.Server).ToNot(BeNil()) +} From a4df94cbcda94120526b15a31603a7c3db23c662 Mon Sep 17 00:00:00 2001 From: Benjamin Jee Date: Thu, 6 Feb 2025 13:59:02 -0800 Subject: [PATCH 05/12] Add code coverage on health probe test --- internal/mode/static/health_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/mode/static/health_test.go b/internal/mode/static/health_test.go index 5aa06bc769..cbab061d0f 100644 --- a/internal/mode/static/health_test.go +++ b/internal/mode/static/health_test.go @@ -70,8 +70,11 @@ func TestCreateHealthProbe(t *testing.T) { healthChecker := newGraphBuiltHealthChecker() - cfg := config.Config{HealthConfig: config.HealthConfig{Port: 8081}} + cfg := config.Config{HealthConfig: config.HealthConfig{Port: 100000}} + _, err := createHealthProbe(cfg, healthChecker) + g.Expect(err).To(MatchError("error listening on :100000: listen tcp: address 100000: invalid port")) + cfg = config.Config{HealthConfig: config.HealthConfig{Port: 8081}} hp, err := createHealthProbe(cfg, healthChecker) g.Expect(err).ToNot(HaveOccurred()) From f8ee138b9ab5ee4f0f55edcb6df76aed3aecec0c Mon Sep 17 00:00:00 2001 From: Benjamin Jee Date: Fri, 7 Feb 2025 13:31:15 -0800 Subject: [PATCH 06/12] Refactor leader election to use runnable --- internal/framework/events/event.go | 4 --- internal/framework/runnables/runnables.go | 17 +++++++--- .../framework/runnables/runnables_test.go | 4 ++- internal/mode/static/handler.go | 34 +++++++++---------- internal/mode/static/handler_test.go | 16 ++------- internal/mode/static/health.go | 8 ++--- internal/mode/static/health_test.go | 5 ++- internal/mode/static/manager.go | 4 +-- 8 files changed, 40 insertions(+), 52 deletions(-) diff --git a/internal/framework/events/event.go b/internal/framework/events/event.go index 5dba6f2af2..2200c8023f 100644 --- a/internal/framework/events/event.go +++ b/internal/framework/events/event.go @@ -23,7 +23,3 @@ type DeleteEvent struct { // NamespacedName is the namespace & name of the deleted resource. NamespacedName types.NamespacedName } - -// NewLeaderEvent represents an NGF Pod becoming leader. This is used to trigger the event handler to batch process -// events and update nginx conf when no resource has been changed. -type NewLeaderEvent struct{} diff --git a/internal/framework/runnables/runnables.go b/internal/framework/runnables/runnables.go index 59da358278..f5e5c3f7b2 100644 --- a/internal/framework/runnables/runnables.go +++ b/internal/framework/runnables/runnables.go @@ -37,8 +37,9 @@ func (r *LeaderOrNonLeader) NeedLeaderElection() bool { // CallFunctionsAfterBecameLeader is a Runnable that will call the given functions when the current instance becomes // the leader. type CallFunctionsAfterBecameLeader struct { - enable func(context.Context) - leader func() + enable func(context.Context) + leader func() + eventHandlerEnable func(context.Context) } var ( @@ -47,16 +48,22 @@ var ( ) // NewCallFunctionsAfterBecameLeader creates a new CallFunctionsAfterBecameLeader Runnable. -func NewCallFunctionsAfterBecameLeader(enable func(context.Context), leader func()) *CallFunctionsAfterBecameLeader { +func NewCallFunctionsAfterBecameLeader( + enable func(context.Context), + leader func(), + eventHandlerEnable func(context.Context), +) *CallFunctionsAfterBecameLeader { return &CallFunctionsAfterBecameLeader{ - enable: enable, - leader: leader, + enable: enable, + leader: leader, + eventHandlerEnable: eventHandlerEnable, } } func (j *CallFunctionsAfterBecameLeader) Start(ctx context.Context) error { j.enable(ctx) j.leader() + j.eventHandlerEnable(ctx) return nil } diff --git a/internal/framework/runnables/runnables_test.go b/internal/framework/runnables/runnables_test.go index fe26f43115..4cbee2cdfa 100644 --- a/internal/framework/runnables/runnables_test.go +++ b/internal/framework/runnables/runnables_test.go @@ -27,19 +27,21 @@ func TestCallFunctionsAfterBecameLeader(t *testing.T) { t.Parallel() enabled := false leader := false + eventHandlerEnabled := false callFunctionsAfterBecameLeader := NewCallFunctionsAfterBecameLeader( func(_ context.Context) { enabled = true }, func() { leader = true }, + func(_ context.Context) { eventHandlerEnabled = true }, ) g := NewWithT(t) g.Expect(callFunctionsAfterBecameLeader.NeedLeaderElection()).To(BeTrue()) - g.Expect(enabled).To(BeFalse()) err := callFunctionsAfterBecameLeader.Start(context.Background()) g.Expect(err).ToNot(HaveOccurred()) g.Expect(enabled).To(BeTrue()) g.Expect(leader).To(BeTrue()) + g.Expect(eventHandlerEnabled).To(BeTrue()) } diff --git a/internal/mode/static/handler.go b/internal/mode/static/handler.go index 90578495b9..b8bf9ccba0 100644 --- a/internal/mode/static/handler.go +++ b/internal/mode/static/handler.go @@ -161,23 +161,15 @@ func (h *eventHandlerImpl) HandleEventBatch(ctx context.Context, logger logr.Log h.cfg.metricsCollector.ObserveLastEventBatchProcessTime(duration) }() - var newLeader bool for _, event := range batch { - switch event.(type) { - case *events.NewLeaderEvent: - newLeader = true - default: - h.parseAndCaptureEvent(ctx, logger, event) - } + h.parseAndCaptureEvent(ctx, logger, event) } changeType, gr := h.cfg.processor.Process() - // if there is a newLeader event in the EventBatch, we want to generate and update nginx conf, - // so regardless of what came back from Process(), we want to update the nginx conf with the latest graph - if newLeader { - changeType = state.ClusterStateChange - gr = h.cfg.processor.GetLatestGraph() + // Once we've processed resources on startup and built our first graph, mark the Pod as ready. + if !h.cfg.graphBuiltHealthChecker.ready { + h.cfg.graphBuiltHealthChecker.setAsReady() } // if this Pod is not the leader or does not have the leader lease yet, @@ -186,13 +178,19 @@ func (h *eventHandlerImpl) HandleEventBatch(ctx context.Context, logger logr.Log return } - // Once we've processed resources on startup and built our first graph, mark the Pod as ready. - if !h.cfg.graphBuiltHealthChecker.ready { - h.cfg.graphBuiltHealthChecker.setAsReady() - } + h.sendNginxConfig(ctx, logger, gr, changeType) +} + +func (h *eventHandlerImpl) eventHandlerEnable(ctx context.Context) { + h.sendNginxConfig(ctx, h.cfg.logger, h.cfg.processor.GetLatestGraph(), state.ClusterStateChange) +} - // TODO(sberman): hardcode this deployment name until we support provisioning data planes - // If no deployments exist, we should just return without doing anything. +func (h *eventHandlerImpl) sendNginxConfig( + ctx context.Context, + logger logr.Logger, + gr *graph.Graph, + changeType state.ChangeType, +) { deploymentName := types.NamespacedName{ Name: "tmp-nginx-deployment", Namespace: h.cfg.gatewayPodConfig.Namespace, diff --git a/internal/mode/static/handler_test.go b/internal/mode/static/handler_test.go index 1a6e5812e4..ccbd13b53a 100644 --- a/internal/mode/static/handler_test.go +++ b/internal/mode/static/handler_test.go @@ -195,17 +195,6 @@ var _ = Describe("eventHandler", func() { expectReconfig(dcfg, fakeCfgFiles) Expect(helpers.Diff(handler.GetLatestConfiguration(), &dcfg)).To(BeEmpty()) }) - - It("should process a NewLeaderEvent", func() { - e := &events.NewLeaderEvent{} - - batch := []interface{}{e} - - handler.HandleEventBatch(context.Background(), ctlrZap.New(), batch) - - dcfg := dataplane.GetDefaultConfiguration(&graph.Graph{}, 1) - Expect(helpers.Diff(handler.GetLatestConfiguration(), &dcfg)).To(BeEmpty()) - }) }) When("a batch has multiple events", func() { @@ -215,8 +204,7 @@ var _ = Describe("eventHandler", func() { Type: &gatewayv1.HTTPRoute{}, NamespacedName: types.NamespacedName{Namespace: "test", Name: "route"}, } - newLeaderEvent := &events.NewLeaderEvent{} - batch := []interface{}{upsertEvent, deleteEvent, newLeaderEvent} + batch := []interface{}{upsertEvent, deleteEvent} handler.HandleEventBatch(context.Background(), ctlrZap.New(), batch) @@ -511,6 +499,8 @@ var _ = Describe("eventHandler", func() { dcfg := dataplane.GetDefaultConfiguration(&graph.Graph{}, 1) Expect(helpers.Diff(handler.GetLatestConfiguration(), &dcfg)).To(BeEmpty()) + handler.cfg.graphBuiltHealthChecker.setAsLeader() + Expect(readyChannel).To(BeClosed()) Expect(handler.cfg.graphBuiltHealthChecker.readyCheck(nil)).To(Succeed()) diff --git a/internal/mode/static/health.go b/internal/mode/static/health.go index 311c025092..89838d6fd5 100644 --- a/internal/mode/static/health.go +++ b/internal/mode/static/health.go @@ -10,7 +10,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" - "github.com/nginx/nginx-gateway-fabric/internal/framework/events" "github.com/nginx/nginx-gateway-fabric/internal/mode/static/config" ) @@ -26,8 +25,6 @@ func newGraphBuiltHealthChecker() *graphBuiltHealthChecker { type graphBuiltHealthChecker struct { // readyCh is a channel that is initialized in newGraphBuiltHealthChecker and represents if the NGF Pod is ready. readyCh chan struct{} - // eventCh is a channel that a NewLeaderEvent gets sent to when the NGF Pod becomes leader. - eventCh chan interface{} lock sync.RWMutex ready bool leader bool @@ -95,7 +92,6 @@ func (h *graphBuiltHealthChecker) setAsReady() { defer h.lock.Unlock() h.ready = true - close(h.readyCh) } // getReadyCh returns a read-only channel, which determines if the NGF Pod is ready. @@ -109,5 +105,7 @@ func (h *graphBuiltHealthChecker) setAsLeader() { defer h.lock.Unlock() h.leader = true - h.eventCh <- &events.NewLeaderEvent{} + + // not sure where to close this, this is needed for the telemetry job, though it needs to be ready and to be leader + close(h.readyCh) } diff --git a/internal/mode/static/health_test.go b/internal/mode/static/health_test.go index cbab061d0f..da52ab6337 100644 --- a/internal/mode/static/health_test.go +++ b/internal/mode/static/health_test.go @@ -34,15 +34,14 @@ func TestSetAsLeader(t *testing.T) { t.Parallel() g := NewWithT(t) healthChecker := newGraphBuiltHealthChecker() - healthChecker.eventCh = make(chan interface{}, 1) g.Expect(healthChecker.leader).To(BeFalse()) - g.Expect(healthChecker.eventCh).ShouldNot(Receive()) + g.Expect(healthChecker.readyCh).ShouldNot(BeClosed()) healthChecker.setAsLeader() g.Expect(healthChecker.leader).To(BeTrue()) - g.Expect(healthChecker.eventCh).Should(Receive()) + g.Expect(healthChecker.readyCh).To(BeClosed()) } func TestReadyHandler(t *testing.T) { diff --git a/internal/mode/static/manager.go b/internal/mode/static/manager.go index 6bcc7fa309..8f4f04a5cb 100644 --- a/internal/mode/static/manager.go +++ b/internal/mode/static/manager.go @@ -247,12 +247,10 @@ func StartManager(cfg config.Config) error { return fmt.Errorf("cannot register event loop: %w", err) } - // the healthChecker needs the same eventCh as the event handler so it can send a NewLeaderEvent when - // the pod becomes leader, triggering HandleEventBatch to be called. - healthChecker.eventCh = eventCh if err = mgr.Add(runnables.NewCallFunctionsAfterBecameLeader( groupStatusUpdater.Enable, healthChecker.setAsLeader, + eventHandler.eventHandlerEnable, )); err != nil { return fmt.Errorf("cannot register status updater or set pod as leader: %w", err) } From 41cbf088e7c2ff319d9390bb926bab58e2886241 Mon Sep 17 00:00:00 2001 From: Benjamin Jee Date: Fri, 7 Feb 2025 13:41:58 -0800 Subject: [PATCH 07/12] Change some variable names and error messages --- internal/mode/static/handler.go | 6 +++--- internal/mode/static/handler_test.go | 4 ++-- internal/mode/static/health.go | 22 +++++++++++----------- internal/mode/static/health_test.go | 11 ++++++----- internal/mode/static/manager.go | 2 +- 5 files changed, 23 insertions(+), 22 deletions(-) diff --git a/internal/mode/static/handler.go b/internal/mode/static/handler.go index b8bf9ccba0..4c5d939724 100644 --- a/internal/mode/static/handler.go +++ b/internal/mode/static/handler.go @@ -167,9 +167,9 @@ func (h *eventHandlerImpl) HandleEventBatch(ctx context.Context, logger logr.Log changeType, gr := h.cfg.processor.Process() - // Once we've processed resources on startup and built our first graph, mark the Pod as ready. - if !h.cfg.graphBuiltHealthChecker.ready { - h.cfg.graphBuiltHealthChecker.setAsReady() + // Once we've processed resources on startup and built our first graph, mark the Pod as having built the graph. + if !h.cfg.graphBuiltHealthChecker.graphBuilt { + h.cfg.graphBuiltHealthChecker.setGraphBuilt() } // if this Pod is not the leader or does not have the leader lease yet, diff --git a/internal/mode/static/handler_test.go b/internal/mode/static/handler_test.go index ccbd13b53a..d83c18a967 100644 --- a/internal/mode/static/handler_test.go +++ b/internal/mode/static/handler_test.go @@ -126,7 +126,7 @@ var _ = Describe("eventHandler", func() { metricsCollector: collectors.NewControllerNoopCollector(), updateGatewayClassStatus: true, }) - Expect(handler.cfg.graphBuiltHealthChecker.ready).To(BeFalse()) + Expect(handler.cfg.graphBuiltHealthChecker.graphBuilt).To(BeFalse()) handler.cfg.graphBuiltHealthChecker.leader = true }) @@ -163,7 +163,7 @@ var _ = Describe("eventHandler", func() { }) AfterEach(func() { - Expect(handler.cfg.graphBuiltHealthChecker.ready).To(BeTrue()) + Expect(handler.cfg.graphBuiltHealthChecker.graphBuilt).To(BeTrue()) }) When("a batch has one event", func() { diff --git a/internal/mode/static/health.go b/internal/mode/static/health.go index 89838d6fd5..a88ea74b27 100644 --- a/internal/mode/static/health.go +++ b/internal/mode/static/health.go @@ -20,14 +20,14 @@ func newGraphBuiltHealthChecker() *graphBuiltHealthChecker { } } -// graphBuiltHealthChecker is used to check if the initial graph is built, if the NGF Pod is leader, and if the -// NGF Pod is ready. +// graphBuiltHealthChecker is used to check if the NGF Pod is ready. The NGF Pod is ready if the initial graph has +// been built and if it is leader. type graphBuiltHealthChecker struct { // readyCh is a channel that is initialized in newGraphBuiltHealthChecker and represents if the NGF Pod is ready. - readyCh chan struct{} - lock sync.RWMutex - ready bool - leader bool + readyCh chan struct{} + lock sync.RWMutex + graphBuilt bool + leader bool } // createHealthProbe creates a Server runnable to serve as our health and readiness checker. @@ -79,19 +79,19 @@ func (h *graphBuiltHealthChecker) readyCheck(_ *http.Request) error { return errors.New("this NGF Pod is not currently leader") } - if !h.ready { - return errors.New("control plane is not yet ready") + if !h.graphBuilt { + return errors.New("control plane initial graph has not been built") } return nil } -// setAsReady marks the health check as ready. -func (h *graphBuiltHealthChecker) setAsReady() { +// setGraphBuilt marks the health check as having the initial graph built. +func (h *graphBuiltHealthChecker) setGraphBuilt() { h.lock.Lock() defer h.lock.Unlock() - h.ready = true + h.graphBuilt = true } // getReadyCh returns a read-only channel, which determines if the NGF Pod is ready. diff --git a/internal/mode/static/health_test.go b/internal/mode/static/health_test.go index da52ab6337..c6b8e1a333 100644 --- a/internal/mode/static/health_test.go +++ b/internal/mode/static/health_test.go @@ -19,14 +19,15 @@ func TestReadyCheck(t *testing.T) { g.Expect(healthChecker.readyCheck(nil)).To(MatchError(errors.New("this NGF Pod is not currently leader"))) - healthChecker.ready = true + healthChecker.graphBuilt = true g.Expect(healthChecker.readyCheck(nil)).To(MatchError(errors.New("this NGF Pod is not currently leader"))) - healthChecker.ready = false + healthChecker.graphBuilt = false healthChecker.leader = true - g.Expect(healthChecker.readyCheck(nil)).To(MatchError(errors.New("control plane is not yet ready"))) + g.Expect(healthChecker.readyCheck(nil)). + To(MatchError(errors.New("control plane initial graph has not been built"))) - healthChecker.ready = true + healthChecker.graphBuilt = true g.Expect(healthChecker.readyCheck(nil)).To(Succeed()) } @@ -55,7 +56,7 @@ func TestReadyHandler(t *testing.T) { healthChecker.readyHandler(w, r) g.Expect(w.Result().StatusCode).To(Equal(http.StatusServiceUnavailable)) - healthChecker.ready = true + healthChecker.graphBuilt = true healthChecker.leader = true w = httptest.NewRecorder() diff --git a/internal/mode/static/manager.go b/internal/mode/static/manager.go index 8f4f04a5cb..b5ea3e560b 100644 --- a/internal/mode/static/manager.go +++ b/internal/mode/static/manager.go @@ -252,7 +252,7 @@ func StartManager(cfg config.Config) error { healthChecker.setAsLeader, eventHandler.eventHandlerEnable, )); err != nil { - return fmt.Errorf("cannot register status updater or set pod as leader: %w", err) + return fmt.Errorf("cannot register functions that get called after Pod becomes leader: %w", err) } if cfg.ProductTelemetryConfig.Enabled { From f1c038906b7acfb006bd17813d33ebdb8e5a942c Mon Sep 17 00:00:00 2001 From: Benjamin Jee Date: Fri, 7 Feb 2025 14:24:41 -0800 Subject: [PATCH 08/12] Add comments and more changes to variable names --- internal/framework/runnables/runnables.go | 20 +++++++++---------- .../framework/runnables/runnables_test.go | 12 +++++------ internal/mode/static/handler.go | 2 ++ internal/mode/static/health.go | 6 ++++-- internal/mode/static/health_test.go | 12 +++++++++++ 5 files changed, 34 insertions(+), 18 deletions(-) diff --git a/internal/framework/runnables/runnables.go b/internal/framework/runnables/runnables.go index f5e5c3f7b2..294cf829eb 100644 --- a/internal/framework/runnables/runnables.go +++ b/internal/framework/runnables/runnables.go @@ -37,9 +37,9 @@ func (r *LeaderOrNonLeader) NeedLeaderElection() bool { // CallFunctionsAfterBecameLeader is a Runnable that will call the given functions when the current instance becomes // the leader. type CallFunctionsAfterBecameLeader struct { - enable func(context.Context) - leader func() - eventHandlerEnable func(context.Context) + statusUpdaterEnable func(context.Context) + healthCheckEnableLeader func() + eventHandlerEnable func(context.Context) } var ( @@ -49,20 +49,20 @@ var ( // NewCallFunctionsAfterBecameLeader creates a new CallFunctionsAfterBecameLeader Runnable. func NewCallFunctionsAfterBecameLeader( - enable func(context.Context), - leader func(), + statusUpdaterEnable func(context.Context), + healthCheckEnableLeader func(), eventHandlerEnable func(context.Context), ) *CallFunctionsAfterBecameLeader { return &CallFunctionsAfterBecameLeader{ - enable: enable, - leader: leader, - eventHandlerEnable: eventHandlerEnable, + statusUpdaterEnable: statusUpdaterEnable, + healthCheckEnableLeader: healthCheckEnableLeader, + eventHandlerEnable: eventHandlerEnable, } } func (j *CallFunctionsAfterBecameLeader) Start(ctx context.Context) error { - j.enable(ctx) - j.leader() + j.statusUpdaterEnable(ctx) + j.healthCheckEnableLeader() j.eventHandlerEnable(ctx) return nil } diff --git a/internal/framework/runnables/runnables_test.go b/internal/framework/runnables/runnables_test.go index 4cbee2cdfa..c4fad7d561 100644 --- a/internal/framework/runnables/runnables_test.go +++ b/internal/framework/runnables/runnables_test.go @@ -25,13 +25,13 @@ func TestLeaderOrNonLeader(t *testing.T) { func TestCallFunctionsAfterBecameLeader(t *testing.T) { t.Parallel() - enabled := false - leader := false + statusUpdaterEnabled := false + healthCheckEnableLeader := false eventHandlerEnabled := false callFunctionsAfterBecameLeader := NewCallFunctionsAfterBecameLeader( - func(_ context.Context) { enabled = true }, - func() { leader = true }, + func(_ context.Context) { statusUpdaterEnabled = true }, + func() { healthCheckEnableLeader = true }, func(_ context.Context) { eventHandlerEnabled = true }, ) @@ -41,7 +41,7 @@ func TestCallFunctionsAfterBecameLeader(t *testing.T) { err := callFunctionsAfterBecameLeader.Start(context.Background()) g.Expect(err).ToNot(HaveOccurred()) - g.Expect(enabled).To(BeTrue()) - g.Expect(leader).To(BeTrue()) + g.Expect(statusUpdaterEnabled).To(BeTrue()) + g.Expect(healthCheckEnableLeader).To(BeTrue()) g.Expect(eventHandlerEnabled).To(BeTrue()) } diff --git a/internal/mode/static/handler.go b/internal/mode/static/handler.go index 4c5d939724..c26d20b5a7 100644 --- a/internal/mode/static/handler.go +++ b/internal/mode/static/handler.go @@ -182,6 +182,8 @@ func (h *eventHandlerImpl) HandleEventBatch(ctx context.Context, logger logr.Log } func (h *eventHandlerImpl) eventHandlerEnable(ctx context.Context) { + // Latest graph is guaranteed to not be nil since the leader election process takes longer than + // the initial call to HandleEventBatch when NGF starts up. h.sendNginxConfig(ctx, h.cfg.logger, h.cfg.processor.GetLatestGraph(), state.ClusterStateChange) } diff --git a/internal/mode/static/health.go b/internal/mode/static/health.go index a88ea74b27..a17ddd691a 100644 --- a/internal/mode/static/health.go +++ b/internal/mode/static/health.go @@ -99,13 +99,15 @@ func (h *graphBuiltHealthChecker) getReadyCh() <-chan struct{} { return h.readyCh } -// setAsLeader marks the health check as leader and sends an empty event to the event channel. +// setAsLeader marks the health check as leader. func (h *graphBuiltHealthChecker) setAsLeader() { h.lock.Lock() defer h.lock.Unlock() h.leader = true - // not sure where to close this, this is needed for the telemetry job, though it needs to be ready and to be leader + // setGraphBuilt should already have been called when processing the resources on startup because the leader + // election process takes longer than the initial call to HandleEventBatch. Thus, the NGF Pod should be marked as + // ready and have this channel be closed. close(h.readyCh) } diff --git a/internal/mode/static/health_test.go b/internal/mode/static/health_test.go index c6b8e1a333..6de27e51dd 100644 --- a/internal/mode/static/health_test.go +++ b/internal/mode/static/health_test.go @@ -45,6 +45,18 @@ func TestSetAsLeader(t *testing.T) { g.Expect(healthChecker.readyCh).To(BeClosed()) } +func TestSetGraphBuilt(t *testing.T) { + t.Parallel() + g := NewWithT(t) + healthChecker := newGraphBuiltHealthChecker() + + g.Expect(healthChecker.graphBuilt).To(BeFalse()) + + healthChecker.setGraphBuilt() + + g.Expect(healthChecker.graphBuilt).To(BeTrue()) +} + func TestReadyHandler(t *testing.T) { t.Parallel() g := NewWithT(t) From 461bec38eb391c4796cbdb2802f2af0c84345e4e Mon Sep 17 00:00:00 2001 From: Benjamin Jee Date: Fri, 7 Feb 2025 15:10:17 -0800 Subject: [PATCH 09/12] Add another handler test --- internal/mode/static/handler_test.go | 40 +++++++++++----------------- 1 file changed, 16 insertions(+), 24 deletions(-) diff --git a/internal/mode/static/handler_test.go b/internal/mode/static/handler_test.go index d83c18a967..6f3d59f2f1 100644 --- a/internal/mode/static/handler_test.go +++ b/internal/mode/static/handler_test.go @@ -486,43 +486,35 @@ var _ = Describe("eventHandler", func() { Expect(gr.LatestReloadResult.Error.Error()).To(Equal("status error")) }) - It("should set the health checker status properly", func() { + It("should update nginx conf only when leader", func() { + handler.cfg.graphBuiltHealthChecker.leader = false + e := &events.UpsertEvent{Resource: &gatewayv1.HTTPRoute{}} batch := []interface{}{e} readyChannel := handler.cfg.graphBuiltHealthChecker.getReadyCh() fakeProcessor.ProcessReturns(state.ClusterStateChange, &graph.Graph{}) - Expect(handler.cfg.graphBuiltHealthChecker.readyCheck(nil)).ToNot(Succeed()) handler.HandleEventBatch(context.Background(), ctlrZap.New(), batch) - dcfg := dataplane.GetDefaultConfiguration(&graph.Graph{}, 1) - Expect(helpers.Diff(handler.GetLatestConfiguration(), &dcfg)).To(BeEmpty()) + // graph is built, but since the graphBuiltHealthChecker.leader is false, configuration isn't created and + // the readyCheck fails + Expect(handler.cfg.graphBuiltHealthChecker.graphBuilt).To(BeTrue()) + Expect(handler.GetLatestConfiguration()).To(BeNil()) + Expect(handler.cfg.graphBuiltHealthChecker.readyCheck(nil)).ToNot(Succeed()) + Expect(readyChannel).ShouldNot(BeClosed()) + // Once the pod becomes leader, these two functions will be called through the runnables we set in the manager handler.cfg.graphBuiltHealthChecker.setAsLeader() + handler.eventHandlerEnable(context.Background()) - Expect(readyChannel).To(BeClosed()) + // nginx conf has been set + dcfg := dataplane.GetDefaultConfiguration(&graph.Graph{}, 1) + Expect(helpers.Diff(handler.GetLatestConfiguration(), &dcfg)).To(BeEmpty()) + // ready check is also set Expect(handler.cfg.graphBuiltHealthChecker.readyCheck(nil)).To(Succeed()) - }) - - It("should not update nginx conf if NGF is not leader", func() { - e := &events.UpsertEvent{Resource: &gatewayv1.HTTPRoute{}} - batch := []interface{}{e} - readyChannel := handler.cfg.graphBuiltHealthChecker.getReadyCh() - - fakeProcessor.ProcessReturns(state.ClusterStateChange, &graph.Graph{}) - - handler.cfg.graphBuiltHealthChecker.leader = false - - Expect(handler.cfg.graphBuiltHealthChecker.readyCheck(nil)).ToNot(Succeed()) - handler.HandleEventBatch(context.Background(), ctlrZap.New(), batch) - - Expect(handler.GetLatestConfiguration()).To(BeNil()) - - Expect(readyChannel).ShouldNot(BeClosed()) - - Expect(handler.cfg.graphBuiltHealthChecker.readyCheck(nil)).ToNot(Succeed()) + Expect(handler.cfg.graphBuiltHealthChecker.getReadyCh()).To(BeClosed()) }) It("should panic for an unknown event type", func() { From 970cd679c83c32090eafc4a1e420bd73fe106ebe Mon Sep 17 00:00:00 2001 From: Benjamin Jee Date: Mon, 10 Feb 2025 10:45:30 -0800 Subject: [PATCH 10/12] Add adjustments to comments and logs --- internal/mode/static/handler.go | 3 ++- internal/mode/static/health.go | 4 ++-- internal/mode/static/manager.go | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/internal/mode/static/handler.go b/internal/mode/static/handler.go index c26d20b5a7..84a658ae4e 100644 --- a/internal/mode/static/handler.go +++ b/internal/mode/static/handler.go @@ -183,7 +183,8 @@ func (h *eventHandlerImpl) HandleEventBatch(ctx context.Context, logger logr.Log func (h *eventHandlerImpl) eventHandlerEnable(ctx context.Context) { // Latest graph is guaranteed to not be nil since the leader election process takes longer than - // the initial call to HandleEventBatch when NGF starts up. + // the initial call to HandleEventBatch when NGF starts up. And GatewayClass will typically always exist which + // triggers an event. h.sendNginxConfig(ctx, h.cfg.logger, h.cfg.processor.GetLatestGraph(), state.ClusterStateChange) } diff --git a/internal/mode/static/health.go b/internal/mode/static/health.go index a17ddd691a..98523d0609 100644 --- a/internal/mode/static/health.go +++ b/internal/mode/static/health.go @@ -33,7 +33,7 @@ type graphBuiltHealthChecker struct { // createHealthProbe creates a Server runnable to serve as our health and readiness checker. func createHealthProbe(cfg config.Config, healthChecker *graphBuiltHealthChecker) (manager.Server, error) { // we chose to create our own health probe server instead of using the controller-runtime one because - // of an annoying log which would flood our logs on non-ready non-leader NGF Pods. This health probe is pretty + // of repetitive log which would flood our logs on non-ready non-leader NGF Pods. This health probe is // similar to the controller-runtime's health probe. mux := http.NewServeMux() @@ -76,7 +76,7 @@ func (h *graphBuiltHealthChecker) readyCheck(_ *http.Request) error { defer h.lock.RUnlock() if !h.leader { - return errors.New("this NGF Pod is not currently leader") + return errors.New("this Pod is not currently leader") } if !h.graphBuilt { diff --git a/internal/mode/static/manager.go b/internal/mode/static/manager.go index b5ea3e560b..27bbb6a612 100644 --- a/internal/mode/static/manager.go +++ b/internal/mode/static/manager.go @@ -280,7 +280,7 @@ func StartManager(cfg config.Config) error { } cfg.Logger.Info("Starting manager") - cfg.Logger.Info("Nginx Gateway Fabric Pod will be marked as unready until it has the leader lease") + cfg.Logger.Info("NGINX Gateway Fabric Pod will be marked as unready until it has the leader lease") go func() { <-ctx.Done() cfg.Logger.Info("Shutting down") From fec9a747c52d6ee133c9b2890071bd4dc7045fa4 Mon Sep 17 00:00:00 2001 From: Benjamin Jee Date: Mon, 10 Feb 2025 11:41:19 -0800 Subject: [PATCH 11/12] Refactor runnable CallFunctionsAfterBecameLeader to take list of functions --- internal/framework/runnables/runnables.go | 18 ++++++------------ internal/framework/runnables/runnables_test.go | 6 +++--- internal/mode/static/handler_test.go | 5 +++-- internal/mode/static/health.go | 3 ++- internal/mode/static/health_test.go | 7 ++++--- internal/mode/static/manager.go | 4 ++-- 6 files changed, 20 insertions(+), 23 deletions(-) diff --git a/internal/framework/runnables/runnables.go b/internal/framework/runnables/runnables.go index 294cf829eb..483f7accbb 100644 --- a/internal/framework/runnables/runnables.go +++ b/internal/framework/runnables/runnables.go @@ -37,9 +37,7 @@ func (r *LeaderOrNonLeader) NeedLeaderElection() bool { // CallFunctionsAfterBecameLeader is a Runnable that will call the given functions when the current instance becomes // the leader. type CallFunctionsAfterBecameLeader struct { - statusUpdaterEnable func(context.Context) - healthCheckEnableLeader func() - eventHandlerEnable func(context.Context) + enableFunctions []func(context.Context) } var ( @@ -49,21 +47,17 @@ var ( // NewCallFunctionsAfterBecameLeader creates a new CallFunctionsAfterBecameLeader Runnable. func NewCallFunctionsAfterBecameLeader( - statusUpdaterEnable func(context.Context), - healthCheckEnableLeader func(), - eventHandlerEnable func(context.Context), + enableFunctions []func(context.Context), ) *CallFunctionsAfterBecameLeader { return &CallFunctionsAfterBecameLeader{ - statusUpdaterEnable: statusUpdaterEnable, - healthCheckEnableLeader: healthCheckEnableLeader, - eventHandlerEnable: eventHandlerEnable, + enableFunctions: enableFunctions, } } func (j *CallFunctionsAfterBecameLeader) Start(ctx context.Context) error { - j.statusUpdaterEnable(ctx) - j.healthCheckEnableLeader() - j.eventHandlerEnable(ctx) + for _, function := range j.enableFunctions { + function(ctx) + } return nil } diff --git a/internal/framework/runnables/runnables_test.go b/internal/framework/runnables/runnables_test.go index c4fad7d561..7a9b8968ba 100644 --- a/internal/framework/runnables/runnables_test.go +++ b/internal/framework/runnables/runnables_test.go @@ -29,11 +29,11 @@ func TestCallFunctionsAfterBecameLeader(t *testing.T) { healthCheckEnableLeader := false eventHandlerEnabled := false - callFunctionsAfterBecameLeader := NewCallFunctionsAfterBecameLeader( + callFunctionsAfterBecameLeader := NewCallFunctionsAfterBecameLeader([]func(ctx context.Context){ func(_ context.Context) { statusUpdaterEnabled = true }, - func() { healthCheckEnableLeader = true }, + func(_ context.Context) { healthCheckEnableLeader = true }, func(_ context.Context) { eventHandlerEnabled = true }, - ) + }) g := NewWithT(t) g.Expect(callFunctionsAfterBecameLeader.NeedLeaderElection()).To(BeTrue()) diff --git a/internal/mode/static/handler_test.go b/internal/mode/static/handler_test.go index 6f3d59f2f1..ed6414731c 100644 --- a/internal/mode/static/handler_test.go +++ b/internal/mode/static/handler_test.go @@ -487,6 +487,7 @@ var _ = Describe("eventHandler", func() { }) It("should update nginx conf only when leader", func() { + ctx := context.Background() handler.cfg.graphBuiltHealthChecker.leader = false e := &events.UpsertEvent{Resource: &gatewayv1.HTTPRoute{}} @@ -505,8 +506,8 @@ var _ = Describe("eventHandler", func() { Expect(readyChannel).ShouldNot(BeClosed()) // Once the pod becomes leader, these two functions will be called through the runnables we set in the manager - handler.cfg.graphBuiltHealthChecker.setAsLeader() - handler.eventHandlerEnable(context.Background()) + handler.cfg.graphBuiltHealthChecker.setAsLeader(ctx) + handler.eventHandlerEnable(ctx) // nginx conf has been set dcfg := dataplane.GetDefaultConfiguration(&graph.Graph{}, 1) diff --git a/internal/mode/static/health.go b/internal/mode/static/health.go index 98523d0609..4993b0b40e 100644 --- a/internal/mode/static/health.go +++ b/internal/mode/static/health.go @@ -1,6 +1,7 @@ package static import ( + "context" "errors" "fmt" "net" @@ -100,7 +101,7 @@ func (h *graphBuiltHealthChecker) getReadyCh() <-chan struct{} { } // setAsLeader marks the health check as leader. -func (h *graphBuiltHealthChecker) setAsLeader() { +func (h *graphBuiltHealthChecker) setAsLeader(_ context.Context) { h.lock.Lock() defer h.lock.Unlock() diff --git a/internal/mode/static/health_test.go b/internal/mode/static/health_test.go index 6de27e51dd..3505479d7d 100644 --- a/internal/mode/static/health_test.go +++ b/internal/mode/static/health_test.go @@ -1,6 +1,7 @@ package static import ( + "context" "errors" "net" "net/http" @@ -17,10 +18,10 @@ func TestReadyCheck(t *testing.T) { g := NewWithT(t) healthChecker := newGraphBuiltHealthChecker() - g.Expect(healthChecker.readyCheck(nil)).To(MatchError(errors.New("this NGF Pod is not currently leader"))) + g.Expect(healthChecker.readyCheck(nil)).To(MatchError(errors.New("this Pod is not currently leader"))) healthChecker.graphBuilt = true - g.Expect(healthChecker.readyCheck(nil)).To(MatchError(errors.New("this NGF Pod is not currently leader"))) + g.Expect(healthChecker.readyCheck(nil)).To(MatchError(errors.New("this Pod is not currently leader"))) healthChecker.graphBuilt = false healthChecker.leader = true @@ -39,7 +40,7 @@ func TestSetAsLeader(t *testing.T) { g.Expect(healthChecker.leader).To(BeFalse()) g.Expect(healthChecker.readyCh).ShouldNot(BeClosed()) - healthChecker.setAsLeader() + healthChecker.setAsLeader(context.Background()) g.Expect(healthChecker.leader).To(BeTrue()) g.Expect(healthChecker.readyCh).To(BeClosed()) diff --git a/internal/mode/static/manager.go b/internal/mode/static/manager.go index 27bbb6a612..31574a9f64 100644 --- a/internal/mode/static/manager.go +++ b/internal/mode/static/manager.go @@ -247,11 +247,11 @@ func StartManager(cfg config.Config) error { return fmt.Errorf("cannot register event loop: %w", err) } - if err = mgr.Add(runnables.NewCallFunctionsAfterBecameLeader( + if err = mgr.Add(runnables.NewCallFunctionsAfterBecameLeader([]func(context.Context){ groupStatusUpdater.Enable, healthChecker.setAsLeader, eventHandler.eventHandlerEnable, - )); err != nil { + })); err != nil { return fmt.Errorf("cannot register functions that get called after Pod becomes leader: %w", err) } From 85d04eb8e1ae44e45c3fe087578a449e57cf5b2a Mon Sep 17 00:00:00 2001 From: Benjamin Jee Date: Mon, 10 Feb 2025 12:06:10 -0800 Subject: [PATCH 12/12] Use better naming conventions --- internal/framework/runnables/runnables.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/framework/runnables/runnables.go b/internal/framework/runnables/runnables.go index 483f7accbb..4c8aac5460 100644 --- a/internal/framework/runnables/runnables.go +++ b/internal/framework/runnables/runnables.go @@ -55,8 +55,8 @@ func NewCallFunctionsAfterBecameLeader( } func (j *CallFunctionsAfterBecameLeader) Start(ctx context.Context) error { - for _, function := range j.enableFunctions { - function(ctx) + for _, f := range j.enableFunctions { + f(ctx) } return nil }