From b78d570bb9455ec0f6f91c125912e996ae05d759 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ole=20Sch=C3=B6nburg?= Date: Mon, 18 Dec 2023 16:00:33 +0100 Subject: [PATCH 1/3] feat(blog): add Ole as author --- chaos-days/blog/authors.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/chaos-days/blog/authors.yml b/chaos-days/blog/authors.yml index 2a3ae7305..370ee33d8 100644 --- a/chaos-days/blog/authors.yml +++ b/chaos-days/blog/authors.yml @@ -7,4 +7,9 @@ nicolas: name: Nicolas Pepin-Perreault title: Senior Software Engineer @ Zeebe url: https://github.com/npepinpe - image_url: https://github.com/npepinpe.png \ No newline at end of file + image_url: https://github.com/npepinpe.png +ole: + name: Ole Schönburg + title: Senior Software Engineer @ Zeebe + url: https://github.com/oleschoenburg + image_url: https://github.com/oleschoenburg.png From d97f4ede5ff2008522034b572ef27c3b7b1ea545 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ole=20Sch=C3=B6nburg?= Date: Mon, 18 Dec 2023 17:49:14 +0100 Subject: [PATCH 2/3] feat(blog): scaling while brokers are restarting --- .../index.md | 697 ++++++++++++++++++ .../scaledown-completed.png | Bin 0 -> 24996 bytes .../scaleup-completed.png | Bin 0 -> 26096 bytes 3 files changed, 697 insertions(+) create mode 100644 chaos-days/blog/2023-12-18-Dynamically-scaling-brokers/index.md create mode 100644 chaos-days/blog/2023-12-18-Dynamically-scaling-brokers/scaledown-completed.png create mode 100644 chaos-days/blog/2023-12-18-Dynamically-scaling-brokers/scaleup-completed.png diff --git a/chaos-days/blog/2023-12-18-Dynamically-scaling-brokers/index.md b/chaos-days/blog/2023-12-18-Dynamically-scaling-brokers/index.md new file mode 100644 index 000000000..85166222c --- /dev/null +++ b/chaos-days/blog/2023-12-18-Dynamically-scaling-brokers/index.md @@ -0,0 +1,697 @@ +--- +layout: posts +title: "Dynamically scaling brokers" +date: 2023-12-18 +categories: + - chaos_experiment + - bpmn +tags: + - availability + - performance +authors: ole +--- + +# Chaos Day Summary + +We experimented with the first version of dynamic scaling in Zeebe, adding or removing brokers for a running cluster. + +**TL;DR;** Both scaling up and down is resilient to broker restarts, with the only effect that the operation takes longer than usual to complete. + + + +## Scaling up should be resilient to broker restarts + +Scaling up is a high-level operation that consists of many steps that need to be carried out by all brokers in the cluster. +New brokers need to join the replication group of the assigned partitions and then catch up with the leader. + +### Expected + +Even when brokers are restarting, the scale operation should eventually succeed with the expected cluster topology as result. + +### Actual + +#### Verify steady state + +The current cluster topology queried with `zbchaos cluster status` shows 6 partitions with 3 replicas each, evenly distributed across the 3 brokers. + +```json +{ + "Version": 13, + "Brokers": [ + { + "Id": 1, + "State": "ACTIVE", + "Version": 40, + "LastUpdatedAt": "2023-12-18T11:16:56.452114035Z", + "Partitions": [ + { + "Id": 1, + "State": "ACTIVE", + "Priority": 2 + }, + { + "Id": 2, + "State": "ACTIVE", + "Priority": 3 + }, + { + "Id": 3, + "State": "ACTIVE", + "Priority": 1 + }, + { + "Id": 4, + "State": "ACTIVE", + "Priority": 1 + }, + { + "Id": 5, + "State": "ACTIVE", + "Priority": 3 + }, + { + "Id": 6, + "State": "ACTIVE", + "Priority": 2 + } + ] + }, + { + "Id": 2, + "State": "ACTIVE", + "Version": 34, + "LastUpdatedAt": "2023-12-18T11:16:35.399362365Z", + "Partitions": [ + { + "Id": 1, + "State": "ACTIVE", + "Priority": 1 + }, + { + "Id": 2, + "State": "ACTIVE", + "Priority": 2 + }, + { + "Id": 3, + "State": "ACTIVE", + "Priority": 3 + }, + { + "Id": 4, + "State": "ACTIVE", + "Priority": 2 + }, + { + "Id": 5, + "State": "ACTIVE", + "Priority": 1 + }, + { + "Id": 6, + "State": "ACTIVE", + "Priority": 3 + } + ] + }, + { + "Id": 0, + "State": "ACTIVE", + "Version": 44, + "LastUpdatedAt": "2023-12-18T11:17:22.000373993Z", + "Partitions": [ + { + "Id": 1, + "State": "ACTIVE", + "Priority": 3 + }, + { + "Id": 2, + "State": "ACTIVE", + "Priority": 1 + }, + { + "Id": 3, + "State": "ACTIVE", + "Priority": 2 + }, + { + "Id": 4, + "State": "ACTIVE", + "Priority": 3 + }, + { + "Id": 5, + "State": "ACTIVE", + "Priority": 2 + }, + { + "Id": 6, + "State": "ACTIVE", + "Priority": 1 + } + ] + } + ], + "LastChange": { + "Id": 12, + "Status": "COMPLETED", + "StartedAt": "2023-12-18T11:13:47.548696276Z", + "CompletedAt": "2023-12-18T11:17:44.207162216Z" + }, + "PendingChange": null +} +``` + +All partitions are reported as healthy and leadership is balanced:: + +```bash +$ zbchaos topology +Node |Partition 1 |Partition 2 |Partition 3 |Partition 4 |Partition 5 |Partition 6 +0 |LEADER (HEALTHY) |FOLLOWER (HEALTHY) |FOLLOWER (HEALTHY) |LEADER (HEALTHY) |FOLLOWER (HEALTHY) |FOLLOWER (HEALTHY) +1 |FOLLOWER (HEALTHY) |LEADER (HEALTHY) |FOLLOWER (HEALTHY) |FOLLOWER (HEALTHY) |LEADER (HEALTHY) |FOLLOWER (HEALTHY) +2 |FOLLOWER (HEALTHY) |FOLLOWER (HEALTHY) |LEADER (HEALTHY) |FOLLOWER (HEALTHY) |FOLLOWER (HEALTHY) |LEADER (HEALTHY) +``` + +#### Scaling up with broker restarts + +We start the scaling with `zbchaos cluster scale --brokers 6` and restart a random broker every 30 seconds: + +```bash +$ zbchaos cluster scale --brokers 6 & +$ while true; do sleep 30; zbchaos restart broker --nodeId $(shuf -i 0-5 -n 1); done +``` + +After the scaling completed, we stop the restarting and let the cluste settle again for a few minutes. + +### Result + +The scale operation succeeds and the newly reported cluster topology shows us 6 partitions with 3 replicas each, evenly distributed across 6 instead of 3 brokers: + +```json +{ + "Version": 15, + "Brokers": [ + { + "Id": 1, + "State": "ACTIVE", + "Version": 47, + "LastUpdatedAt": "2023-12-18T15:30:19.184655204Z", + "Partitions": [ + { + "Id": 1, + "State": "ACTIVE", + "Priority": 2 + }, + { + "Id": 2, + "State": "ACTIVE", + "Priority": 3 + }, + { + "Id": 6, + "State": "ACTIVE", + "Priority": 1 + } + ] + }, + { + "Id": 2, + "State": "ACTIVE", + "Version": 40, + "LastUpdatedAt": "2023-12-18T15:30:19.111616286Z", + "Partitions": [ + { + "Id": 1, + "State": "ACTIVE", + "Priority": 1 + }, + { + "Id": 2, + "State": "ACTIVE", + "Priority": 2 + }, + { + "Id": 3, + "State": "ACTIVE", + "Priority": 3 + } + ] + }, + { + "Id": 3, + "State": "ACTIVE", + "Version": 8, + "LastUpdatedAt": "2023-12-18T15:25:07.484956052Z", + "Partitions": [ + { + "Id": 2, + "State": "ACTIVE", + "Priority": 1 + }, + { + "Id": 3, + "State": "ACTIVE", + "Priority": 2 + }, + { + "Id": 4, + "State": "ACTIVE", + "Priority": 3 + } + ] + }, + { + "Id": 4, + "State": "ACTIVE", + "Version": 8, + "LastUpdatedAt": "2023-12-18T15:25:50.616089815Z", + "Partitions": [ + { + "Id": 3, + "State": "ACTIVE", + "Priority": 1 + }, + { + "Id": 4, + "State": "ACTIVE", + "Priority": 2 + }, + { + "Id": 5, + "State": "ACTIVE", + "Priority": 3 + } + ] + }, + { + "Id": 5, + "State": "ACTIVE", + "Version": 8, + "LastUpdatedAt": "2023-12-18T15:30:16.949863252Z", + "Partitions": [ + { + "Id": 4, + "State": "ACTIVE", + "Priority": 1 + }, + { + "Id": 5, + "State": "ACTIVE", + "Priority": 2 + }, + { + "Id": 6, + "State": "ACTIVE", + "Priority": 3 + } + ] + }, + { + "Id": 0, + "State": "ACTIVE", + "Version": 52, + "LastUpdatedAt": "2023-12-18T15:30:20.920293674Z", + "Partitions": [ + { + "Id": 1, + "State": "ACTIVE", + "Priority": 3 + }, + { + "Id": 5, + "State": "ACTIVE", + "Priority": 1 + }, + { + "Id": 6, + "State": "ACTIVE", + "Priority": 2 + } + ] + } + ], + "LastChange": { + "Id": 14, + "Status": "COMPLETED", + "StartedAt": "2023-12-18T15:12:57.790824149Z", + "CompletedAt": "2023-12-18T15:30:20.920657536Z" + }, + "PendingChange": null +} +``` + +All partitions are reported as healthy and leadership is balanced: + +```bash +$ zbchaos topology +Node |Partition 1 |Partition 2 |Partition 3 |Partition 4 |Partition 5 |Partition 6 +0 |LEADER (HEALTHY) | | | |FOLLOWER (HEALTHY) |FOLLOWER (HEALTHY) +1 |FOLLOWER (HEALTHY) |LEADER (HEALTHY) | | | |FOLLOWER (HEALTHY) +2 |FOLLOWER (HEALTHY) |FOLLOWER (HEALTHY) |LEADER (HEALTHY) | | | +3 | |FOLLOWER (HEALTHY) |FOLLOWER (HEALTHY) |LEADER (HEALTHY) | | +4 | | |FOLLOWER (HEALTHY) |FOLLOWER (HEALTHY) |LEADER (HEALTHY) | +5 | | | |FOLLOWER (HEALTHY) |FOLLOWER (HEALTHY) |LEADER (HEALTHY) +``` + +The operation succeeded in about 17 minutes, longer than usual because of the restarts: +![](scaleup-completed.png) + + +## Scaling down should be resilient to broker restarts + +Exactly like scaling up, scaling down is also a high-level operation that consists of many steps that need to be carried out by all brokers in the cluster. +Before a broker can leave, another broker first needs to join the replication group to ensure that we maintain a replication factor of 3 at all times. + +### Expected + +Even when brokers are restarting, the scale operation should eventually succeed with the expected cluster topology as result. + +### Actual + +#### Verify steady state + +We start with the cluster topology that we got as result of the previous experiment. +6 partitions with 3 replicas distributed over 6 brokers: + +```json +{ + "Version": 15, + "Brokers": [ + { + "Id": 1, + "State": "ACTIVE", + "Version": 47, + "LastUpdatedAt": "2023-12-18T15:30:19.184655204Z", + "Partitions": [ + { + "Id": 1, + "State": "ACTIVE", + "Priority": 2 + }, + { + "Id": 2, + "State": "ACTIVE", + "Priority": 3 + }, + { + "Id": 6, + "State": "ACTIVE", + "Priority": 1 + } + ] + }, + { + "Id": 2, + "State": "ACTIVE", + "Version": 40, + "LastUpdatedAt": "2023-12-18T15:30:19.111616286Z", + "Partitions": [ + { + "Id": 1, + "State": "ACTIVE", + "Priority": 1 + }, + { + "Id": 2, + "State": "ACTIVE", + "Priority": 2 + }, + { + "Id": 3, + "State": "ACTIVE", + "Priority": 3 + } + ] + }, + { + "Id": 3, + "State": "ACTIVE", + "Version": 8, + "LastUpdatedAt": "2023-12-18T15:25:07.484956052Z", + "Partitions": [ + { + "Id": 2, + "State": "ACTIVE", + "Priority": 1 + }, + { + "Id": 3, + "State": "ACTIVE", + "Priority": 2 + }, + { + "Id": 4, + "State": "ACTIVE", + "Priority": 3 + } + ] + }, + { + "Id": 4, + "State": "ACTIVE", + "Version": 8, + "LastUpdatedAt": "2023-12-18T15:25:50.616089815Z", + "Partitions": [ + { + "Id": 3, + "State": "ACTIVE", + "Priority": 1 + }, + { + "Id": 4, + "State": "ACTIVE", + "Priority": 2 + }, + { + "Id": 5, + "State": "ACTIVE", + "Priority": 3 + } + ] + }, + { + "Id": 5, + "State": "ACTIVE", + "Version": 8, + "LastUpdatedAt": "2023-12-18T15:30:16.949863252Z", + "Partitions": [ + { + "Id": 4, + "State": "ACTIVE", + "Priority": 1 + }, + { + "Id": 5, + "State": "ACTIVE", + "Priority": 2 + }, + { + "Id": 6, + "State": "ACTIVE", + "Priority": 3 + } + ] + }, + { + "Id": 0, + "State": "ACTIVE", + "Version": 52, + "LastUpdatedAt": "2023-12-18T15:30:20.920293674Z", + "Partitions": [ + { + "Id": 1, + "State": "ACTIVE", + "Priority": 3 + }, + { + "Id": 5, + "State": "ACTIVE", + "Priority": 1 + }, + { + "Id": 6, + "State": "ACTIVE", + "Priority": 2 + } + ] + } + ], + "LastChange": { + "Id": 14, + "Status": "COMPLETED", + "StartedAt": "2023-12-18T15:12:57.790824149Z", + "CompletedAt": "2023-12-18T15:30:20.920657536Z" + }, + "PendingChange": null +} +``` + +All partitions are reported as healthy and leadership is balanced: + +```bash +$ zbchaos topology +Node |Partition 1 |Partition 2 |Partition 3 |Partition 4 |Partition 5 |Partition 6 +0 |LEADER (HEALTHY) | | | |FOLLOWER (HEALTHY) |FOLLOWER (HEALTHY) +1 |FOLLOWER (HEALTHY) |LEADER (HEALTHY) | | | |FOLLOWER (HEALTHY) +2 |FOLLOWER (HEALTHY) |FOLLOWER (HEALTHY) |LEADER (HEALTHY) | | | +3 | |FOLLOWER (HEALTHY) |FOLLOWER (HEALTHY) |LEADER (HEALTHY) | | +4 | | |FOLLOWER (HEALTHY) |FOLLOWER (HEALTHY) |LEADER (HEALTHY) | +5 | | | |FOLLOWER (HEALTHY) |FOLLOWER (HEALTHY) |LEADER (HEALTHY) +``` + + +#### Scaling down with broker restarts + +We scale down with `zbchaos cluster scale --brokers 3` and restart a random broker every 30 seconds: + +```bash +$ zbchaos cluster scale --brokers 3 & +$ while true; do sleep 30; zbchaos restart broker --nodeId $(shuf -i 0-5 -n 1); done +``` + +### Result + +All 6 partitions with 3 replicas each are evenly distributed across 3 brokers: + +```json +{ + "Version": 17, + "Brokers": [ + { + "Id": 1, + "State": "ACTIVE", + "Version": 54, + "LastUpdatedAt": "2023-12-18T16:24:24.619548623Z", + "Partitions": [ + { + "Id": 1, + "State": "ACTIVE", + "Priority": 2 + }, + { + "Id": 2, + "State": "ACTIVE", + "Priority": 3 + }, + { + "Id": 3, + "State": "ACTIVE", + "Priority": 1 + }, + { + "Id": 4, + "State": "ACTIVE", + "Priority": 1 + }, + { + "Id": 5, + "State": "ACTIVE", + "Priority": 3 + }, + { + "Id": 6, + "State": "ACTIVE", + "Priority": 2 + } + ] + }, + { + "Id": 2, + "State": "ACTIVE", + "Version": 46, + "LastUpdatedAt": "2023-12-18T16:24:16.029577221Z", + "Partitions": [ + { + "Id": 1, + "State": "ACTIVE", + "Priority": 1 + }, + { + "Id": 2, + "State": "ACTIVE", + "Priority": 2 + }, + { + "Id": 3, + "State": "ACTIVE", + "Priority": 3 + }, + { + "Id": 4, + "State": "ACTIVE", + "Priority": 2 + }, + { + "Id": 5, + "State": "ACTIVE", + "Priority": 1 + }, + { + "Id": 6, + "State": "ACTIVE", + "Priority": 3 + } + ] + }, + { + "Id": 0, + "State": "ACTIVE", + "Version": 60, + "LastUpdatedAt": "2023-12-18T16:28:35.518105574Z", + "Partitions": [ + { + "Id": 1, + "State": "ACTIVE", + "Priority": 3 + }, + { + "Id": 2, + "State": "ACTIVE", + "Priority": 1 + }, + { + "Id": 3, + "State": "ACTIVE", + "Priority": 2 + }, + { + "Id": 4, + "State": "ACTIVE", + "Priority": 3 + }, + { + "Id": 5, + "State": "ACTIVE", + "Priority": 2 + }, + { + "Id": 6, + "State": "ACTIVE", + "Priority": 1 + } + ] + } + ], + "LastChange": { + "Id": 16, + "Status": "COMPLETED", + "StartedAt": "2023-12-18T16:07:07.208363298Z", + "CompletedAt": "2023-12-18T16:28:58.836369836Z" + }, + "PendingChange": null +} +``` + +All partitions are healthy and leadership is distributed evenly: + +```bash +$ zbchaos topology +Node |Partition 1 |Partition 2 |Partition 3 |Partition 4 |Partition 5 |Partition 6 +0 |LEADER (UNHEALTHY) |FOLLOWER (HEALTHY) |FOLLOWER (UNHEALTHY) |LEADER (HEALTHY) |FOLLOWER (HEALTHY) |FOLLOWER (UNHEALTHY) +1 |FOLLOWER (HEALTHY) |LEADER (HEALTHY) |FOLLOWER (HEALTHY) |FOLLOWER (HEALTHY) |LEADER (UNHEALTHY) |FOLLOWER (HEALTHY) +2 |FOLLOWER (HEALTHY) |FOLLOWER (HEALTHY) |LEADER (HEALTHY) |FOLLOWER (HEALTHY) |FOLLOWER (HEALTHY) |LEADER (HEALTHY) +``` + +The operation completes in 21 minutes, longer than usual because of the restarts: +![](scaledown-completed.png) \ No newline at end of file diff --git a/chaos-days/blog/2023-12-18-Dynamically-scaling-brokers/scaledown-completed.png b/chaos-days/blog/2023-12-18-Dynamically-scaling-brokers/scaledown-completed.png new file mode 100644 index 0000000000000000000000000000000000000000..ce0f2d0421f1cf3ce4d77b41ad7d4ab04b791530 GIT binary patch literal 24996 zcmc$_Wmp|u(=~W-cX!ti2oT&g1b26LcX!vI32p&GaCZm}L4vzma0@QeC-?n5-}8Pm z*Ie^&&d)yGyZ5eLyK1df)lrJ_A5ahp5kVjj%10@2We^BT3Iu|nhld5O7{~r91%b#x zAH_vfJ+qEiJv_0^=3&o$8XaoOiV_nQKcPpvGEAweVxtq$M5d zd^Z&7oJW*uK*vrjaD}9%rl9Dc)2VH{dan6MPD`y*ck-gA!EDBEv3Go$Y&tGDk%eJw z%olBBl%L3eO->2^U4G#3hLRqYCPzYZ2&uiq#FwM>*cbo%+reckL(b3NOqAMH#{KZm zrTJJ|svSfg>Qnj9Nbqk+rCRv!IF#S|A*23YD#b;j{(GrFnU(SqvB$7)fWf--KJ;cP#n&kD_AyUt?H=`tc0>_Ueni zTut9DpWh!Qod4mn(cbmM!@@7^>072V8cN)RM?e*km*@0fyT;lO#JG z-SNFUZDh3{o7KN?B5|D0PX1N*!C`0@v8XtAfa|EUuk+bwC3{KY&*?1QgrEQ(67G0^ zPYB#fcKj?hA`R`DKbNSe=*8;ok>jPRb;y~Ay!60<5*Ilm8=--J#}^4q@PuPD)yPQ% zozWe-u6RLtcCt*T8ID~3Prr4t1pP2UiD~{zZUo-s4ZNVLt_M_OW8-S7U5%Z+95H8S zE`IK34A;YyP@1!@->29lX_E{HRm`4H92^`AFGuPJuDr-Cw%mK2cZh3VXDGI-jlWhI z3VfPKJ^eQMJoj1_xn1wbK*DNjG~+pAqH1PgnRRtzvl?;YQd0D36`E6~7cQ5ojBM&m zJ=bM*j)47ZF7|U*5^4Tgx$c|r$ndZvU^GykOuBuu$&b~${SAoYrH<+1VXtpd9HThA5~4?d+Sait^0LVia5RZp3wxAvV&JDzo(+}=d5M9j9 z=gza}JT{%5b7nkrGhLrdqZPnEHF)5uf7gwU9=Uw5qg`>Q8;Ea#fq<}ns(8M* z(8luFe7;d_TL|TAMU?BJP!`Kb;tfjD|B(@+Q}L1 zc^q9EI6_+QX_LV*R(671qF(3qF9(SnlzA!}Wu5=K?HxAH6TO=TEku(bjt8xZ*y<^#p1q$Ob9r_x177 zZeDIup%Y_mzdL2xccJ zxtIya9n1?7BeeBc@0hOf8ij4I-;g+P)oy=(#vv{#4pE(T(8V9XDoabX=6j@OyVOpK z<&YZ+I7h><2_rTllcyEyr{p%{{*mIlyXoD7`6{8kF+KRGzRMYVd>RTk&T>_o64ny2aR@knmX{PR~ijgFmoOE#();TAAnYq2b}PEWcY5)pg=- z(>jAX9^L1KXj4btUBrL9yJ>T~yLoh&ta1q8cU%tvfuh8t)AzdKi~3BNbZVf_-uhwb zwYm11BnU1VcKJPCi;0V4uX!G+9k114HLZHng`>6*gYs0ry>mDf6>o~`89_@oyqZJt zw46JJy>L2N+TOi)$ZKsytYXm*v|HxdJmov<$;Nt}-Z}b`kq8aqb}hYd`T2819Po}$ zn0#EG?Jc*nOWW&CkE;vS7jVAU!_J`YYb&(P@S5$5fc47$AM<-Xwwy-eNJx3A^fdvm z`o5Qo(xC4Ba-ko~XH3?YEt-7H4lP!Y^4aphz5Dq^MbLF*eu$f!b|8?6FDzxE-WGysFs%*jnUE3-yDV<^PishclNnXbv5+$qf1NaONcYUX@k&t z33)11=d5p&3a;ilqzMFSbAJs&J>fI@QGr$CGeP!e*>2$us?oOLr*%<+Cmb5HjI=?0}P1n2N^MUsp5GFc2WdiuM5s##H0qaA)KlH@p?i?iUA6AAxwa zX!~R3ej;x`G2}-9Bq)ub{;1M!VMe56db7d<7ZP+swh%)R5>aT8Ah>Wpc!ux0s3FV(s{2bUz}53#)^mMtC_0~Ch)m7RK_F8$f}8+%!KJ3Nz~mBTr|VL)@PmVzUMU%& zS<}k7_|a#$8^aSW0}yCrbo64cfaYNC59aaHwntxIy_lOD4yVH+d_^PSpaQXkrKM%^ zcJF6NT^>x(`6}9xt~W6_X!A}mzpuuN~mXM@Ez}h7qk2x zUm-!%wC)2vVgXc~jU@idH60@QWo3iE8f2Ot2K^Q8anR&_@F{vICM|<8$mU6NtL$~T zLBeKX#g=y-L5{5EbuB>P3w#RZ2g3eY@Uz<1<>-ez7iG4BLX%0FmN!LH;SQsV$KPS^ z(^FGLE7T!CyzU3k$-^@n4Lsh*QXzMJedCeQu>-AD^T26_hor_tNRh_5_SWm&*YtYf z;dIN6k;IA}83~`)}l9v~Ecvuet3hz*_+i!Qjdebcmv}1`bEED+3U~wr5jQur5Hn{yMn-0FD=P%=n}fJqt_p{9fmG=@ zA6kLHuA7%Az#Slk+3z7lL_|FID|SoEST~y18;#C7SBHiUR05p*v-fT&;4w4CXfwvX zUQL`COlY7(o0{kJ{E%Sbh<5m9vh@4?sIBZoPDfI1eo`QOnARdfnxv!^bAi=U(tf4P%Dvs@a zb8PoV@Jq5mVnpu;RO1;pF5!G`A)ET=^ooU-(Yc3!H`vm#t*5X6QcNT{2FGvRJ}Y)9 zgMcduV25PpeuM*aORV>rs^LjR` z)wsQ^qz3!Y3Izh-hcLI|1z$Q0a(iHP%`o#U;E7C9*9%48^OWZM`Z*L33|a3_AFDmq zPakNaUT@Yuah)FqbDf{+yg6Tvx0|$Xz;|AJ=Xx-S06>dErI@q5H4UHms`(=&b^Ykk z2!pVd6=HEo$;}%C53Yr;&5gZlC?*c!y37vFa}!HWi&;wZ@%;eW?~6uT|6pVUF%a7c zAz6rFF53$43}y4*pDo)333)8JW%52&q4bJzL2*s z7KG+PH^_`{MlNQTX73JuXESXpUlcUFsn z$wPg_`2y9eguXwspza@ty)iA#E;&g_Uj2+KY+40x(5v5;*TElNfL0;o`o9>}wFq47 z^fn)j4>w!)Nzv~hX36t~wOr-e)poeLoRtd`VZAB69CK*i3qra88i zTh0C201XYzc@TKJgEdiTG6?-M!_DB?1ThRU*S{XOtHgE?ZgkCV_KCPyxh#MF=rBF2 zzriNK>?|NQtZKrc9(#Oo<-YUb8^AvL20cn0cC&%>qEaI7s6_(C*fA0k5^aqiL+F~X z;)n71Zg71sc1wmvM#T5Whw<4wxnFKimLBYwiQZ$2Md-$NcMIDt+!YelHRvVM;*Sd6 zL4Qaf{gGB7Bu1HJ_|(j&sHPSb0{80f;cXi66x>cr^d24jlB0XyplbkKhKc6)$#KO3 z7?(OaCZ_r{z+e+#`%s20mH~6IgNAIM&tGai5|ffn_hRi-l(gbC@W%V~u+Dl+68K?0 z%SLb0#c$r@y&k2Y+WgfuUW#VZ7Qi7N1H-Ep5F_^M?`gqqPI(FQ|Ds|1e-b+W|G#N? zB^%Tdpb@oU2B@!_+jh3$`dUV3pY;wkHKb*gZ|NrRAXSkq6P+r z(py}qe=aqI>ifY(-`{(kj)!OyWWHbcz?dvnjNsxyDBl}X{af_kH*KTg?LI=f#6c{5 zQ%)xvoF5cZRef*quXuF$e^;2&)`!XQZWI6+W7-8y#E~Q zb=m3blYW)W-C42yN0-taI)t~5=3uIrXliQu{Pa^laCRK+8~-sRv#CaM_)9wKk30ym^RBZ_2oOH9Hp2wO1YQSq9noEuWNW>5g{vXdJ<>ase1&)6l&fwWrYUjSSI&e)R61QrYz4Mw@ z__O@|ehVwM-TOVjvcUpXq2J7^FDg2P#*mW92Ji6PS5!qP^p)&z4JAf35J-eFOq*l_ zYo5Gp#@eGqn6&R|Fy!DA`f23;qI-UIU26bxFo;ftJ_Srlg98IW8L=G9d}+Mh&V`y2 z)ZDv!bDTI*H8m&aKc`)4EH8p#NB1e0#0!83biCTs_QHy>7jwG*qtEXTb9>H&$64Sp zV+q=GJPmu>UteLA-;Xt*-QnXV9r|%eLHMekXvTEj-jkfo^FpWolN4m9X7z@6>~=6@ z)@I|!j~}dO{b6?p+h*Kej5y9Mkj?<>B^T}6P7nym5qK0XQ#A&niy_+=$HQxebQ<}; z=H0PXpCDiLU)GC^it3+M7%vO>USE&zvF<_?9gSFX{>RJHa=;e2b4&b?_2=hXWbuow zx_>=9NAh6SBQGO2cmI^XG_`EjIyZ-~K{BGJ1{4@8}C zRoB*YTeXog71u>iWCz^oz>EaqY~(Bew%G{r6v5^MTE_W#~hAB;4| zDAvM(cOZk7Qw>C_qXMtrV*``#Qs)~v&GoK#d6Q6PE1ZnEdxSJrI4Ku3rI98v>I=(k z7y&pgH89cH2?72x$!OR43}KNt(sW5GvwB?^3pSt4Ysbcooal)|t`$)=P12>7o^eh4J$ zOPLXJp6lJ4w^ijTu3wQ;TT&0+(GPso0xv764G_b-iV)RB4UGIkU_p^Gq>)-O30ngM zqZFx{l>*Y>(TuVrz>B+$k~@efb9f0uP^btKA2}){0YvtvPi#`C*&*ywphTBakn-oi zgG#%sL>%zgEkxjVOY@m0GVi(Cec%I+ap?~@2uoQo_T=*7*5ms4@lQ$562s6w>#SNJ zFab!83@~$giU{IC*831>kX1V_I9v|Voij3D%G|it@cqZ6a|b@Z87!Qh-}E}!nh;*& z#5bVP0Rz-Z$S?VS&kz6s!P}!`3yV8M+RKU#bItzQPBxDWdunyI3TG z$OUMKqHOQwfQ{N|&?HY`$i7q>i&~l>y61RkL8l} zEyr`EZ-?Q8H`?p%Bx^> z^I#P0IV31*J8i|Yb|XLaGo;M%-BXGs#;7HdgQ0<3g0 zjkj6sY+R7RWAlt3LjDGnH!*W{T97b0^!4#CB0A*+C=ezlrqfp6uLryGZ;VOV$#FLq zB-xp7ld$MRQkYMSJ@QL;-u2zYiCzxf;0pv~sIeCAp-Dtg)NuCyg^4Z;Zggb;c8&-R z<_DN1ItU#dJu*7_5_VZtLxX~xJa5>u;o)i9ISiQtwot$-5o!j)8Wh|%?TF+Si5aZ4 zWJs(n10WSMGhNwSc-6fl^fd-}ij!cVZg=(@L9gbefb~nGH9R`{XM9Kz#vpn~Cjk>h z*Ka@fbSv_e%k^pvF6I9VLp}Xz@6pLHbM8KVKnt9OfH>c|0FV}6J6a}E(XT>#;q^|a z^Xs~VIU#jDz1ag(c5Zt7+1a-=^h@5Jlo1RZi1^2v#(0@CWF{6Bq9SvwwvV2ivU_Pr zK3;j)!8=bZiPbkE-1D+Q&>+kfkcjo?6&&citA-2aH;Iw$NU!G2a0VXVdTTby3iz&j zIBuCG*FFcFPk(weWSgu1p(NnM&s}0!S0+pdAawgqEB+kc2c-ko)`!9ADfvR1^wWi_ z`CJevjuzf-d4Qb#NH+PgrMh<7dhz$TfOwbR362O`%1*k-`^B4(ig}%Zc1uo?@(TMJ zVd+P9EWN?T2un_}L@DxU%9n>@_-OVoEbaqL`0gUed5CIx`y>cbt64!)$B3EEjcM$H zb` zHKb3JBO|h3zC`t&8NbATO#GvHiyrH+hDUdG#N*a+r*GEEr=V|`>(9l153jsBp44=S zU}z*~l4rp1;Gcx&&TYK*&O5WLvGE^|WbI)kIi92x6!{AQhAeCT9!WBow9cW*h4n|I zQt{j^0@IrrGj;@0GCVQN^X`3aR}mY1d+gjb_sFr{$#qkIY1)D!`Izrrka<^b?Ue^u zyxR;{)Zuu?2G#zB1 z8BgBZV;)=+;us)oTE~k+hDKzpW``@Fd;cM!-`n4B;X|`Fvu;=_nUB|()$OvR*2jog zUt6-!Xji2L^5x#n_hYOjX87TGcVW#(!=YKW6mtzx0{Re2;h;E{ul4vs)-Q^dF(`>} zQWP4)-=oW7%9$pYe3=3goTw`sW;m8NIi#Q+`6s==m}Wl9v&}7L#EE%5k~1g_9tqd! z-mj8RxgJ z^;9~Juu)IZjGCx+NVgQ1oLZ00zg%>cL)^C>)yW!avYXvlZw6Su{%)QbBX~Oige8uB z*?4GH49dewLAJ~woL7rgxsx)hse=r6m-27lR(O#lu4pr&)eHZb60W@l6#QFiZLF$xXso0E|QnoDOhEUN;!?!4-3`% zxt)Q%X?Z74K>wk?g)wX4Pz&ahE;AemUuPFKWal?GY3cZV%|^gJN=~I-QRF;Xd3UYD z3_T+T29|o*#|gOYNrslu_}@p{6z>#!0jS3k?^+WBK$VMxDiQwyqwfZsb^S#BjhsG9 zQ6*>GaNRdC)!DyVSIA5bx2F6@2QoCmMX_)KM||}`3gY68NGO;WBVws{>zp(D|76GdU$*~&4y&eAu_ zFpmeW@$9HM}n_=mM)@5qj3hgKuTsPh)25n#^m@%+ zmCHb3SrVTL|K1MEkiU(L=@s}5lce@`AHauv9wnmU;`*j9A!_vW7}YyR1M}iU*5cx! zHNwu=01O~f4#Nf+vSqAfAtXt7OQZ6l?6U!|s&-uEwMMt;VTn%8ql}#pMm+pW<`bz_ zf(_r0LuoM1q{hPf)}!qqDbwdg62`crQ+c_U;saR%Mwd6gj1-aC+;d$#cmY(A` zVVV(1w}xnY-H}!QtjX#FTJz+_vI|xNc_wRq0DD+CqZEW5&+_aii;n;_q~o+9rz|jE z38=$e#ftU3=@{t$6^1BH6N$Wy{CAP-im-3;Tnu5OhH#uwafBJM&{rGc+EgegDbZX9 z-FS7^CL%4;Ee$R6eLRe!pfQ*di3e!hceua+kS9c(^Z>w(ZSC63`b{YAHlj53LC%m5IZreIf>6{FAs?j2 z$&>bZ>ez`^2VF&_zQdaFq@K!0MEcIOO6(2|r;}?^84p0ewfEv*_?F=!;S3G;<=w7% zof>OA@#wk83fCGl2v3BkEA?*O0nMAG4O1V=)IXl zxG;}I|HUqX_L&$j%8ItK4&yQgII1npYkIMX(r=x5g5EZr!~@uyb$U(gzQxZH%Tp|7 zS>?(T1z25ih~9CeE`z`>6J~f*2wP-!MC1@h!H+RTMkc!DV36yPW#}$+$7wlhDrDkp zC8`bY-j>s)`8NKvZkDRIp#2j^D{Yfabd992Dfd~(4n1I}b;S!p>Py21M5rlu)57U$ zZsPTfy;Il7p}}Jz+TJUbpYJjuqM*^oy5E#TI#J6l*{+~Dr1zNCW>xDDsgPMf7u7B` zqd_@QuST1A$;38PWyi6~^;c-E--TD>nY_CYlp5~_5G!okLQ;v00#8gn`y*IVDam{^ z#1Xs~fvF&svUm8g^p=rWX!>&aAub`@a7vS%caZ43U$RAw)y>I6x1{b~^M!OG9eRFiyR|K z&=(09(u2)e@{xiS~0%BMCgzryE6MqcIvIM)klAkM8w3hqIaq_SHxb8 zfFHjjkm$RKb@gBr-`PEFR*&e^i_NIh-9A+<#lcCDLMIqMN+bs3c}Sx7iCiB6>&W*S zuV%wyc-7dz&KYje3+%l`3XM-+e@OkIj6f;6I6MvH5h$3q_Ps_@wXPM5Y4AWX^nU9re|TEC4R!3NH@sY?)b`l!>HfXaDq zlg?x@xZ-J$(*-Z&+t%`8vs*in{e>Qk{1FcT-}b^N9FM$Cv2`V!8(%F$PfE|=s=$Pk zA3C2k_pjcX07n^sZAUCzvo@2%d%FBmesVbnIlD}7-0A;s=`8CDc@jH)_PY@AnWZV^ zapS)2XD8cUf(%(&j}kF)(&MP%+0r=#&4-4i)w0A)oHS|lVOkW_8n5ih%TyW1yCIS5 zqvfuiZ6M>)(+_!Q8;g$@Au8uFl~1i&p)VG(brH4OAs#R)C&G1HEoi<&J}Ud1wmQdhcDa)=$Dc;U>hz1&HIj8Z ze|}*+Gw@Ox0+AwGY z62#CNYkvkzjqvZ(Rtuc7-h86so(yH+%5)mt?osdG*^xp2j=^dYJa=TmFVNMR(QL`S z6#9kLp0iDV$}6lGlJbXm)|iV2?b*ZaHGunI!;~R}H`}3C z#f$fpkri+OD4BZC8|r~5D%C~~>hAXO7S-^*FgBQ0M}dH(#1bQyqT%ch^_W>8Hkoj; zjk02t#|~=FVWsC)o3s6E^S9$T!@oL`Nu^=Pqq}2N{tUfsy<-p?Tp)}@pDeG#U~9P@ z^z`<3W9?|er%2T{ZlxY_+z72*+&5(zT}Q-tab4T6lO87-+@H`xuv0@wL4=7OF^zO6 z7glPy8Vuzn2j%dS)-PE{+-|0LaRKcFLJkiIS{Cg2Tn9CKF$(0#!$~O_GosY3$%hBnO=n0`h|+qz`&!s$tn0~y*XgeO`#vMHs?>Iihv%p~{&A4D>-!@kQz|9p z%ud1cywmGdyQh7;(+IM%c&3!=aYo=j%{yCLtL)6*kAcX?u1ua8<^N~t4U`%E7)yx0 zjiaTI@@L|L8IIh^mSiyh*_Zi$W=**~1a*cP3jI%fsG!=L{fF7J>bZI&xbp=)$;g7; zKArbaELF(eN%ecDWmNrW4o608kjkH-Sjm>WvRo|&xi2)|9RHjcnBSZx!(o2%LI!CynmDBYd{7G6A^##qB&D@mhj7Wti+H-{{ni^XP{( zNh*tZ{@LM^&+iC+zEdTvd5tllA7=8=58E#;$yybq`<6-6g$nQ zoU!C_&#rLs!K&mfSkHfiS!6LLt94cs%swa{>E^;*@D?o_JfDA1ti12SM?CQ#{EBNU7R(&&@ zCI*lgSV6*P`hKieJ%a#5kHRV&tTz3vGc;~A>Yz}pUhda=*_EhFHMr2PvGMxV!OhbE z*fK`B46NEqWVd-!C)g;^86jGBm36`mlu2Uk4(K{HehgVfEQYMS?^KNKsh*uU&?6Sk z`R(W#?0OFCIm{j*6p*$+fe7=5;NvjC;^uE5224*$>DLWQP3FbqGs?^JL=RNP!-i=@ zGNm)8St90x^PocB)+adQr)`_WTqpda^s&(4Q@{2&?%qFNnz!cFLjOUDX z|Lxg8yE5#At0ps({X!qAmB>kpg-ZR2Y3W_?+cvt#Kc_fn}b`58BAK>PNIwA~c@5`|}x6T9fctJ=PfC1U@jk z2b$qBFs{UIv5iM8L|0|3W3FTim)E$~Y;#D-ZNp@_Qk;>Z(H0`K3G#<7=qDZcs`+@7 zY~yS#e_=gQCrPWYv9`+gNPu(Z1bZV+We4hT(;yTeH}97LDK=?+QiG6*{Uh3+ASH`n zB?qh5>;{(kL%%+QrMVL0FEN?^<+!8sWr?Yy6Eac0av}&s91DMK;a)>^$)YAPyU=lo zF3HaW!ns#Nm!tGvNeTVS&iMd#u8M1!kDPKZF6-hnkNi!Ii!J;B8`qWOhE19D4*?{BVFhyjn~$^6O_?#O>du7 zL#&J-q(gq`IfeAnhal`3kr-e{{&a+t$UvP2+4sgoJ076bt5pi!GZ)&&y!5>WAMV>? zhw%DnuX|^>UVns^%!yJqJn72=JGccg3(*CwdUJ^w6v$f8n+!d&Z@mKITreeCjnJz&6*S0u42T1MC}9U?zvwP zKZzNP@4AVN60PZrd`xZt1tO6Rl^(OXR{mP#PNu%G(bMt3_=$ ziIP)JC@WHmKqFDN0pL0JE3&$%Mh`5C_{-tWjcgY{v`-stTql3S_%^pgvhB*dwlq>z zW(q;mapv|!jaeMFy+yE&9RGw}( zmvG_Aw0uepB(FehJ;|n~)?#jqmJk0C^QT}F@UW30k-4a@vaTtUX0QRKqM!mh3}(<6 z_hm^T)``h1B2}hT!S4gLVRV;f7j_zp7jYJ@E{sf1#$d?F+f~qmO(63T)yUaClObbx z6$UG>G0l+zF{Gp{*eh9+^3q@xGxi|OH1xZBM78Z3N?thNi3Y}BDNZ2Msj|nh&;fA^lRDXTBa?S8>DgnXC?&H5uH;!>{4Km{&K-F zG{GJFXVmNq4+lK0*Cd5MozS<=l&EAf2={lx;?`VlU zBa16~v*mc$R9E3FOX;jD!qB#`2vFuB3x*KvNwv!##*_g#IQHSnMlB)jC7XcAS=VNMyxGtloGU3@{u8AL-Rj;L9hp^<7Yg%904OdA%7AUZ;x&5o zpSs9-c$9dK{VOX$POfqIBC-^9+Ra?mcf>vb67e-ZpSu&tN(wVxJ!PaVx$)_IROcCg zN(wohr`tMf7}WW*AU@OnzX>G@DV_v`1Q>10(lc4LgwSY#k9#q=T=)05Tb_of-R;2P z)O#X8CXM$w`y35?;&Z&*1$WlogwLBjS2m2fC<-iaJPO$R9g_cW z=$@Ooxj?_SH3B>XpJQRM;wu%zZjZ34w>zHg$HF4Nt0>xPnR_s;u737J%d8ZMw!us2 zuj+^iiFma-_q3HIHXqB1Yg+dsNj`1C@$NuHlMh%Zi^5!#27)+c7yFg_TS!$(+yLB- zQ&r!&&q6LB5DDC1D-RZBt)x*|VC1`Bt5%*knTSP%CMY>xr%tx3tu;N{^SXW{jkeDO zAL2=a*YXBO)W{4EHZ~O*Q|0B-hH!!U*|qJ3+z!|OVBNm@t4yC{du#|J`ALA(HvSco z0qB;vhw+LdRgjYEi^AGJtJVW*OvhQa^$BMOj=|zL%0PY3WeZvPzsq}}>1P;TpZ=Sn zV+}HEq^L({g@he&()IxC6(bNRZf88~8DyN#XiF8LOekrRFOD-VWTj}!#TUty*h0|G^=H2$6b|YMaO1rvqiOvbo>Th zoPbjP98n{!2AU)(40Lo*^2HO6v=_%DXTg%yzF3YwYT4g4^S4B0i%h88zkluR90HHYuu{t}gs4U+ieHk{vmFv3TnTj*7BMk;615*G^7U zA0hK)hu7|0mI=C-qb^~iGH8r^bMDvYtGA&Iz-3nsd0B zW9C2wuGf|~Al~^V3t_W|L*DllsOc=YqD=MMzM8R1#u3Ks>{P-d&lrw8vjd$y*av@f z;pI;Yk=3hXNRrSp|07Ap@3y~0@=pK)YJuOVeam2f>_HU_`3$qh)%O)Z-i+bQ7mFo8>lkof%h_5` z_}%P2ZdUs1OWvDlDx2ItqbFNnQG=8~g^UhT>(P!sjc%~}WC&%fHpx3B(3 zdZiSn+DaP?5n$ZYO(O0IqJ@Ub5MXRV+TJP{v&6C-Cs0D&TkGu zPi`}fXkWi<0YwBx#w8vA%k#%EN+FN{U^pa}ST!We7onBzbDlCdyWBI(4h?smD^G@# z1YvHCp;HuCSWxXaHw)5=plR6$)4E{Ma4pp^|I>NURRpNOK^@_feSuWVIrJ+m~ z%|}!QQBDRZd#K_bGB14Fd?dsy53&WTlnC*s^Njz-cmv%4GNa^xd!JfQK2OU=dlw_;dC`(*u1p`^c9#B0wt` z8~`+y_yLgcxDd>ky~6)V0g3AGVS#8UO-M3rZ?A@P7I@;YqTt}U%M!Q~)KfATOh+)+1pAmUoanC}9YzMXrRhBy z){`z&p2rR<-VGi#ku~QTc~k6Zmbz5v>_Q!XYymhEz*ddeGK@;!8^7WWDt^*Bodp~y zw~)0Z0x8K;g4h`iCtq^;$%m;b`(}3?aRLV$H)Q;IFt3&x@5Ad|m*hS@iSbWBtnQmz zm5xZ*{ZDa9ZiJj>WJ0Z#52e7H>Otovkx3KB(&bnq)3$3OL4ftB(aO+fI(}|sEQ#A% zfUe2nK~>gAu zijhRecX#sfI&%R9>g*&)fQ$^#(@i7(!6@AlXCbPZloFYW7{TPC*j{KYX+V((n5q*n6V26R*M_hc(cw#UjA62bj{%!VZ zHe!pgNL1+P@%KK+DBpNWMcBe|=wE&IZ+G;w1SlyT>TGr?Lfzmcn^+M5l{@QuwRmu( zV-As42*A*ZizSB_sV?FF-e|)1rTVB>Eb;^=YFwki!-IE69Hv;9TE|Z(?C-5H+P;G{ zXno&{KR9j2?j}kvtcQLXTm*D%^#^Gv_X_^d(9!KlD=9ue$Q1^ql#yVDkfxw5g6hoDDdYrEh8o{93S)*1piUD?&ahD{vEXU47vW_wBK})= zW_hxf{hFvLf6hWUh;KnE!zdKsKqZ1cx!W2a*T_J>iOFXJs7$``;5u{7+NxDz?$$8| z#(ngEoqoz7`sja;zjiDMaQw%B+PKhZn#SYoYZrYq_oaz;pQLyu?o7?Swfl_ce`r%RaHan3YMFjs z5{DkNax}K#m1Y~Qh(G$@JX(E_b>OAB=#9zb?t|sAwj2ECwsDS!PHhI``F)xL0g9DF z{x1dUaG=167ki*1XAkHJB-L=Mp|VD>NQfN5D?Y2vvR(ET^>z07IyPt=`|MPv!?G85B5d=={*R4XAK)Q-#e=1I|)3?Y;@)|2(PkUxgRM_;iX^o#{CI zg~MfD@Y8<_!qxp;4a|T`VI-9d@%djIC4&Ezf5d;WzU*x%hX8oB;Bz9f*O zrV12fykKyx|5q=-h|Nki>^;Wc=jn!p)I%wzeS|~dAyC?8ad45OVr?GG>s)hyPcnvv z=z*qwp>cBy9-DM=P8FaQ4AAuk%730@RV7l&QVKda#TXLOp4sq$DYl(5L@B|M`nTFL zcN%$IfUGQh2E{H#1*etM3)D;|q!+N*3%YOv15x;FrK@HyudXxf5hOw2Gfz43$)g_q z*XoZ<=EUZ{N{RpFUlHy=`&PJux?hyZAZj7-iMW)ELND;4G1Z$ly&tVU)d6Tl&vFdIrbV4Q|J#vrl0`rrB9gHCCVgWlDP(A3qPL<+zAtMZ&v4{= z7`=Z$?b~iTGYCHMjcF;X=PTXInj{H2sQY#<4>la`M`=ZiVJuHN&_pvL@d(Sr&buGI zbMQ+GEsPjIHT`AYJGgpxg-hptNBBBW7?9p>oTBC?vJ>J0rnNEs+_##YxYKPh7|IL) z3c{cI5DZ0^a_;_Z3Cnomd9|CbtBE#M8>Vv)y1O{wcF5}|8;@b*%3cw_SPanqW+@tC zIF);LT+RWa5TSveg6_7o=rtJSDcX0I+}ulHcGtG!0Dsa{J8mxOFljP|mn%%0OvS+Z zn-T+(@MQ|;ZuZci^S<>>$j+n&{%TG{hXIL6?7`A6ki<#!TMQ3L; z$G|RyVMqC^#su=_bXo5;BQmDSp&iUJvXdYdd+|yn&8!g^qZ##XR*Kko#LKQM5I=3n zW$>#Q<3X}PXhfB{y$-(+e9W}Mx zN%wh89y3W!95Q>ME(qj&dvt@2r zlB6h9xE;zwmL1u@u7B%~MbNfgFYJyPeXrRe0+S6v{}!+Md|Q}U#-Yp-qVs976F*ri z{!*GjU#qy0aUjX$-Y1qTada01o%^&Z+_?R>2^ah5HDtV3PnZZ$-fDaeFE|YTa{7jENpcyh&OQ_Nb$!imkW1_I zl`Me9?u-3FL15sJvXWv7g{q=1@`Oe_R6X)$wP_UM1C}m)Ep!4@Dn&FDjD!UL2s-!1 zW+17eu=*%$5Tun=X|Yvik)3})Jv5TtfmD9n+5YzCw$Z8Xyq|%`X-xS%-Mo}mCq<{~ z?o|UnFn64M_mIqkhlS-V?&j1g{iBci_NWX#3RwDl&@QD2J_9YCN+VWOtVxH3K3ffT zolSP@BOCb%-bb#KW|;W6fdYkF{&2&Vp@*BrL^uKgNWXKk>wfz=aN2OfoKGG&%4>Jj zVk-iDVMXXN#*8&`h1|8jiyku?A3C|KV5BJ1GkYci1MYUHSp;H1>4;;O!9 zbbhK*45SvMs)j6?UtL~{@3-zlM+q{IL>S$^5Bi)X#o%vI0qb_U=y9w!N@g2MI(Qd?U$=ULJ{O@7&i?3s*(K) zsGB;NaIhPw@t`kUb6nv4cec_*D=|T3$G~*64=KYg56XbIE7*&YOzgAB5QD1DEz7_t zDt_f)w;u|yB2dO^dxNM!D*HlyT(E5XdNd}4Xb1Pa$o9OD1N1$6&R>M+csi)QskODj zYq$6)OLMa%g<7cqi8x0t0iG9()D}BrFI2V02K>~+TUyAU|5s^O9oJ;rwg~~1R6#&u zCEE!`k7Vx%Aq8x5mI!x-`2^LgLj_x`>=zi)qT z_r2@v>p1US=P^!uwL9TDBo(+__yo<|iG>I&IzctwyZ+%I{^<51UsC6}yMi@wwq_pX zOYp??IIf#`u{wx)O23LI9KBx~S>WVyCVsF>P4P(EsN^Z$9)0jQlN>baUY)VEWQ6z^ z8rIa;-k)@4XkTT9ol>}xM6tm=2CWuu3O#)acS5iMt7uGRXZ8@VHRMubhn`GtF4`C3boC%@;wPiqurEO%;|rO7uKXI9(`9hb;9dn9)oFf=(^BN3EC zv_)C8Z#+#U zYrIO!(QHqfpvY5T&QSN;+!{XS>?%0)Lz_hr7ZMK#6ja3_ z9tbeMv_}=)!V^>!q}mtgO%92N7K5nd!p0O-vd7uzUgOm`H{5Ds0+j9rlrE9RTc~iL z0+--Ji`GgFu!&^*M_K$;^lWq9SwRGMil=Djt6jqYhZ;};f{NS}wNog!qm1K972}X; zW43E|bPc3`9X;*I7K>WRq#~Y-S;EADyt|C|LbJ_jJz&$cQGZr{5>LNdL-ufa@;?N?E*Ymz$|MD2x zeb2B$uWoj$`T##tVBnYpn7=HtGu4qpYVu>RF_Q@GD?r6-u5~F}ZkTgY4U1|iZ%9q0 zT$&JYc?fkmNAB$z;d{e6s0?nvQ6^MErm{f&6Az=RRrEl=V`><=h z>(*>=J2r{u>?v)Vq5(9a2hp3~kl@uK{PQ|RIdzA)d(7{U{ z|8Rje7W^sJ>R>_#1;lz1~ zPlb!)JH+01gsZyRV>pqF$b_H6Ay16)b2*(eQJ~p&pDg`YOJ$z7-uS{tnH3T${*id#mlO&Vxd^IXv~$w;Ba8b5y_^QxCUBQH*#dHAFQCf zGZ8hIvst1xeMEWW(hDr{@^R*GV%z1v!K3%G%{(2h6&TJO_oqu|z9Mg3nzAdOHRDYa)99sfH9+EwI}+QM^y@V!;g6nA^%$J*&zX{~1?*rwZTW2M@`w>n<$twz`$ zqH#13AJG@V5rxa4o@x(F%cVxo~z5GkczytDeJNc`hj}`4;zkA{m21p7l}7W)TuV*z zX(i5i$2RKAVHR9O%|2BhKWGE{iwC9BWznCd%)@m8Rqe&urI0>X9Rya#@V&_V@wZn~ zFAt%{hY#0*Ic|E1q!e3r@w)Z28#fWgnZJvoLlabpDcjY}pDYvY{_5Z0&{JKzompEi zAEXa4;J4A*7KScgi0o?}?;Ea4fl2Zj4w@wI$6WZhl51sFC87L z&=If2M1n^`qw;UwESPXh@qX#=U%ctL6K!DQKgz;PC>$p9Lc=b*dy|CWU>3A*ETV2H zfLcL}7MBNI_zY$8dqDFLawh)dSAiC!#GM`plO?sG&6p_6RpLoTQ=55eA?Hx+6-KmZ z5>~Vi{(i;=%ROVWUYQ`F*Gth$ZsgX82(m3wntOeN{DV&DWjf*d*jN|BGAHe(F00Xv z7%En&pCVu}%TZwLnC9L+`GM4GahJmnvFIUr2&R?nh2%_cM@X!c0fVO_#C^P^+^Y$> z3O`>tz+1RfN}xZ|WG*#aWS&#h*Mf_S?EK=s0JRmAaX!(Bxv;Qxzq#+D7HD}$3&fEb zUd&6P(DJ?Ts11efcT=2s@Z#WZiD7SVl1qB_Af0>4@dkxahG0^5wmAi>I8jjB&rK(_ zM{h67%>(OKmAG7G#P&yFN;OO=_ym|RTCq3|M0ltt?^kZ3_`-83T~61H1Gc108azJ{ zWyy#eELH)L_8=LtNxYH#*iY~5jerzLKHhsOu>ypq6tU~{=8_|-6DcLWT9^E6H&$3_ zxmniq)LbGZg&sr_+sDq$t&6&u6idW`&-4f-CbmTQl%+_^bae4s_HjwgErM|;-R?8W z%_et=0+SZ~f>%I2$zD9CJBu;Gvfyt5UdCSmE`bQ)NUA}pMmPF_oaB7-ZW6kmY5@omIQiGLM;L(kZsbR zBRR^tyI8KB6Hc|yY6n(p(8MXTazPt~%&b+yh(WfqlJvsIFA0_WsQNzc`s>!uA zy*2`${_KzGmZzE6fiOqew-_~rIJI1iTtFLZw&!FI0In%cq&H5aqVg!{d!qJRchjxP zrI7a%-`+GF3ZW0T{H{o_pi>`@OJs>;K)^HmQ?zXV7ir?WQNnz`{IEfc*zY26W{MN+ z!?OlGSF0axQ?(zXNJvSa_^xZtj&M>nN~ZyxYj=I|&xkP)Y7E)4A6fA04}fu$&FoR8Uj2I%)+I(A?1s zGG!Bu|hzd_I4nFpvxl(Wvy{L%g*&Oo#2`KsrGEg+hTo~2>AiX!9YqmR5o?Sp% zHufP3+KU+Lt3QbH@!Z2IN{JQ=czJoz`6+8qt=NLXim1OrzCC`7L4_GLUp{;D=K2)_ zgL<{y7{ke;1r_V`5WqN5U|2-O{wdmQWI1`V6(#^|4i@V%u}yocMahff*$H7qb>ZXrAJ7|QExx87WV7xk|Xw)ax`f(S^_)mW<%x`kPFUb>6 zLN42G)i-~-$8lM^$8}G{p5mvW2gGb_gKkS{GGNZsdWWnsl(_>6s7K?z8#*wHzm92P#Zj#8Fb6z-rys zdlp`hO~Fi1Bp09DuM9h0gId)gkh@%J%+@nCe7{@wAAjBWC+zO_DaL8?vW_0EBf}qF zo!x8fh#Ey(Y)2O!o02eJhmR<$t*4HjYwzfLjtcC^VPm|x zV$o3^;ebKTO|n3uZ7&T#ahh>@xTV(K!NK+wmvYyMWAYJWckL|l77-G_?LDu$poxynr^lOg1s+$Abjw)U3jJ5t5&_rR!Juu~h-59|JH`^A`RIW*{(O*Sq_TCdJVM(!Jw_$N@( zditvfhx1L%9h>&7fGoJFb#WO)!~xQ&wBmDYDR_tykQTFcYg@}SQIrX7GwEbD;bdQm zz%}=)Fj={>kp|rg?id7|F8F?s!uuHNkE(I7nkA1HWYDrKEIiok0Ku~Vx~lYAUR}ML z#>K($HxL2jEPee54sP-6=Y5aqhBR?+n5)TK3WVJkejzUu70c_0*XQRSOoym20b4HJ zJDUO;(x6yYYWM+`{$Q>fNE<#U#}{cj+qz90FzM83GL&ogwukG8%WN}gbse-Lt_(Im zFkxNkICqbVMTP*cja><~Po0l?i*ZAC54?gO%nE?5uU}nZo$b#%e#}%l*Y5+A1r~5> zlppQ{F*^wqE>4Rf&LFo*_+0?4{j9*0WNCy<6U~l}Fv+O!MM;hFj zQGA9P>z?Nwe!#YbC)`c(-kR*CC!Zdw$8T{OAN!V#_cYGFEBe)&2j z64FaJGeL&10gv@zXsM%%;Wu-z@mxYsFqM|BUe01-wjX!cGzm&`q+u8Uo|nk&XI+t? z7Z6Ad&Jzf;+?|PU-%NiO>xHwwrUrXX!5b+^iKWBVX)=xJ?Iv0R%gv3f&fdq1dmnn7 zI9F4*tkh7kJatgDTq|YSF^BDlo#Jq#6)W{?ev6&XEfSXylm-a_b|(!cNW*6<>ZSz2!~-fENf@MvQ;Sw)u4U8g2D$3>y| zUzO(sCj3X?xQ5}aGbRQp8}WF~R=`u7oSFnsC{{9~M6WS!F#SqCo^yic_HD#h!j7rk zQSU9t(B}P|{d$od$jIpERJm&!ZW8Mbj3&kH38-2kgDaJSCfrS)NhV()YGdoWB~un9eNL@XiBfD zVduP2#j9t{vt{5?eKq??{icg?909NKB}>swNR6VZH2k;il9%r!H$dSwSt%Clywttu zf}Z(Wm|J)D!|96k>=-s_j+~t}FiJVU%^n=RSl$+WfLjPs`F!}j+g%x`yH)?VTw|m` z&|-J#ox@lDQxnE7Z2`8#nfH?K)J3P%lbHAFji3 z2Uc_v5U8PNosR&3q6R)+i4FSk@(m~f>}X>$r!jSq?Q+ZMStkB3?$B1#bDkj~i{b0u zG9&bj>}l?s44g<1>miI|DN|IKLuphi4sdhmdRmJQ#u zhJ?@p9p;~RG~@{X;UZBW{6Ail??yO6Pu{XJ1Lwg)_LN$5clm}Qpi#raZ_13VmqUKd zx_Z!`T*;pPWH)NN6xzeN?beO4q@_z7Lc1}8HsUewTWFut|VXDzggzTk)`9>93K>|wa z4;mywnD^2maWu`X8n$Wve<#G2IK8`((eF;`!hYvOLR87F`QD&I}p3qN!!Up*!m*5fbOwf^W z_RH|+`P(@|3cbrPj6-*v4c}2U1t~p0`0{F^=w0-}i_GPmYwIh2mib#4=@z+NKN|?T%1u^j3hV%oK(Au2Y0ZCmDGC8t7Q7f*-zy|EuE!?GvQ5E zfQKx%$g_UTY)8OsZ>CGk^|iLVQ)%`-VidLDFwX=HETE`=<$+~^EYdE-*}68zKVlck zzmo?3e%Q?`-!*vMa!D<|+YN{_E+HPEX-rRaJ7>ts7z70z92~6W&~qM)Eq*LMHGPEN z;9bqTgxp@jMn?8CRnvJk1U+qx@zUH7Xhkd+2pk}Bm=U0lkZw3xj;zh4^Obj2`F#4I ztXU@IeW@b?f3tsx1Ng$AZKKP9(o>L|7>Z=j< zWPSmgbj*lb@t%)8zDK)cDt#zp{4nH+!E9flH&ml!LjOSlu(mD;)%k8A0aVF~?>#hffqm(6e^ zkBd)N02x-mD0+D0_117O%XC_j1Y=jcphv|0biwbumhW(S|2JX!jVF4Qg> zM2}_=4{w&em;b~UL?_EjVls1FVvno{t*fbR7%eucyqVkwk4m2baKHW8iZvTdxKKVa zb;m6jCfh%bf|8h=WL%@Vg8HZF&l;?I6A}_uIgTZrtPKF5InM2z8(2P1b+hh4-x~VA zjUXRG^js?1iWctw?IjwZG{de3349RWn%t%W=tDb!vPn!s#|WSEaQ~*9*IGZiwz5jY z%1R|C-&HU7pgSc9_G}lZg#iz-7T-2?d6UfgVoY(PUM_>_T#uiG`QZlL5iC?>1>?VO z&rRqpn3ihfkO?NtXW^Ejg@FK<&J6{AkY4W1e%>7gl9jixsKQmxWo%!{hJr=^hvuh@Ed9n(dB&LIJ>N{%hr6*H;-wf$wc!BEG8- zjx>mgjzV@R|L(k&$lc8MFgz1Aqf=2;L-8q^`DcM<9>3?1UG@G7nEcf9l8nB|!##d+ z$hKNPG?Gt>u2iRFdP|90FJ(|kU7d@Qv+H7dXehHMBt#?K{4`rIS0=WNPDDvGEFr<( zqW>K|3XbWC?YKrpGMc?N_Az(L@NFYi)@$kRb?Z;@si{f>sbE?JxDmnvps2@V(geRt zwNItMzgq%PH5dBpy546Xab^f9)y}q2$4Y44+GfeW%duh#5r^{k#E0r7M-V-JyScM! zl~loWe0O~n;D@g;-7^@o#pPtFRb_?QXBICxJm!9dj=hI|y(nR^o&0*g=*qBRv+T;2 zWvN@$6oCHXNp2CrjVHWE()P2q-;C8C^=P%H1>R_mDeFRf#EijFwFNDc@n$u!rZ|@+1gmS%yor`Tx*W@7+ z!R3~ltIur#6v`xa(eHox!V88}=_d@3!gdPrVvl)%3MvnQh%Ssh2#Jd7va9y?Z9#hf zQ|2J3@6+@Rm>t(da zdvNmE8PG9UA%ii0U9Ny{P~BWDf@j9Z#Tg{ME1l;qEMpmE}od$?}37K zoG2#~#KbRIoOiA<{$(#rjH8B|J?&iv8+Yj=Yy%0>YVDWLHXSX7f8NSYKXtnc{8a$q zYy5u%)I71==%cqECfGf0EE75`kM-SONS@rDD!y~_;6O0h-o$M@2_3^%YFJ9 za6-3w@_#%T`pqkRymZ;3WQDs(UB;ku-C91zrEve65*;6~?MxQ^*##W^Rj$kx_5$_T4}51fi_Cg-?l>FNbL*|3NIz|KHHW+)<6{5(-Uj{KFs& zJh_Slkb+w99oYNA#E_8d=fo<>jVgb{e31Ncbb*a;0C7dOxh#+%u;~l%xeyR2zEXQx I`rIV&UmPVx%K!iX literal 0 HcmV?d00001 diff --git a/chaos-days/blog/2023-12-18-Dynamically-scaling-brokers/scaleup-completed.png b/chaos-days/blog/2023-12-18-Dynamically-scaling-brokers/scaleup-completed.png new file mode 100644 index 0000000000000000000000000000000000000000..ff063971bac8889d35f85ed1457da47ff9495168 GIT binary patch literal 26096 zcmd43byQVfyFa>VL~7I0B_JrJbi*c~?=RLpo zocI35xPRU?hC^IqE#_KtKJ$4#@hn3Xo>^$bSllz%LjUPtG z=X5-@ZdKP`>c%RS`Cjq!quVPYf?i{}2=M_ZVao`R6vqKE%+!XYxsLY5$(e zvj5ME=&zkY5G8FTBL|3NA0=ZU4=5vJ1>N4kLBaMaB9Vp$xY6Q1i4uH^s;IO!52hI} z`Rsy6NGO)ahXo3;o+O|Nitf(5K$t7H6L?Kfl}Pgvc0==CJf*gfJvzCx`(9<8_rvh^ z%^xVs0>ML?m<8%i3L{cV-NtVxqC>>A}l0kA57Vy?qz0 zQ9uiVn{o@DlXxFmjf+(cos=DuYwOpq_+sZuQ1H;e>V46x!T8I1*J)%+n1_|_j>m!9 z>0U3+mt9|=U>;mLb34(6$$V(BA{G_vI+-lQ@9*c=IJ~;4k0N^e2N@#H*~hi~#xxEB z92-bGvFnfdWV*VIz&Bqn z*PY%4aV%kiwQ(9(hq;!z99Z~${e!O%Kr-(yx!u?abN=A4NN%TmmW+UynGp%zGc@#W z(9@hVR0_%T`eV_q0`p(7Rj1pG{Sp>3>#BK(uY`oKZg1T{K60rzteZ3DbKZI^wN~=y zX11*sd+~hZxw7wcTz6lQ$_OdAJsf^I>OTB*+&O>ws=xoHFC09hJ8?j-V8OaFkY_4Z z^adW}dHta3x*O=`dgA+guEv98Bz@MMXYjo-`B3}UQCc@X^gLkRB~4t z?n4kZW;d!x0-h_K&3tY6X20wW!wkO1OC2B(vU-1&e2yt?az?lQ;mMRmt+{fs>*;y- zU~eh?fB=dCmQg4D33JKy9REX#?Q`C4XJKbK&Tn6uiM%NUm3~W#ZkN_?AAJvrjP!4X z1Ut2rb<{0;sqUXr&^Os)k*FFAN{1G7`Ql#AVgvSpq`o{KP{NJS43lE@1 z`YRu;()QK|kBS?AmrN!*&s(pDnmWbeNh%f=Lc_ifAkcm7m8s|BX64{xHnh5@7==Q_ zvkx&vFd-UCP%D6I*%j-UzAeyce>PsIOxyNa`z?5u3R7HM{KjgrOzn2^nnSRRjqPxZ ztra&;$Tg3h2Od;ZgFYmv?uQj8)DzS3)XCEJxY51ue!H%EaoQ1g&1$zM%TT2jbFqWY zApSB}?t9edAWTUqGwvt~p`aWJ`gOBjFgk_vn>5dRw?2n=Ph>`9qun&o!^H>t-%Tk> z_#YES-nX+^oriA@Vb5mLxL+TkXjc%Ve`&?j{K%!;y5+L)g-b$*kC<2%)!trS?>GH> zX151ze6mG?o}Qk>4@E=GJ5N4_v<3C`M0R$mp6-P-K)lAb*=v43-Y+GK>?3zW3&tdC zhaNMxhaMQ-Tu-uOdAEs7AGsV0U>Pj_!E5IA#{57>qvN$H6kZ02>@6=|nXEkXzx`;L9`auzo?PR`U5<*d;M!~>hi1fi1lL1m!~m$D+erxq+YkWnS5nfF+>+i52R3)rUK zgl+yKsW>M$hNDXRb*iIFW;3tRY})R4YmcS6-f%5Pb@-RUffy1PVNYnbl=br4my`&n zw!D?qpY(+_Hj})_Wy2mM$YQUSJU>0@^js)F zc!ca@>mL4LYAJORc#MbUQW+0hc%J=!0)enbJc+m>+MYetoq05$rmO1JL0PYJZWY30 z4t`fXPxq?81D$N1$(Qf61ab&|YfMYzl558Q-WRnN$))S7Qq+SR{Xhb2F*s0fR4ZKF zqCLu&`(2M_-Y0Rhnm`~usAj65%(#9q1s*|}z$m{EC>UsH%q3cRlc|zQO7WUevaqB& z-C%{dJCkhF2b2f3<_&F7d+nijrqu_J1^06$%jHwm&4-UO#l3%)$+C&n)nDA4o#PX^ zEt+{g91{iaRC~@jp#` zN_c*TfsF9@aPfX;x(dtr_;)Bt?_!8Q%Jphn_%br+XYNC2w2%hocS;Z_EDS9+?F$G2 z8NC-+L=foDHAO^fQ*DRYJFPQrj*box$ct>Y9uW~y?=xLA3GHxW46SU6EpC{T#%otY=E1?ia<4vJg(3ttlPR^=PkUeAQ)F)qMk|)hDlg@A zCg{vIl0OyM2zJdoy-Ij`ItY)hu%E{29~eLe^>5lY-`w+ecUOx`Y$@J9)ENfs9CroB zIPq8Ys_nb$9Q|%WSbjR;014+!uBHYKNbM}(n|oVH41FunXN?E;76&Ua#?5yBGV1f^ zMW^?+d8=I#@SramEuGnYFeH)nd?AkVRRUwR#WXPqXK6|Eh;^O*@#7+qkh4L}or#fF z5~orzQ}6C%9yabIIcTia3&n_>Ps7T6p;ViX)%gOkI?_D6>*-#*r0nW&T|vd9Ba<)3cqaQUn4W&esMHn-pCw+r9wT<VZahW@)mA5xlXh%Discd7U(Md zfx~XAx!C<6i(0a>P^AD4lw4=ooxk6=wjTY>@OVj^+t!xl<0R+otDr^(1_ogOwIM@)|fZ>l@c2O%sJJ&cX-HR<~8Q_%J>ASUAN~Nw9fm!0Cdtg zY|34Z2Ey84bV(az8+hZhvZB{~ee?udj&yJ9_R>XYZ0cf_z1^2Urb{YR_PImOxx?D$ z(Z^YROsMeo_Hrwcfn_=qt+`?5eU+ryY@wk7F^S%l^Hyx*YyhGZ-m2Ks9sBH|oyzki zkTBf?>De>wilb|exn^S;0DF{hvTqgs7#U&`=#zXKrPu54qJNT+3EMF=7dL zXc4`L9WdG%Zy~I#EKCF)Pc&$5w$UVuQ&pwx%5-b)4oht#Zx;uuyIil zP|!PP4$aHeUULYlNXM1iO&$}myQ*y}HK2W^U-TpCl@|@w(KUpIzjQnb%I|0Rr(_RWz+~D2Q)mTzh9Ory!FrMZ1{QgHJ+LQVXqmDK!O6+h}OwwHT zh?kah$M3h?@n1mMl8Ye4W>$;p)9B?2;1AOiOPEXqxf23_fAx^~fE+-LWd}u_a1rwK zhZX87a(6tB>!aRwI5F66Z?4sotGUCV_(Jx}1a~BjseMDbuG4_cunV~lFF#$Sti7$Y z0dw@}&iMX*U*2qz4%tvbzJ0W<7-GJgBP}_vt7!`@X>h$CH!Z*4AH~L|?9-k+76Ve* z!QwVi$F0>NOOkV+K>gEFaQRca4*H8TmH@MKD0{d+Jn`oZrhl#45#mk;| zNPjN&!rfTfe1|n38J1BB-u}5H3(=h|E_4VzcGxEU)Ib-J9#P%KNkf&%gp5V10W3_d z_4^)em!)WW3U2~nF7ad(yEg{vl$%Bi)4jV-^k-Ld2z7eOUKNcOtJ5B(II}n(F6r>R zJT-Wd{X$YU>P>Rob_af3sDzWq2KytUi$H4+PrsZ&6XS1MC-(OSPlrw3hB1ENro7>CmGKy#b=XXm|c>6ePj~E;zk?7TE z*sVcSQt$GIgCh4~+3Kw+?P^_(&}_Zp!Hy!Aa2{VY2Fa{$E+K4@3R&u2O10UjA&7tg za7|K5!iJV7m)hw~(CR#(BMplYLy2V{0E)zTuoF7yx?!54$svI3?JG~$Qu1s^7vG+C zuaAo8W_8sqv-H(1Pp1IkPSAWZ@hQn}*%C-Ih|=)D%ST0jV=jHI`|uD}mZ`*O_t4Kq z^wyn!rbO3&xPWB5&E}HP#M|n9PjN7LVChdHkL$yRHA4WOkLC9?KO!y1le+Wawy!i;Khg<#nsh&wZ}X z|F|>B{mGV?f<@C$b*oj;E&bqN-PQc4&2IbR?)2glOQ$90>NUpa)7wM}UAEf1G&e*5 zn9@t2Q#@0oT2oV@(@DGbjNHsAyJ&2*_iS1WQr!}W20|KnREQ5tz3=Vm^sMemg=AV` z^XT`tc_-t2;dbX61!m;aS-pq_rQ4ec0GJBKHfm3uJPMS^7Ll{&)ba2dc{z!--(|qI zo_zdm9pGay&cndWEMZrXU8xO_Hc>G#JB9}kfZV)$%tK*0Oi4)eTIb>C51D68nzDD3 z@8a9rGc52=(%kmMt3MNlt4m+2=+K8z3qEalSjsJiyrgJd&N!V;l*tMDZCRLNKa=Z( zE_lE4srhuP`F(C@Lo%1{Y%e2YB~Mm1ROQCN4- zzuMECP0WB%z4;kl{I@;3m3Bd!U`b{MZjanvcnsM4Y`8@Viu`FySQf)eNOJ8mdiR+i zG%HKA_~um8^I@0}L>hke)kFX@c{m75&=c$T@4gn&3I~}B-sZ>Np?bqTZD8|zi!o!h z3rS+G_wdL!<#ZCmL=DAH6MCNx{?RZz)s>VSuH$e25kM@^0sZ0ckznWtdjeoiqn_dX zJ)_7&C-u*%&HrCt^won8WI<$6LoAU!4%Lsr0)eE(>~V)a%BPNV-@>(9VGw2EJ0ezV0t1hz&Y9`(a?=NPo8T!*?*A_nf!lb!M~{z} za_UUFq5pb97+>vYhY(_*xZ6FER)DPXsb@x^tDpi>4KFr<7@K2{hE4DUWmeV?hkxlt z@c*eB@zf|D02yCzqFXcLV*!A4({EO~t`${G;6TlsHdFu=&Tl^c{??Rry<_(%^4s6# z?DyYGp}cHy2bH#cd7bv<3Jk>6=}wpYdadSh`&pSF5(pw8p|3LG#r-{Tzd|oG4g#KKU1yFeZfdlDPB%QY945*#8UEAGN56snhb8WS9 zQ~0$LWB-l94WSgq2@vSij*QJ?#xUvptcvwJDTyp`Br>1?Bo5wrUS66Kxg8R9JB0v6 zZ~XnA{5enbm<()ufURqXr|o?p%SWbkak&;`7q(%m#2*umfXiMb!Czjx2b4A!cMm9w9CT zmqowMP6yZZVbzn9V)`+l_(qWmL^PD~qewU2GyJpFW6Qw*9?upvKe{#$GYl{&^X53> zYY{ZzK$Ue{eA5zSbLb6C^0N)Hj_CxT6{X}kUpA0WaS4x;wOFKT&mTRx23ntO1nmqJ zluiQ%YxYkEK*;my{`3tA*)y5Q(r0+ZDq0YmJv%jWvl|=fWQ~i56KajSUTz|1QDP-A zN`n5NP7^O;pEpkw{#pSMIb$*@iMIxzP8Mje1mq?R^(|5Os0XQG!+V|o)kun z5s9OPIHULd&X!%D1w`FK62%;m<%%k7r~_VYsZM~vexe)SPvnlsu#gabNP!}E91RHD zr-(OTdS0Tm_^kFdXq7F~mX;eok$66YUDAHTh=i(rTml=>@kZ5An;e)`#q2&XAPr;n zFa3xXbfFcFqeb-t0f6^18-d{$zju%G*RN$I_jR<=b1nBac*caf6~lA4VwArRv5nXh z=RxfS2&Vft{lLl+b49^kzCT);M8Wf8v|kq>UCC*b3@D0|Xnw~4mEX1q)X?#QV$MVv zF-v%Z7E(>icT2RLtQ|IHNjHuKg1u+RB|GcY%&5J*5QKr9H@c3B1CB*eM>n3k?gHbH z%}zH&F4DoFt=td9@zxE@?RdUyT-TI?vsEf!oJ`Doia3&bzl8F2Mc6?k9<~zabJrL>t_r*P@kUm0ojv zM^pa8s@`2g)h1XJ0{LRHpzzjS4kbg9o(X3jI>z4_1sD2wE=JW3iIt)Yk81L{@!MfYcIX@vv7X=9}NpMN&{qyBc*bQQkNYtw5&prZ0=9i}j z%Eq8D7DU+v`i;zojvYG`m(P_06Sx`kZ&*nuJa}ECDoO|sI5V<_h|$AsO0*-$^@`u&pS8QYo?3@mT8 zLKl~yjj)RDH0%LhyMA3C_$4G$?`ldJy|sh8|At`qiUfBLrf_d>3bKtMTfBaI1mq{( zu~q!&D{!n(Is~wMAet!o>*eC?eS>YTQ zKVA=gFXK&#L$a#HL@=Rs2R2L=T}f?$6Mi-Ci%uRGkOE~)ghX0D#IkUgvm^ALn_M*I zCD(~jQCn16Bx|Tz9g@fP|Lolx{8 zzu9Z7J+pqh6a|}@IB3zsd8Os473u5l8D9=An8QRS5;Z)WMrxp{iJIZ{;4qj&5c-Y0 zJLYGa6cI^y*DLiU%;1px@rtC2x$kii)%fR`Ofv>YzZzvriZZ63g`9LD7L3qHxh&XW zP3=)GI6NuwpD!2^3ApnEh8YsbY`PL7&P2#p{J1T2*JTmC>UTgexo5Yb2rl@Rmrbq1 zeu~_rYQFq%-QR3l0Oyp(pqUI4s2f|qfQ5_v5|%n();_!J88q=x;OzXo-0Ds@WPc@T z-gdDjyZrHbIp*WmVO$V368~5t568OhkqgZ4##BCqB~MP$hBD8(9)q$!FR+e44hv+>t>;i z^i>?WNn12S#;-%^x_b}`;96y;?P?FZtAiJS#RdYQqN9sShnCKJp$C0LM4o9L$kgy! zbq{2a{NA%CL$rbDltJwCl#;Z;v)3*R!f_RJ{l=>lxrUoFy>y&nacsb6nm=U`O_7}m z=?=)=h=-#p;Q)c8;pb^VQ zx%o_W#kpjVQbhRLm1)Dta@Gvm{T<9uM+0ha`!iU+9Mc#f7j}5kT zXw-fff&Q^$Odxq@%RndF{A`r}j_3p3r_D=s^}ex&PlN|^4fvppal~x#@NfMC0*Z!0 z%5V9yV-t>ugSxkW6t3K^n4h(S;_<%KU%dZD%d@l*Nk>z>_2&skg~a84e#=T-(_zfb z6P>n+%#urM@No9R zl?Y{TFd*(K!fyl+^)9I$E}bc6XG()O-y_6+S@4G5FH`d!Y;q9LKD$}*$eG-RWUBaS zyz4Zp=9wNz1|sJrpxC0A{OV0<7;V>M70$%guY>AU{VqR^c5*!zIyQtxz1!GK?T)xJ zp*bv5N*Xqi@0EjWR6!!)w{WcSpfMkHq{Qz-hyrS!`DXTq1XLU3xp5^fs`PHl7=Nj+<=1l>$3G)8w7mJ!g7%b>|Axmt=|nAm@{m|FxnML%?~gw< zUWt#qYEk)9{`ura*nN4Nb}Gdt%=9!-6<_n;kY_4`Hg94z-_Cpncv?8tx8`tfp-XDO zTL%Oj=GX4tnUe5y)MqG#dPpB$pD(Vkb8aa;LJb(Yd|re~$u14W?L#89V$hN}M?!Yj zhi}dZUs}8IqUR9O@vTB@MoN+kiSq1a#dVp;GlkeIfwA-pTge2 zmX*72D?@-c=!%{c8&du94wm+m6_-Cw%YR@J4;5(_qTg`c7}hh|YUJz`Ni=2Kt$6$L zg*nrfm};?80al5e@IGdwBTxWC_9Otz#Jm~LtZS!dj0_YF;Wm#({l2_l*l+A$LyOm? z6r1HHiO&KAH=PuH;V&Z7foGFFaa#&_O!_934Zk*ouJ!!=wXX1x4riu=)4O*??2*b9B1!esb2_3BFSd zCK^oreC3aK`NKdmtzvyrjOboL-m#6197V+hD6Uy_hmcnEH_6iFP`K~E6rJ5W`b28t zQ;}6z7)w@8Hb!yy>f;%pDcV(9~Fp+Bghj|Lel?z-bHMbJ(t^UQlk&;Lt&#W zAS|Z)TvNFMXiWMYI0XE~ebFTGYAx!!A1SmPrfeYz$X7yFLW-(i3;}UKEoKno2Z&Lhtxd4_6vMDb zKAvd}d|aEM{S@ATP){jZ$AP;Uk*bw`JY zs3l_?Sr7VhN(M*G*7n3my02~=!5bBiN)&uU;tJ8STc)LH?AAD)>(m4IicT-U9<%5^ z#5{{BGGchkDso0hnD|Gqk%~mUfPQ@Uk24X64R%y9pD)e6oRUftP1d~Z3*%0XmJ(OT zOk{$hx@;<&NdT}mhQY9vY#h5#iEK;%RQH3I*T#$3Bo9ds*a=eJ$fyKMkbf)}?mkVLU?ul7`$N4Ao81h8FA1=Eqw^ z&RstUamRiGUa77pzrjC4VaQHU9?Lj}bdj!Ce>i1|_mabXNpEMAIMrHm154%g6jhsW zkcy)7yXf-ur9}T&mu~}L9y3@?IuJ^a{2RtnIbi+5ZE`8;@JXsdzw(1y{qi3e)?pEQ z>z+E#oITx9wR?N0oz31+uXj@RBMQU+7uyflPQa0Z(rRGz z#J)k;A46mlC<-TqgN?Hxl#S%*mL=ogYG_JTyUk#!-Gb^d!{Shoo@D^HO7-|w_PcRw zXvl)pi(;R^T z4(IMVCt67mC2G6utoD629EY(upbjp-qkZ59;om&%I)~f2xQytY=1}@;y+&P4_(MprDr0qA3$vpY5%3_AM3^r`Xupv zjIB55c(&+dG!$eLEG77Jib0qP1rmt({`+y6EGz+SNC2SNq2f0nr3Pzj5id+TdwbPW zSl!i$wly_T*?7ZEs8Sg6*qxdC%IIwkBcnGpqCedUaeVyPGYDU~|}kc;%Hm==L$s6JTR-~ju% z$&yibt~S~SFHStzw#K%x+O=Mv5RAE^;w%b;RiMmtIIcfl4OV5G0~{UD`0iEiY{ON)Uwt87G9O1g6; z&Snnp>s@dfiVeK8upxk{vzfw6_z#wf5B?)=5g}5;2Wc;b%WRX`clV;ZZAwQ1n zZ%V`Q$7kJF?}^%@UymjQI23QRMR@-WE2>|6%h!H$;31tNt3+S%UeUYT1;O1SDZ988 zJO*-b&SO5q!7&_;NyOrAx!1@3;$m>2+XNQm+*Rl*r9kT z{ZI1S%AU*-E4z_vmsbWw!*<3W8&!>-TJ!xw|0vX!M;}YDyb9!&6V2r zkdXVxkJ&yA-9ouoG2T@_tfZc9iDpJ_pQF{lXK!Oj^nHe^ERiqfi5t6}G+O+wdZBS; zxury+8YPzIWM>cAV5)w}4ne$=j783Sw5OHs1iF6`wJ3;*f&QDWg~YAwPkD$yw^`@i zfDIKF7u3oZm^)tMa@6ONG5K%_|Cd~StYm&NevaIwX5>rF3QFzmo@$7EERKEI?}hr6 z=FIpnPJAK~u^-Zn?lI^E-it=y0fdG&va*$&XK)mL&bHpcMXwYGfh`1#t;CKgbU>i zeWRG9ez1fh;WX2h{H$zvV!a28y8JpH?egJ#>kffu&h8}IcMbXWdb(2 zed|(sY>i!CwL)F|Eho^l19;>80^bp;dLqhNAJOd_;O)rPEX;Q))M>$r+94unYV{~_ zDALVS_OEyh0~Ae5KSY5j`RVFsOjUBR8Nbq~&ZEMQIom!j6MD>b-(4OKnKi**?ZN7- zc!-&heNZ`hTwm{jCZb*L_kBc8dIRt0Z;65Z`JvtzvEYAL2c4VSL@L7v5fUoP=0rH@ zp9}rA{ptJTab!kG7S8t*Dpv_49Kg=&u{9hhseOG)uP~VPw+VL{23ybiM*!`3?>91c zGhO&GlMZWsFn-x^VUh|xobBxA6{*VGA2^il2Jh-OwHO!)GCmS1=v^ERhFHH4iKAQ{ z=t~zfFpy_RkXd-=hGwZk!X9nhnC(GL-9`!1UICq~nF&BJq#91>4Sb_|lbTHyvW|@N zq_-R>w;rQB(_3SOhW*Pf1kOcd&PCp@TdhCTZvk=Mrc~)Lr29#Kl?QNFnOt3GaDi?; z9LXs1-UZdPBD)8G7#QX_cF|Qpj%@ABPa3&fdn&ww-_nG=AaRKCCGCDF!f2EW5ucBH zkfTV{>e>jokFA9S*$U+wZJ+5*6$c*|T|GUZT&?*A9~HX@Lm-bb_y=hm{{uAeJVbix zZl%+Y7pmgTV^{{_YMu+&M*!HyTXVmT>4#_X$FJ=hA;bas6B);I>LG_bXqu`looSat zd(-JE^kR@tFJJ(;504KwZ%z?6p_TDD0=X3Au@zn|;;2Nf$siN+7KuH;jP_UKbO=d0 zx_PCQ24sGmabg5`*pHcVK_}SD%TqtR)d5hNg}wzq*+#VJbjhQr$sKko%)8dRKC6eQ z1Kl7%8ENd*%v1u!AvSY&`P};z4&@g;QInBa1tUHSJyl^KR!k;OEk;GPt!C40MdOPd zwJ#PqdwAt+{R+So5Fa3;Ir_)46=fD2EpWoZ1D1ZJBB`OXq2=G2u{eMtxC^@6$I&Z7c z|8ZL9-LI#c9A2a~Kz&|~1~pQ#kA{Rw^Y*N{DB(?lR;pv@GxZuES=ENQlY#&AjR{WZ=0Yq%V z1rqYlGQl8g+%-VGLzuR1s&96(3I&5X$RF8zVE%b5NeEzZTpqo#`miw?RV?HSmcJHH z7JOrCe{Bw$9?}$Y4u)$mC9TOyA_8+@Y+temAj-r%!#RjMj#6L;qRt)xpw8NRV|#`> zUXt)~hkW5+)#Mm}&L(n`C}ph^ZcuRbS6JeQV_x|!6iya=M#)H<^h?w~zL1W3c^!*J>Mzz8esR$H z${7+l99nX_!zS!6(sDq=#YbeEqAodQblZiBlQ##TJKuao+c2_m{O&~3UJ;Mi2FQ|t zJMr8mSPY_*H9de${tL#HBt}z}58_N+fWi{5`G@)pi5Qb z+Z#cq#$a-|nAhET+~1o*iNSQ%IPaCR+5F%1k6^V^yxPv3V4(a>rW--~EanyB{eEcC zcq4sYJPrr`BdcFG2f*Pjf1(rUZ(ZOD({?qz7#%g%+wWxi*vc{mXUnZYf8V|U@Bnvk?5r$z2u!=q2Z;0Lr{fvS zv)~m|@`Bl=qO8BFk*D-8R=3TR|S!3z{^S8Vn&_`zQN1Lk)=mg6|p|Ja$0Sq7m>~2^%f!ZUx-rVeBDf^Q$ zd3XrUv^3&_Y#-hujMsVt-ySi8r=x4T>+iTV zdN}rH)%>N_X!#6i9z@Hjd<=gZ8seGS#1;1POjXQ7L%#x^Z-T@cxoz6A>yaYLpwBdT z?27^ux_dc_s3uSDwfMEU@6OVjl)cey=!IRWtk*o)O(r%vnv&-d(|Iex(4jVFCB|a5 zuh+S~@scnu_kWrf*mU4nn>Zv?RBQwV6Gz?e$N%0FgIss{e#0C$I2rI;mu~GYw6@$V z-CYRRg8TI5K6&Za&1Vk3v9mb-34mrX@0p!DE4mpk76w2zf%EwfAh9q}R+f0{JS5NW z>np6^nl=P{q==gT5p#%F)LbKvKd zEz~r1{9{lEYqq8kAJZM@%u}#%2Lw(Y}F`GT~q+PfvY^e%kSYX7kzJ|wh!r9HT( zCKLayO3w3w&s!?n!oiLY)W7I3A38f%bM8xwm5c{U9!8dqQ&3Uasy(M#eoggy?x<=z zQ4{B9q<3*w@{=0{)cGeB%Q3CV0vCz7cTT5e>)99!@a+vQ2U=1+3J!S)OEB# zO0@?*FLF6O&eMT?pmuez7$|~bDj0-N57v;IpW~k)w;4GdBMQ@u&6&4;O*p8YkOGbiXGRAn! zUw=W$eQ#{}%UA1VpU)j`?2l$FPA0C|m&ap!q#jgcqb|p}6GvmVB>3214$&-$0O@_P z^RWB@&HrTr&Indk0T4P54IlMcrG781M=BJM=f>VFg?+>=O1=FGxUUKf>uh=w)Dzzh z&YiiZAm-s-E-bV#k@qqKO(lc7t{bY44QCs%1G|r71T6Nv6>stQbi32t zfr2G66~dZW&m|(Gk%Fm+A7DI+b#D9J#^Uu_mQ}q&^RrZ_x_%wPe+hi$dg|B{Ri*rW z-j`VhbCyv7eFdD#QRm8Ggf77^6Ay1Lu#ttPZ~@o5&o|)(U`u;O{AaeAsZ*g}DYRWC z{i{DKVGZi>{Br+xOLuGr*6*vfql?%v?Ksm_PO73s1+o4Q9PI^n5zr^f=GBaGI<5lf zB|6)jBjmxJlR!@iU>(ro^!BdV%6zNq%A^x5dM@m#xqN?b1aJl=ESQ0w3}{FexIf`@ zBi|bQaedo*FCMCyhAC#A@WOhzRGDXSI}C%EyUDJcsgG*e@E^mQC~oSx%Gc>l`!&c2 z!^E?Y6+AxRi$=*8w4GXe$!$Kd$96gQS3OqN6jq(Ub6YM@t!>eZ**1m@g%&*iat}A8 z-Ca#9|6z$u+ZUoYGt)@_{=YFtUgu8ts)~#Rm`DDt{=B@De6y7F+ zT>4YiMm6VI{tAHz!w8dznxh*$3!GQ3lZbidX{D8@ra9&Bm+!(c0zWYO(@KXN5Jgo< z)?naqxP|rn>>G(U`dOj|(zc7$yqh7ov&QF(8o~uPNC9BzW9P1uQd58$6V4i-O0O`4 zFXpsdW_dg#)F#cT##c63cZzPS$B0EiRJwI%0D-NwhOi5Md)0raS`DFA)OQQ zbE6NK5+#{FL5tQPm|MV%Q+YzNxYTAI0?pz z!sE&35JHKA3q!|UJ}ciCCK?j;C!9fF=g4Bb0W=23hE@d8c{AIM>-Dj%oxLeKz&h%j z51@4TgjvnQ>)2fTCWwFHIlT)ZrJ)z6Oif~XuB7bO+PMnMg7%iG>%GXoLnsj=T?=Pf zFeefz+ZsP$&yvP+kuJw!nGUt;YLVaX1X&v8q(Sobq54zJBHWcnh=7R#SM+;p%WOaw z|JLj!A1uExpH1O4{W~adda@VA*j}4}*1VK}$R_+>Z4^CoTiySZKI^WIixLk03e|w7 ztZ1J*1=UwTDAI7htKdWL7g$$I8tD$%Ig_c8IJkSHNRb*6>)VGKOf85gvF=t=�y2}{Dqa&^PNkMh3ck#^-C{J!hFWb{4{pfb+W(tH@(tN0E6kBy zYtuicRedWtq+cA!%7C|#9PeCGEEUocLkcGxY*v~_X|FP1j)2}nhsrMb-Btv9Y*lw$ zCn}8iN10s3UuS?iK_Va&xG$jIjATndDJ&M@HDG6I-a12Gx5#e%01UyuG=?RDeB)SX zzbB-hgmKXGj##dOhT1+hFL&QTe?; z$-kG*wRf$Z1`~#+GVBv7uYF4A9%|~noDf6*E;}UB?~an=qX?tkJU2~lG}ro&ZhZab zE^GV`=7o_7DC9`7=3RY{LJ-_r9H3$)Cfjm>=M8{Nxdvn&YC;Bh{PG_MF$b;JLxeo0 zMo*Q4m5U(TJ2TubYOdrRzBn*z8*n+CYBqpgDu5(mo?)f*0L2eXx!rI1>96H^;E|pU zb4=}h+0{C9xk-ErUpA#Dsf6M9`*_RspY|tr^#9rD8?nlV4}FDb};oEMPF z`HTA{XaC`zNI*V)j!WC7Qe)(z6@k$~ywVpb^GhA4dn2Sj9JBbB&uQ}uz>(ZKROs(| zZ45tHEF3dmClUKMv|t2pBlA)cAb4!&hh#BK2m%vA+GAPYOEdVIgYJXy;@!Mzk%0cRt z$^CT)Koy$OmjPVuC1_?0%B{gmzPb_-cdI2)NBINA+o!jtBfvLllZ(>=;8=M@K!s52 z7_q5|if4V&7a;gjDv_aKFknqhQDe0$f-b!m8car-Ul%$F;!EK;?YpM6^b_D9B$ z4f=m8CP1^O58}oV&*9-%KL+t3Uenl^Z9n3Rt=4xGsk|>K_gx8=m%B^Q@5a z*TkU>?}>PNBP-$l2M!M}rDK>&zGs1O+YtE5Pc)9%tKGt>2jwdW*6Cf=t=&J%W^}Q=(fh|J*7Ren=%a6E01s zE&F4U!t!>~V_VA|W^~s46ZhlNqR}MD%S>ccL~KNH1>Ue~CihF1617vbEIh>y`22S{ zs02zs81e~$pDFz&HW)YI;U}mvkwu@;23#!CHnj=is8G`yeU+_zmOgua!q3XL{QWIo z@Z0l6hpi7it5KJkA6-!g@+MG=cd>jHF>8J((?6~RvNNmqYyrJ*<7wi#HAWJ$KYv2= zRVTxuqd%Ki%gW0$U2$r~QRd|3ou0G2CwofcdU9xVYVb$CB*FG#-9rO=KVfa_Ji#$B zdyU$^nhsuLy>l~=9m(2g?VI2i!f#_ncY6o+x!yubw3CMXRAc@w*`w#ve^dhyQisri z+_2yNN;&q*doz(M(Iv^GLkAn)8Hpy&ySt?fzIZ`Q>=p&wN)%yzle7LmZ)hhb-)|9e zWUHE|Q1I#HpE&7{iR}g;&bjV-31t{id_L78xfDSgdf4k)jbwY*%0${U9MoafZH;oQ z_jcG_9J^+XD70e0Si2i-=tr80fz#^6?+@^vZx;(o8#V-_oDa~w?6`-2EQTZ|5?(Jt z!w-rrG0Ay%!iK{`>d72@Oe&bL{@m;G%2HPnoyO{_6N*E$^nQQdpKH0n?yF1DKjhO? zd+g=KF^t<~tw-V#oFi((j!b`J!+7vk>hyFwiQ61~6oW`Kfl-+UyCz!EFi7|GmIfSq zHnL$1-xO*2d7#?PKliZX;IQuiTo-F>m##ZCn5-kfzT^0bU{wC9?}zeC4)DGD?VZxH zVo|i6>*6zFr?U6a6^GpAYYZqVq{H72<-&Rn4o8OtP_XR`u{fE(ch)9UX}3nB@r9GM zN2Nk{XP=&_TFF7Wt=#UC2c%{;*pKox_fyW-){WPfmeuyiVN$zN0_h{^%SYQgf7FPb%fzA;4^J*mDcLxYScI_ha*?0<=B|FG$c~0K z<=NWxM7=V)m0`-=58=G|eb4GnWPVj9Ra??*feL!!+T@H|pEy0c&gkiiywyMX0Y590 zZj|vcBQ=2b+N>&pGBkvHK+Eq~I4)~siP*y6E#r0cx2Jz#Zm2)8kQU8LKfU#7xcka*?<^SpI zs>7n{zV%oj(jgs6N~h!i5+Wr?hjdHl(2Rlzh=6p1QYr{Ycf$|^j5G++J#-B)1Kcy; zckl1H_usq!n=|K}XP>>-de^)5I{Q5VR@HO@=z!nr>FWishu{;b@A@DJoo);PZ1qj2 zYaRJy0h#^T`VC{@dsdd@+b*oE4K@Mqt-Rv2BO{Yo%&v1XzH|)Ty*C~CGcl1~Zuq1P z<-?m!gY&!qqb!rH^e6mN*6RIsQ1bR&Q_GiK3)gQWX3`SXSLqB6t<~+UbN0(ruR2?} zI=bMI{lfMdkg$$t`HVjJCO1I|Ksg+Bp`|7bY!%Yr+IkGp!4>V{7q9NFH6HD6Bq>0Tdq$+G31yYbG3 z@|9D!ogw&i_Peat7FNI>6qLqpDDUitk)*0}X<0@-_h8@5?;RBSE$f@(@ zW_n1Fz(OhkH!C|$+Icz?IySAWmDJQ&~KZ;jMzLt&7>uq}XO8-uaYUp^aKYRZhoHKeZbSs^#Ec6s> z5^(Nz_0cz1zjT4*YcU-?R$r>e&cuIYIU`xMQdJH;k1w*(7>roOVnQA` z8I!3{JxDb)t9H8}CHe`51&=k7yc&vIyY5v@_)RNZ5kaKinvr&l zzuwgqdc^vInR(gF*+?|cfQhA2&F>wOjg<}$&pj9QjZ2P<)FZ(tQ6%Z3YpF29{!_fzd$yTY%^2jg(kN?r7r@=VEZnvfCETq?Ku{IGC4N%RydI z?8^TeRXF`5Cq8a3OU2y2+HIbbdYFi3$9uG%`H)tggIU{naQ=F3&I?2;c4o1X!0G&2 zVs^`=0A^2e+-At!^uEe%D>?VMWUkauUDGS#gviLm<}2x^-DP)8gT85v;TVbBN5H0V z*#{4GJXx8wHJ+Eoezm;SwR1v{5UDI-Pb84|^)ZC4$TfziEV{1~na~^U+Gez}rKmVi za;`vC?=BP`N%XY*++RK+N>y&A(4-i{i5-H_k+H}Tn@8MT!L&yo^W=7&VP~3!uj#l& zUk^*gJlhu`BmHqd&!CrN=5(M%`9H(GxV9fvc-NdPkhfUBA7<@q=Z@ypZ^iUy#tVDS z-uoZ9J$*C0)D5Til_)G;{CL!v@IZ5N>V zcU2U-@g#JDQg;p-f85DjrlL2_L%ZXK%pXi=Z-GNn!yD9E{1(?Nw1uxPO|ZS)uWfL~nE{N96_J4Xv9xlkMw{t)+KcP> zj?n(7yXSuzpbn1!!|P9^kBU_8@p2#XC>FW@SRA}qi5TXf!ShH=-x&{MWnRAP2GtZQ zTcX+F%MJ-_58J{xIegI(RA5wNlNf5h&SxaZp=j?|p_Sxd_T+6iU!La8-R-Xecf{l< zXoq4i=Z6YUI(g$XseM~G6AJrFR4$th>#tYCWm#>he?DGo*{f02#{-u3N(rtQSL@mU zKwL|!#JV0DU;yF!2unDbIvvkucl=EvMM}z%b_c9>!d9_Y7Vv#Y@T#^0G&M2A%)!}s zzTr7<2PDSNj{ANl@D)Wy)^n;QB!8di6lz5ua)V!Gt^09aNNy7<1NAf9tZ=(0(jj%!|c*-@ZO&---3-iUV17Z8U10J=tI6`s0BJ zR{QgW=U*%n-$>LTC3E25EOdsiCNLJzPB;lVERqxjiAVJYF2waD$lciEHd&1^GrM-} z>>F7Ieon$#x>1xFgMv+}Br7|BgHO;Kp8@XavpX6F>>#lNls*i9Qp%)Dhn=}dEw8Pu zeH^-TV8djdv$KLs5iD#d&x3(4>%GLlmwR)vMc+N*WcHS7Pdom$HJPC@>e)_`bOe1N z)$aB$qICojh`BD|eOFjl$-x>Di2tso*~q`!b6nZOrK8C}NBSP7w_G(eY&>FwBHQN1 zOEsf*IWD}pIDB*1>KloP=k~XBowfGwJg{<@U#1K+nM=2Fu{r5G{At*bMMwwr|Q~o7XtiKmDKDT8<2?=?02Y>i!7V8b?1tdGOy$FqqBer-XEnWIEn%VW)soWzD zIY+Mz5VPw}2aHW;y4DjCwd-`8=dG?JWi$i%grg8Ok4wQSA~N_C@#y06GCnIS$==A= zn5(tbfX`+aDhaxiV@E9M^P#r1(u?w{)POgy@m>=H?cMBN#bJflxthf6qDu)omDN^r zEkS$qD|>5R8`K8-+yY;B;(la>_?`x>FbIyp0=xvcq!t*S-bZxDzDt?1UEacjNHq(y zhE0o)=~=6@-w~D4#j6#p%nctqWW0clm&TKifraqn=B_9ti{9N!6me=rF2SFgUbCB4y4H z5tggD4wVDm4`S_he%uFN_g4S1VJ?XBRkrzC#B~8v?y;|0A5T9CD*x-rs})T0R@b5{ z&%@d9gmUW*o79EzTZt;eDZ4ud4hUUJey$KBpvw$0L6NYSj9>n6B@pK7&UgJ%Nx|MY zGmk5^Ca1A6%PnFF=uy?F)Gy2KNJ1$br9p!Ax~D(xrTq9mQ}XZgIGO`{%R1eh9yA*I zRbG4H8yPJw-4dm17V3#(zSa z-9QaL2D{`)BFLXyH$5;tBG4|)s7XILi}=JSl{Gz0eu9*`AJ)+xsKZXt^p=W{XBSzMLR2kwwKEne@5YUwQS{}Yq>?UlP z`sQjxB?p9njDr6;$wU=kf7s^9DaU0mxiFdRw$8tPjSi4Vp@~t-*%N*X{IB!^6V31-D~7crGClRrbLuB`KxHf0b-(tnD;zJ_+WtuNz(Wb+x36p`_M_ zu3}#VNNZX3J}T3hJ6lrqk=vEO(4A{^*NN6w&%LCHrsT2ie7#RbYdmzA9v8>oWk%Cq zXMWYkA-pkKz3$CBQ_qc&C4MkC9_=R`p0fhI8|D2oPpti(&yBAB_Br1p>ANXdb23z zB^lXpmE-9V7zvDzQtKI|ERR|HM?LtKXQ1!Ap9?*LU^8aOF5huNAjG^bMP{8NJzvKzl^7GO)lp$S_zD%a_q^{=0@ zcAl?{l3em(fHy%J!fH!rk1_ZNiz<6zVEMW`JH~oqJ6CPxcftLSvofm&RO5U}vOT5v za&jHbA}3Ghb3{^t(I|J6?7r=0beEscIX(zH>n1>{hSLHRoc`2w(VW8ktYVp3eg>1G zUd>&C1a^|OFxO~Wr8bGsqL&$mqray%Q9I-XMMd9Zo^Y6WAYPTb2Alcq*}`H2vsqpc zc>%h6f+syo&6sVAFIIo0zT^I&H>z9mRf=U)l9vy)PVcbz5{SxC&%BnQ;$tOaVnWp8 zvCK62Qh|KWntKnO4hV-{f6hu6)P$#gPEYSgv|$uIJ;{bLYd~KdCyKMA=?pG$*`J>s za_C%m4--^Y3NgStyO7H($llO!^Z599Iq#E@4>5l6 zCN_qWiGet^wr_G*%An@vW`~8cJGp~7asdIfT3TqMnr-_=?>((cT&{5Kxl0QX+9-*T zB&HOePRE+cD@19QgjWW@AZ`QpmAE`FZ1|H#Jxe6z`}ZpwJ7c4oORDYnQ(n#`%>Rzl zObrqNPYM6 z?`)6{3!|%Oy1VJ`S&Xd{#^T%Rj$d=T0!1_#H0nmr;B0U9rDDrvne2U!^yOOOi-~(R z_-mRT$j%Du7vhkeNS3n?>%8=*EZ}BFK(CXTjcw-eVm`R3A-~s@&0krYpr@2rG&2VE zt$8<+5buZLnrgB{2s5mLw}x~N@eoA-7_?+Sa&t>CpdpH$t__a1e*JK3UJg%Nf{KDd z-O!Mbd3(bC_i^xjyt3}J5poHiOA0#CYYkv>^YXp;oI^rFSUEV>qBPYsyLzXH-!4GS z;js*}(~Y~t#rlM{gMEi+DFvhZ>z zgNM{^0@Q~>fol!7RMpga;BdOIj>x~R+*=AHR31gaeQ|%nKYso0^VKo=RZg&-zzQtx zZu?ZPnC2{qs+^^@(35_d_IW8WBO_z;XgC}v)mu|y3 z(blWq4c>k9-T(3!vStF&r&P*+_hana=Y)jLLvlet*0s*t0gALm78MmOBU4e#l++~3 z0md3)+hu>H4BM7`g@i^WLNL?|JZUe`5C*k1@ot~Ws>(>2`rO`JP2*(*vPEAYvT;G&vP^x@r1T0cKyn|Fn2&(2G22OopB- zJfFmXQ+4*hJNH)FkePa>AREH7hIB1c7)7wOMfUJXZr0&fb=nGu1dwuD?SwSzUctfbUEfIMm)W#0=;ad-lp+NA=s} z1vkZ-ES%toy3F5ymI)}?&0~sg*8v)mLOFiT{K?z~>^#-9n&2}EZhb!PnB{2v|2(+5 zsQ_&ZL7)#SfIGm8Pl7o)c3|lt5TFcpUH3-%>hi32b~Y24Df;d86ywhepXccenc|3< z+9jYbMV4JiERz6(!3QTI)&i?@^2ISv7P1=b9p*MA&7dv>OEq1BnM2iXQ}MrxC+HE} zdIf>F3dfXn(<{rvNj?(Yc!)E)!G?2jO1?|lzO7DrVAHSI{O{0{69UTMo2RmTSJxja ze2_{4ev2gA<1vfVBV9oPr>4DK^*}Fi&Y(F&yKZbOEZIG>Ay}YJ{^OxC^76((lu11} z;tuZFEfTHrTX!yHHCq}9lPl?n-v421sD@YK8(PNClE98t$rmIdiz(< zMciPv*f$$ud6PWx-8&{`*f?kYyc8eE_-wj=jikD{wI^mpO2-1z!A}s9o@rdY+!?MY zgSMhXPTuH`MzHkgj$e{clk{Qu`ORfqU=%4gnEXGHp5Mh$1aT(;p?iP#Sf4zJv=)4E z;eB=jPL$Ps9~c3UGq!&fiXk_V(Q_=hGwIkRV|^=g#kiCc*mS(# z8fPKokM$e2#x`?(KI^d}Lbk@Z(cgt$zBvJvOy+~xFLPUog{{|d=SEie_O7NBY)z}V zk5I&mInepnlRYF@g6VPa1bLf+m6tc!|G3R!`P`QQ{-<{9**?c589hZOu$-TMc!|uH>W?5m(FuN1vBo>=1 z4vU(m&e;`9D=1Jju5kpUfT_`EyOu{%E58hRL34k8;J4&%hUJzs;Zp`b-+0}0IQd>o z+-KWnD?|cnEkM0_rD#4+dxN7jf(rr!v~kJj8M}TzW;wr(aI2WzeLqEH|GPc)yFneJ z{czU$ie^!z6N>23w~dXBhiVPzGU(FU+ECNJ=9g36U5BwE=^vb&*jPTD$N^9S^4(dV zPP?j+0u^CxodntWz1+I1e2dyr{HC40G>QxnvR%K!fj7Mu!WsaQx2i&E~l{lmdvA z(Be!70o3Kmi#Ud*!xy;)0jJv<44O!!aJqnaxUpred?|GKQ~VOsW!0NH`*+Cjh$%K+ z>7jUfQ;NW&y9#ClTknL^!PFO*ev~1ohq5bE-|O z)VS7eXS0q*);x|MgDfT%6c(<} z+0>#Ako7$8lXDiO zJ6?w>RoU?hWDbE8pEE5U#D@X8;d=%ECnbC~#y+H?9BDt1VGcUCMzZG-E3Gvx_#ki8 zR8=3Or>8s5RnJ#s146h}Rue~aSN6vfVqSvjS#ZH@l~|Bt_)MCB_*yaY-o1NX*Fhb4 zo2M@`;}bGE0p@!bTl|`7bP?X2g}DE$avV|xM6sJ7`+y<=HhX^bO^i#VI>Wjpzc&`( znzA6QzC18d%SDvD$2$mTQ7`zA3&4`Sg@lEpr2VrVmZXebrkok@n9FTU&3HY*0+cqz z^KCgH^uxJK4>lZvaAO1lk&Cd0hD|tjSXET|%-YFh3zNScNah+VMKWNWzW1_rhv)Vk zKp;=$f;+x%V6c-#%pVUAZ%KV_3yhWk>b0%5$|@~3QbtghZ=Qt%iZ5<_bX;qUqckJ= z$309bs4aXQgFy<9TO0AI$nfxf=q^eEF>mQ)YZ?sK$N|e0s_N0-)EWxWL+fUDr}-fZ zGrQC=?7jAfhxW(L9DIDSpFa=yr`)@Tv$C=R-!VF-hks%xM^2WvcwRgZZyU~0z|L%Z z-V^Q6lUelZH?#XVnBq$c$E)$GE$6J7e0P9k3Anp64QEUF&e#L}<}nn{gRbbd#zr24 z)7eII8Zn3vQVav@P#6JXb<{ABB$OVq;D4HJa-&L?=NdUZWBh4+jL#tDT@<1itX~Ro zy+Mj%U};+9=RUit8r?aIUvU{dE~!O#&kBymhS~JO0bz@aW!YSzorw|kT)}W4H5o6~ z%e;kd}nQ4j!L7{*iO+P<(iU+_6s++ec*MqJr#?XLFwcN&R;{o!|M77gw z{DI^tK;@lse0cQnGezAB5IR>_^J^ZQZ$7}lF~9sC1IGJ^g(a;03rVr=4|$-y+nCbZ zc!do`1&>^%qyH{2NI~Ae5iSz&?sMk~*Ihr4vK~ykSkhbhsm~2y*lL5E!q%)k2sAc-uP;f|26>u*$B8Yu z2q;KoSNc*`IPE5=78K1EW2MY;OB#;ey~*z63TDxXj-4p)ifx|H3#$_wVK& z-m`jfX}Ff|&Be~^53S(Hs*O1gEv`2}w8*oL7zQpX0@gLLZCU8+SIfPNJVW1yl=|5! zl)Z5(yf6VT}4*n|0e Date: Tue, 19 Dec 2023 14:14:14 +0100 Subject: [PATCH 3/3] refactor(blog): improve dyanmic scaling post --- .../index.md | 440 ++---------------- 1 file changed, 34 insertions(+), 406 deletions(-) diff --git a/chaos-days/blog/2023-12-18-Dynamically-scaling-brokers/index.md b/chaos-days/blog/2023-12-18-Dynamically-scaling-brokers/index.md index 85166222c..ad8a2fedd 100644 --- a/chaos-days/blog/2023-12-18-Dynamically-scaling-brokers/index.md +++ b/chaos-days/blog/2023-12-18-Dynamically-scaling-brokers/index.md @@ -13,7 +13,17 @@ authors: ole # Chaos Day Summary -We experimented with the first version of dynamic scaling in Zeebe, adding or removing brokers for a running cluster. +We experimented with the first version of [dynamic scaling in Zeebe](https://docs.camunda.io/docs/next/self-managed/zeebe-deployment/operations/cluster-scaling/), adding or removing brokers for a running cluster. + +Scaling up and down is a high-level operation that consists of many steps that need to be carried co-operatively by all brokers in the cluster. +For example, adding new brokers first adds them to the replication group of the assigned partitions and then removes some of the older brokers from the replication group. +Additionally, [priorities](https://docs.camunda.io/docs/next/self-managed/zeebe-deployment/configuration/priority-election/) need to be reconfigured to ensure that the cluster approaches balanced leadership eventually. + +This orchestration over multiple steps ensures that all partitions are replicated by at least as many brokers as configured with the `replicationFactor`. +As always, when it comes to orchestrating distributed systems, there are many edge cases and failure modes to consider. + +The goal of this experiment was to verify that the operation is resilient to broker restarts. +We can accept that operations take longer than usual to complete, but we need to make sure that the operation eventually succeeds with the expected cluster topology as result. **TL;DR;** Both scaling up and down is resilient to broker restarts, with the only effect that the operation takes longer than usual to complete. @@ -21,12 +31,16 @@ We experimented with the first version of dynamic scaling in Zeebe, adding or re ## Scaling up should be resilient to broker restarts -Scaling up is a high-level operation that consists of many steps that need to be carried out by all brokers in the cluster. -New brokers need to join the replication group of the assigned partitions and then catch up with the leader. +We start with a cluster of 3 brokers, 6 partitions and replication factor 3. +If leadership is balanced, each broker should be leader for 2 partitions and follower for 4 partitions. +Using more partitions than brokers allows us to scale up to more brokers, distributing the partitions such that each broker has less work to do. + +For this experiment, we introduce chaos by letting a random broker restart every 30 seconds. ### Expected -Even when brokers are restarting, the scale operation should eventually succeed with the expected cluster topology as result. +Even when brokers are restarting, the scale operation should eventually succeed. +The expected cluster topology after scaling up is 6 brokers, 6 partitions and replication factor 3, leading to 3 partitions for each broker instead of 6. ### Actual @@ -36,132 +50,42 @@ The current cluster topology queried with `zbchaos cluster status` shows 6 parti ```json { - "Version": 13, "Brokers": [ { "Id": 1, - "State": "ACTIVE", - "Version": 40, - "LastUpdatedAt": "2023-12-18T11:16:56.452114035Z", "Partitions": [ { "Id": 1, - "State": "ACTIVE", - "Priority": 2 }, { "Id": 2, - "State": "ACTIVE", - "Priority": 3 }, { "Id": 3, - "State": "ACTIVE", - "Priority": 1 }, { "Id": 4, - "State": "ACTIVE", - "Priority": 1 }, { "Id": 5, - "State": "ACTIVE", - "Priority": 3 }, { "Id": 6, - "State": "ACTIVE", - "Priority": 2 } ] }, { "Id": 2, - "State": "ACTIVE", - "Version": 34, - "LastUpdatedAt": "2023-12-18T11:16:35.399362365Z", - "Partitions": [ - { - "Id": 1, - "State": "ACTIVE", - "Priority": 1 - }, - { - "Id": 2, - "State": "ACTIVE", - "Priority": 2 - }, - { - "Id": 3, - "State": "ACTIVE", - "Priority": 3 - }, - { - "Id": 4, - "State": "ACTIVE", - "Priority": 2 - }, - { - "Id": 5, - "State": "ACTIVE", - "Priority": 1 - }, - { - "Id": 6, - "State": "ACTIVE", - "Priority": 3 - } - ] + ... }, { "Id": 0, - "State": "ACTIVE", - "Version": 44, - "LastUpdatedAt": "2023-12-18T11:17:22.000373993Z", - "Partitions": [ - { - "Id": 1, - "State": "ACTIVE", - "Priority": 3 - }, - { - "Id": 2, - "State": "ACTIVE", - "Priority": 1 - }, - { - "Id": 3, - "State": "ACTIVE", - "Priority": 2 - }, - { - "Id": 4, - "State": "ACTIVE", - "Priority": 3 - }, - { - "Id": 5, - "State": "ACTIVE", - "Priority": 2 - }, - { - "Id": 6, - "State": "ACTIVE", - "Priority": 1 - } - ] + ... } ], - "LastChange": { - "Id": 12, - "Status": "COMPLETED", - "StartedAt": "2023-12-18T11:13:47.548696276Z", - "CompletedAt": "2023-12-18T11:17:44.207162216Z" - }, - "PendingChange": null } ``` +The above is an abbreviated version of the actual output, which contains more information. All partitions are reported as healthy and leadership is balanced:: @@ -190,145 +114,40 @@ The scale operation succeeds and the newly reported cluster topology shows us 6 ```json { - "Version": 15, "Brokers": [ { "Id": 1, - "State": "ACTIVE", - "Version": 47, - "LastUpdatedAt": "2023-12-18T15:30:19.184655204Z", "Partitions": [ { "Id": 1, - "State": "ACTIVE", - "Priority": 2 }, { "Id": 2, - "State": "ACTIVE", - "Priority": 3 }, { "Id": 6, - "State": "ACTIVE", - "Priority": 1 } ] }, { "Id": 2, - "State": "ACTIVE", - "Version": 40, - "LastUpdatedAt": "2023-12-18T15:30:19.111616286Z", - "Partitions": [ - { - "Id": 1, - "State": "ACTIVE", - "Priority": 1 - }, - { - "Id": 2, - "State": "ACTIVE", - "Priority": 2 - }, - { - "Id": 3, - "State": "ACTIVE", - "Priority": 3 - } - ] + ... }, { "Id": 3, - "State": "ACTIVE", - "Version": 8, - "LastUpdatedAt": "2023-12-18T15:25:07.484956052Z", - "Partitions": [ - { - "Id": 2, - "State": "ACTIVE", - "Priority": 1 - }, - { - "Id": 3, - "State": "ACTIVE", - "Priority": 2 - }, - { - "Id": 4, - "State": "ACTIVE", - "Priority": 3 - } - ] + ... }, { "Id": 4, - "State": "ACTIVE", - "Version": 8, - "LastUpdatedAt": "2023-12-18T15:25:50.616089815Z", - "Partitions": [ - { - "Id": 3, - "State": "ACTIVE", - "Priority": 1 - }, - { - "Id": 4, - "State": "ACTIVE", - "Priority": 2 - }, - { - "Id": 5, - "State": "ACTIVE", - "Priority": 3 - } - ] + ... }, { "Id": 5, - "State": "ACTIVE", - "Version": 8, - "LastUpdatedAt": "2023-12-18T15:30:16.949863252Z", - "Partitions": [ - { - "Id": 4, - "State": "ACTIVE", - "Priority": 1 - }, - { - "Id": 5, - "State": "ACTIVE", - "Priority": 2 - }, - { - "Id": 6, - "State": "ACTIVE", - "Priority": 3 - } - ] + ... }, { "Id": 0, - "State": "ACTIVE", - "Version": 52, - "LastUpdatedAt": "2023-12-18T15:30:20.920293674Z", - "Partitions": [ - { - "Id": 1, - "State": "ACTIVE", - "Priority": 3 - }, - { - "Id": 5, - "State": "ACTIVE", - "Priority": 1 - }, - { - "Id": 6, - "State": "ACTIVE", - "Priority": 2 - } - ] + ... } ], "LastChange": { @@ -337,7 +156,6 @@ The scale operation succeeds and the newly reported cluster topology shows us 6 "StartedAt": "2023-12-18T15:12:57.790824149Z", "CompletedAt": "2023-12-18T15:30:20.920657536Z" }, - "PendingChange": null } ``` @@ -376,145 +194,40 @@ We start with the cluster topology that we got as result of the previous experim ```json { - "Version": 15, "Brokers": [ { "Id": 1, - "State": "ACTIVE", - "Version": 47, - "LastUpdatedAt": "2023-12-18T15:30:19.184655204Z", "Partitions": [ { "Id": 1, - "State": "ACTIVE", - "Priority": 2 }, { "Id": 2, - "State": "ACTIVE", - "Priority": 3 }, { "Id": 6, - "State": "ACTIVE", - "Priority": 1 } ] }, { "Id": 2, - "State": "ACTIVE", - "Version": 40, - "LastUpdatedAt": "2023-12-18T15:30:19.111616286Z", - "Partitions": [ - { - "Id": 1, - "State": "ACTIVE", - "Priority": 1 - }, - { - "Id": 2, - "State": "ACTIVE", - "Priority": 2 - }, - { - "Id": 3, - "State": "ACTIVE", - "Priority": 3 - } - ] + ... }, { "Id": 3, - "State": "ACTIVE", - "Version": 8, - "LastUpdatedAt": "2023-12-18T15:25:07.484956052Z", - "Partitions": [ - { - "Id": 2, - "State": "ACTIVE", - "Priority": 1 - }, - { - "Id": 3, - "State": "ACTIVE", - "Priority": 2 - }, - { - "Id": 4, - "State": "ACTIVE", - "Priority": 3 - } - ] + ... }, { "Id": 4, - "State": "ACTIVE", - "Version": 8, - "LastUpdatedAt": "2023-12-18T15:25:50.616089815Z", - "Partitions": [ - { - "Id": 3, - "State": "ACTIVE", - "Priority": 1 - }, - { - "Id": 4, - "State": "ACTIVE", - "Priority": 2 - }, - { - "Id": 5, - "State": "ACTIVE", - "Priority": 3 - } - ] + ... }, { "Id": 5, - "State": "ACTIVE", - "Version": 8, - "LastUpdatedAt": "2023-12-18T15:30:16.949863252Z", - "Partitions": [ - { - "Id": 4, - "State": "ACTIVE", - "Priority": 1 - }, - { - "Id": 5, - "State": "ACTIVE", - "Priority": 2 - }, - { - "Id": 6, - "State": "ACTIVE", - "Priority": 3 - } - ] + ... }, { "Id": 0, - "State": "ACTIVE", - "Version": 52, - "LastUpdatedAt": "2023-12-18T15:30:20.920293674Z", - "Partitions": [ - { - "Id": 1, - "State": "ACTIVE", - "Priority": 3 - }, - { - "Id": 5, - "State": "ACTIVE", - "Priority": 1 - }, - { - "Id": 6, - "State": "ACTIVE", - "Priority": 2 - } - ] + ... } ], "LastChange": { @@ -523,7 +236,6 @@ We start with the cluster topology that we got as result of the previous experim "StartedAt": "2023-12-18T15:12:57.790824149Z", "CompletedAt": "2023-12-18T15:30:20.920657536Z" }, - "PendingChange": null } ``` @@ -552,125 +264,41 @@ $ while true; do sleep 30; zbchaos restart broker --nodeId $(shuf -i 0-5 -n 1); ### Result -All 6 partitions with 3 replicas each are evenly distributed across 3 brokers: +All 6 partitions with 3 replicas each are evenly distributed across 3 brokers, leading to 6 partitions for each broker again. ```json { - "Version": 17, "Brokers": [ { "Id": 1, - "State": "ACTIVE", - "Version": 54, - "LastUpdatedAt": "2023-12-18T16:24:24.619548623Z", "Partitions": [ { "Id": 1, - "State": "ACTIVE", - "Priority": 2 }, { "Id": 2, - "State": "ACTIVE", - "Priority": 3 }, { "Id": 3, - "State": "ACTIVE", - "Priority": 1 }, { "Id": 4, - "State": "ACTIVE", - "Priority": 1 }, { "Id": 5, - "State": "ACTIVE", - "Priority": 3 }, { "Id": 6, - "State": "ACTIVE", - "Priority": 2 } ] }, { "Id": 2, - "State": "ACTIVE", - "Version": 46, - "LastUpdatedAt": "2023-12-18T16:24:16.029577221Z", - "Partitions": [ - { - "Id": 1, - "State": "ACTIVE", - "Priority": 1 - }, - { - "Id": 2, - "State": "ACTIVE", - "Priority": 2 - }, - { - "Id": 3, - "State": "ACTIVE", - "Priority": 3 - }, - { - "Id": 4, - "State": "ACTIVE", - "Priority": 2 - }, - { - "Id": 5, - "State": "ACTIVE", - "Priority": 1 - }, - { - "Id": 6, - "State": "ACTIVE", - "Priority": 3 - } - ] + ... }, { "Id": 0, - "State": "ACTIVE", - "Version": 60, - "LastUpdatedAt": "2023-12-18T16:28:35.518105574Z", - "Partitions": [ - { - "Id": 1, - "State": "ACTIVE", - "Priority": 3 - }, - { - "Id": 2, - "State": "ACTIVE", - "Priority": 1 - }, - { - "Id": 3, - "State": "ACTIVE", - "Priority": 2 - }, - { - "Id": 4, - "State": "ACTIVE", - "Priority": 3 - }, - { - "Id": 5, - "State": "ACTIVE", - "Priority": 2 - }, - { - "Id": 6, - "State": "ACTIVE", - "Priority": 1 - } - ] + ... } ], "LastChange": {