From a7c4aa2a3ca19a41279673751705c8b67d3679a6 Mon Sep 17 00:00:00 2001 From: Renjie Liu Date: Tue, 14 Nov 2023 16:56:06 +0800 Subject: [PATCH 1/4] feat: Add shuffle for iceberg sink --- rfcs/0077-iceberg-sink-shuffle.md | 67 +++++++++++++++++++ .../after_shuffle.svg | 21 ++++++ .../before_shuffle.svg | 21 ++++++ 3 files changed, 109 insertions(+) create mode 100644 rfcs/0077-iceberg-sink-shuffle.md create mode 100644 rfcs/images/0077-iceberg-sink-shuffle/after_shuffle.svg create mode 100644 rfcs/images/0077-iceberg-sink-shuffle/before_shuffle.svg diff --git a/rfcs/0077-iceberg-sink-shuffle.md b/rfcs/0077-iceberg-sink-shuffle.md new file mode 100644 index 00000000..923780e4 --- /dev/null +++ b/rfcs/0077-iceberg-sink-shuffle.md @@ -0,0 +1,67 @@ +--- +feature: Improve shuffle in iceberg sink +authors: + - "Renjie Liu" +start_date: "2023/11/8" +--- + +# Shuffle according to iceberg's partition spec + +## Motivation + +Apache iceberg allows users to define partition spec[1] for a table. The partition spec defines how data is partitioned and stored in the table. For example, a table can be partitioned by date and hour. Also, it's required that each data file in apache iceberg can contain only one partition value. In our current iceberg sink implementation, we don't do any shuffle. This means that if the table is partitioned by bucket, we will have a lot of small files in the table. This is not good for performance. We should shuffle the data according to the partition spec. + +## Design + +We will add a shuffle operator before sending data to iceberg sink. For example, let's assume the iceberg table is partitioned by following partition spec: + +```sql +CREATE TABLE prod.db.sample ( + id bigint, + data string, + category string, + ts timestamp) +USING iceberg +PARTITIONED BY (bucket(16, id), years(ts)) +``` + +If we don't have the shuffle operator, the writing process will be like following: + +![image](images/0077-iceberg-sink-shuffle/before_shuffle.svg) + +After we add the shuffle operator, the writing process will be like following: + +![image](images/0077-iceberg-sink-shuffle/after_shuffle.svg) + +This way we can reduce the number of data files in the table, which helps to improve the read performance of iceberg. + +### Implementation + +There are two possible implementations for this feature: + +1. Add a new `IcebergPartitionOperator`, which calculates the partition value of each record and adds it to the record, then asking the dispatcher executor to do hash shuffle according to the partition value. The plan is like following: + +```mermaid +flowchart TD + A[Source] --> B[IcebergPartitionOperator] + B --> C[DispatcherExecutor] + C -->|partition_value| D[IcebergSink] +``` + +2. Extend dispatcher executor to support iceberg partition shuffle. The plan is like following: + +```mermaid +flowchart TD + A[Source] --> B[DispatcherExecutor] + B -->|IcebergDispatcher| C[IcebergSink] +``` + +I prefer approach 1 since it's more extensible and does not change too much current shuffle implementation, e.g. other lakehouse sinks (delta lake) could have similar approach. + +### Caveats + +When iceberg partition spec only contains range partitions(e.g. year, month, day), we don't need to do this shuffle, otherwise all traffic will go to same sink. + +## References + +1. https://iceberg.apache.org/spec/#partitioning \ No newline at end of file diff --git a/rfcs/images/0077-iceberg-sink-shuffle/after_shuffle.svg b/rfcs/images/0077-iceberg-sink-shuffle/after_shuffle.svg new file mode 100644 index 00000000..0d372796 --- /dev/null +++ b/rfcs/images/0077-iceberg-sink-shuffle/after_shuffle.svg @@ -0,0 +1,21 @@ + + + eyJ2ZXJzaW9uIjoiMSIsImVuY29kaW5nIjoiYnN0cmluZyIsImNvbXByZXNzZWQiOnRydWUsImVuY29kZWQiOiJ4nO1d2XLiyrJ931/R0ed1W6fmYb9h3J7AgCfAvveEXHUwMDAzM1xuM1x1MDAxOfB4Yv/7zcJcdTAwMDaEJlx1MDAwNJZs9d2mIzq6XHUwMDE5pJK0cuXKrKys//7x48fP6cuo+fOvXHUwMDFmP5vP9VrPboxrTz//NO8/NsdcdTAwMTN7OICPyOz/k+HDuD77Zmc6XHUwMDFkTf7697+Xv7Dqw/7br5q9Zr85mE7ge/9cdTAwMDP///Hjv7O/XHUwMDFk51x1MDAxOTfr09qg3WvOfjD7yHEqTdzvXHUwMDE2hoPZaVx0XHUwMDE1nDCF2OJcdTAwMGL2ZFx1MDAwZk43bTbg01atN2kuPzFv/bzMP1x1MDAxN1x1MDAwZVxuXHUwMDAzmt87zJ9cdTAwMTSKv64rvUJhedaW3eudT196b9dUq3dcdTAwMWXGjjFNpuPhXbNiN6ZcdTAwMWT4XHUwMDFju95f/G4yhDuw/NV4+NDuXGaak8nKb4ajWt2evpj3XHUwMDEwWrz7dlx1MDAxM/76sXzn2XxDIYtcdKSp5Iv3zS9cdKOWIEojQbhrMNlhbzg2g/lcdTAwMTdumj/L4dzW6ndtXHUwMDE406Cx+M50XFxcdTAwMWJMRrUxPKTl957eL1NQi1x1MDAxMMGdp+g07XZnXG6fcWlcdKaYc1xck+bs3lx1MDAxM0qlQlSoxVx1MDAwN+aco6PGXGZcdTAwMDX/Wd7wca3fPDK/XHUwMDE4PPR6zns2aLzfszlalnih7+/8vbwo8/1fbpw5sbaCt2nzeXmlXHUwMDBldPCX58PHQfX1krbuRJ5cdTAwMWX1uVx1MDAwMOgvvvf3n/6HffvxY7VULGlbUnXSbJBcdTAwMGU7XHUwMDEydq66epb5+Wvj8dB53Pd/LW/Lw6hRe1x1MDAwMzFcdTAwMTZaa4okXHUwMDE1WNDF5z17cOe+Z71h/W6J+z9cdTAwMWNcdTAwMDNe2FvYVS6O43uT3nEoLIZcYiEwII1cdTAwMTRBdFx1MDAxNY6CWIxT7EDKXHUwMDFjRFx1MDAxY1uUUS2VUlhRrplcdTAwMTdLJMxcbuJcdTAwMDL1V9v31kaw8sG7jWGwPFx1MDAwMTdUXHUwMDBiP9Z0PFx1MDAwNVx1MDAxN2sqXGaPkFC2XHUwMDFjVjBrulxya2VcdTAwMWNcdTAwMWWUXG4zXHUwMDFjtFx1MDAwMUqXoDNgg3t7/uZRlrdmOJie26/N2TlW3t2v9e3ey8o9N1x1MDAwN8n07La5/p91XHUwMDE4cXO84lCmNjinxVx1MDAxN/p2o9FbwdCkXHQjnp1qic86nKpcdTAwMDbvjo+i+I/h2G7bg1rvXCLgcszxXHUwMDBm54jHXHUwMDE24b5mXHUwMDFhyS1SrIJcdTAwMWUwmJokXHUwMDA0sehusTbZuc88oFx1MDAxMS89XHUwMDFlNXdGpP06tZ/SYzb+bpEpZSGlV/3P3C9cdTAwMTK3w4zbL0psYcklXHUwMDE4XHUwMDEzR+a1RGdcdTAwMDT3iJmS8D5Wv5d/nHSqvfa9qp5dvz7eT072c4eF4mNU/5iT9o0mzbvsXHUwMDBlmex1bXn2Sz72ovnHNX53Wqp2SifV4/Nuub0rTlx1MDAwYvSwXHUwMDE1w3GfM5O7cW500y/ky9Viv0jF6dVVbP5cXHKCP+zP/V21w1x0XGJcdTAwMTnoXHUwMDA0lFx1MDAwMH1GNpDO/lx1MDAwZj/tXHUwMDFjobmFqDSGKimn3OFEZkzBiYUkWtEscVx1MDAxM1x1MDAwNbBcdTAwMDCRWnOKXHUwMDEwhDJaaC9ROFxc8TtBgNymXFyAgI5ZOniIIFx1MDAwNKNUXHUwMDAzfshcdTAwMDZcdTAwMTjd3mubm3hUb942x+3/XHUwMDFknJtcdTAwMTPF5NRd7nuNn3O77/ch/VhcdTAwMWSRx4n7SVxiykJcdTAwMTT49f2ujVr85rRyZb/2+EDf5PiJV4H7+H+DacG0tVx1MDAxMo+Z+8lBm0OMQChhXFyBXGKXXHUwMDFlXHUwMDFjYiEtRYRG4JI06Fx1MDAwM628QKTMXHUwMDAyTsBSM85cdTAwMTGllIeY1z9ek0dxvHN/XHUwMDBmWlx1MDAxZCPMkUezmedcdTAwMTms5LhkXHUwMDEwcEH8XHUwMDFmu1RXjGsunZjePqAs7eVH91lxWeA2blx1MDAxNFx1MDAwNznV6GVcdTAwMGW9cG72evZo4lx1MDAwNjNcdTAwMDSNgDaFpOZIKEfCYIZqLS2qmGRcdTAwMWFTwZlgXlRLXHUwMDBiXHUwMDExJFxiwVx1MDAxY1x1MDAwMlKtsFx1MDAxN9SYWUjjmbcjUnDB6DeqN0E1XHRENShfarjCk6AzPo1cdTAwMDSiXHUwMDFhXHUwMDFllaBcdTAwMWGU+zawXHUwMDBlkK5vSKyXy+XcMW/v9GlWSXTxJKZcdTAwMGY8Ls2mOFFik+g2VLN5rMFx6zRcdTAwMGW6dYKBkiHa4VXXybbTbKZ0tqPudveeaveszLrFo4vz9ODUX7ZcdKUtXHUwMDA1bkzAXHUwMDFmXHUwMDA1wdVSM72xgrBcYpJcdTAwMDIgxjH8Jznt9jF2eTdcdTAwMTPgLs6wcqSxXHUwMDEzNc5NJF48XiBcbqRcdTAwMTVcdTAwMGKCNDC7IIIjx/1Zh2l8Uc1cdTAwMWRcdTAwMWM3XHUwMDBmni+Holxc61x1MDAwZo/H6KqVdkxLTC1OOMZCUM4x41x1MDAxZUyDSqCKXG4kXHRLN6Ix+GKtqGSf5G9SXHRpXGLoXHUwMDAyZZvAWiPpyO6sQ3QuV7tcdTAwMTb3V6eDm4dfbdG4b1xmT7LF1COagjqTmCgsTLBKqFx1MDAwN9FcZqJcYlxmPE2wdkxcdTAwMTakXHUwMDEz05SZOVxy+llcdTAwMWEqnZgmgXOtWHJcYmK03lx1MDAwMNSH8Ivrbm9/8noyvcrybr990yinXHUwMDFl1Fx1MDAwMltCSlAgXHUwMDAym+BLukAtLaExl1IqXHKRWcqlXHUwMDA3hphcdOIqRP7RoGaBWVBcIoGdOJfRpcdkcnlUvzLhLj1o0vPhfb1+cJd6TCuTXHUwMDA1VVx1MDAxY0kmKUVEeDDNqJRMKOBylXaehkuQjCVTWZBcdTAwMDZEv0eylWaxVXp5ebxcdTAwMWJcdTAwMTau2pl83t6rsCiT9MpcdTAwMTSFXGLQXHUwMDFljHCIWF1pQoGwedZcdTAwMDL+ICRcdTAwMTDVnodcdTAwMDRQXHUwMDExRFwiuMPAcZRRnywhPCTw+UxqpCVaSSR+51M8IPObuVx1MDAxN8ZcdTAwMTaVM7vvmNdlgS6YYlxyyJeR2CrhbOBcdTAwMTKI76lyYuTfj9tcdTAwMDf40vRcdTAwMDf2n8BcdTAwMDfoQfyjKFx1MDAwMfdqQlx1MDAxZLbyrXVcdTAwMTP6vWZrdZJrNfM/XHUwMDFkjvzn8pfPcHUyYOXq3Jn/oMuJcVx1MDAwMl9cdTAwMDZPzoFaxpJpXHUwMDFhKUH2xlx1MDAxN/dnvcvx691l98G2s78q1fp9IS/TYz5cdTAwMDFpXHUwMDFlpixTwMIgJpZaqlW/RDWywFx1MDAxN3CCXHTDoMyFa1xcMTqmXHUwMDBmTY28XHUwMDFiNVx1MDAwM0SDK5WJpHlcIkzphzomc3GfIbVkXGKmleSabILpXHUwMDAzofartNhv15uDh2FcdTAwMTnnLmntLPWYXHUwMDE2xFwipixcdTAwMTXiXHUwMDA3JVx1MDAwNGcrmGbgfzUjJm1cdTAwMGU3hCqSXHUwMDFjpuNcdTAwMTBbXGauQ1xijL5KbC1+45Pr712eZ67zMlstjlx1MDAwYoX9s5uxnjzk48v1I8Q+IzhcdTAwMTEsMIuEsVx1MDAwNKtFztTpOos5Q3dcdTAwMTlcdTAwMWLtT4ZcdTAwMDc7Jy3Rb91fULuTeotR2lwicJmIU1x1MDAwNFx1MDAwNKrdXHUwMDE2gywpwFx1MDAxOWJBmUp/eKIxXHUwMDAx56BjKNyIPzz5JC8gKFxywjRBXHUwMDFjwjfCSHQvQFFcdTAwMTXJl/LZcSNbqrRl5W5Y2WmmXHUwMDFk01x1MDAxMlx1MDAxM0sxkFxuilwiXG7g9UBaSC4kXHUwMDAxicNB1adcdTAwMWTSXHUwMDFh5Fx1MDAxOZMslSH3Z2Fa6iBMY1xuUTVlm6T7a7uDaWu43767QWdcdTAwMGa3k+5xo7PDU49pKiy42YBawDRXrklZo2xcdTAwMTCSXHUwMDAycbBxpkjKlY1cdTAwMTkkyHn8XHUwMDBmXHUwMDA3dWChXHUwMDAxRlJSjjapNGhcdTAwMWRcdTAwMTfyO6ryQHpjaj/22pXC3mkt9aBcdTAwMTZcdTAwMTBkaokpJlx1MDAxMGIq7Fx1MDAxNlx1MDAxZiDmQXWAjlx1MDAwN9hjmnbxQZGGIIulM93/WaD2roeZg1x1MDAxYcJPISVcdTAwMTLRxcdeoSvb+3TnKttp3o0ylcntfV2nXHUwMDFl04pZlIOQ1piZ2VniwjSEoJpcdTAwMDOFg0rF4NV/XHUwMDAzUCOFhfh/XHUwMDBm6qAyfqZDlvpoTlx1MDAxNCXRQ0Q2Kd/sXHUwMDFk3tr77HnwWi91OqX2XHUwMDAwpVx1MDAxZNFwny2tJCVcXJtSIVx1MDAwZqI5IJppwU1cdTAwMGXOmciOXHUwMDFi0JpYQjGBXHUwMDEwyHoknZZcdTAwMTVhcuRcdTAwMWTPVGhQT3HMX21d2b8xbJej+mg6/6cnue7QvVvn+7fO7DtOXHUwMDFlqaxcdTAwMWajUFx1MDAwYlx1MDAwZU/44+DiXG7CKVFcdTAwMTBFRJ+I7lx1MDAxNseX55hcdTAwMWXq/f74pJg93S1cdTAwMTbrO2m3Y5Pwp+CBXHUwMDE5ldRMSLtcdTAwMTZcdTAwMTBzYfGZWSVYVlx1MDAxMUeqX1x0XHUwMDA2XCJCyESSPFuv3vvwMrv3f4VcdTAwMTFcdTAwMDdcdTAwMDVew06TSaxcdTAwMTApOC1cblx1MDAwMo5cIrZJ0YbOifJuc5irjPRwdCdcdTAwMGVG9uFcdTAwMWVJva1cYmxBdGLWrCmIxlZdXHUwMDFlXHUwMDAxXHUwMDE4c1xiSIjGXHUwMDEyeIOJtIs4TIDf5JdNJIRjWrBPwXRIXHK0UpqYXHUwMDE1itGj7ee7RmHcvXm66NyOdskwczpcdTAwMTh3S6nHtFKWueumn1x1MDAwNShcdTAwMDaBXaBcdTAwMTZcdTAwMTC3MFx1MDAwMlAxzVx1MDAwNFhyOi4mTHNMMOKprFx1MDAxOP00UIfM+Fx1MDAwMjtpRET0tChvXj3Y3Yvd0SFmapi9aHWrp1x1MDAwZmnHtEn1U6qFWa3LldbSg2lcIiA2oaB9gP/SztOmf1x1MDAwYuPM0Tvin4dpQYLnr1x1MDAxNDdcdTAwMWRsUPRcZlLmRXeHt6XHm9O6PTzq61IrM6qmXHUwMDFl01RYXHUwMDE4Llx1MDAxNcJcdTAwMTLBTeGGXHUwMDA301xcm1xuadBcdTAwMWSA+Vx1MDAwNFx1MDAwYnNiwfTMoTCaytUqn4ZpXHUwMDFj2Fx1MDAwZlxiw3PEhDrF2TpQP7buMupXjlxy87hHWeGG39snLPWgXHUwMDE2yGKaXHUwMDEwiOhcdTAwMTiTSmpcdTAwMTeoITCkXG5zXHTmzZROufiAqFx1MDAxNFwi5XRcdTAwMTb2f1x1MDAxYaaDa2dcdTAwMDSE6GRlgdo6SIvT5+fL7GX1uTVRk8fLo/rZXiP9U7KKWsqUzWBimu1J5YE0XHUwMDA0XHUwMDFkXHUwMDE0JFx1MDAxOGdcdTAwMTBHpnxKXHUwMDE2YkRFNSepLO2PXHUwMDE104GJflx1MDAxZdy0TVx1MDAxM1x1MDAwZXeOR096VOUkc5zd717vnlx1MDAxZkxar6/XI3YxTDuggYEtjFx1MDAxOUWmOVx1MDAxNsHCJaYltlx1MDAxMDMpOUCalDw5Ma2JZbL0VJg6TU2wT9OetYl+uFxmXHUwMDA1UjqWXHUwMDEyg+1T/ZtcdTAwMDJ3Oar4U/3iK1P9jpN/JNVcdTAwMWYly+rKsK4gnFx1MDAwYmphxJnwqGpJLS4hkERcYlPMtCO7vOg5RywsXHUwMDEw9m/GSizAXCLDmlx1MDAxMkadNWXfa3I81rRJj1x1MDAxM5Nu5lx1MDAxMNX79jhcdMnoaWZStyxSV4Ncclx1MDAxN+tsmoRfvD1cdTAwMWHa7lx1MDAxOYTlv34sXHUwMDExMvvP4t//+dP32z5gNK9cdTAwMWRcdTAwMTdcdTAwMGWXh/nDdbifvdpkmlx1MDAxZPb79lx1MDAxNK6vZMbmvo7JtDae7tqDhj1orz6396bSUdp6XHUwMDE5pNRGRjODwF9tXHUwMDE3OSOw+oO5JVxi5Fx0XHUwMDEwXHUwMDA2uFx1MDAxOIh3XHSH4Xsg0Vx1MDAxYzTWjyV8Xm7lhEQzic1cIjuBXHUwMDExXHUwMDAzXHUwMDFm4lx1MDAxOTBcdTAwMDaqXHUwMDAwb4jV22xcdTAwMTlE416YmjuUMTTTadY8oIdcdTAwMTE7P3PO+Gw1hcmC1yxpLSTFaINsn61uLyacXHUwMDE3rtr1Xu7iyb67QEeV9LCLv0CB2NCCx+bqiP3WdVx1MDAxNFtmwWRae45qRGHczinmNExarms5+mv0i+xcdTAwMTQyw5NbPmlcXJydvpT3WvqrW4PK5vnF6eFN+bKLKrf1/OXp8V0rXHUwMDFiw3Gf6fn4ovlyMtaP5VI9X29QpEW043rI1dtyXHUwMDE0wjOxSTvHUJpcYlxmYULaflx0YVx1MDAwMrxNJrn8n37aKVx1MDAwMuSy0lx1MDAxY0stmfLrOYoszlx1MDAxNU+0O/FWTUeNp8E4nqTS1lx1MDAwMYvQRDgykZ/XdbTz0Go5vV6sjUfX+LrAxqPuQX2k92g0XHUwMDBmj1x1MDAwM3NqXHUwMDA0XHUwMDAyWs1cdTAwMTjfIE9807evXHUwMDFmXHUwMDFmXHUwMDFmu/dcdTAwMDDTSvu0w7Klvp12+8WKWFx1MDAxY3Q/dkdoYLBcdTAwMTafmZNK0HS323BcdTAwMDNCRlx1MDAwNT9BKpGJu8Tce+vu6ah8PT2gw/55u1G5PqXFw9pmblxyMc2100CScGuUXHUwMDA1RnqcS8nBr22wUML3olNvXHUwMDE1yuzugTDi3GfzXHUwMDBmYFx1MDAwNmvWYJsnuVx1MDAxOc1m+4gs5rRcdTAwMDXE4jr+blx1MDAxZZu4tY1h+jG35t54XCJWZ7aG1TfeXHUwMDA0w8+H4Y/4MFx1MDAxYahBQXQxXHUwMDAy9L1B/9TSUfl1v/lU363QQfeZXHUwMDE1s1Od+iZmTCNcdTAwMGIz5Fx1MDAwZVGNXHUwMDAzw8zjQVJcdTAwMTWlwuNBjCazLC8xN3Z3Vnk5PcmMXHUwMDFlmsfl/aO7UTZjXHUwMDE3fkWN+nRH3ovbvn21v3fTb19f7apG/Xz1LFtFk1x1MDAxZLq/lz05eEIn2adyr0Y69av6SVxmx02+8YJwckVcIu5cXFx1MDAwNa5cdTAwMTFTJoll0pqRXHUwMDE5wv/hp54hpIVMi2XTr8BcdTAwMWKkUkUtLVO5NVx1MDAwNpaUIS4wiUHp/i7u/DP2xljj6JLfXHUwMDFiI5Jv58GWy8y8g1x1MDAxMlx1MDAxYvj2zPh5QK5Od8SDzFWmXHUwMDE5XspK8lx1MDAxYmSghaW1Qt5cdTAwMTlEKqklKJNcIlFccv4h9y5Am2md0DqAxNw7PtphmVx1MDAwYtrdo+3d12LvLLubXHUwMDFinER1l7fV04typTxcdTAwMWHbO2N79IAzlfz+bVxmbjgp2bA7aZyzJ3w3tMX+UavfKF/0z+Pbx1KarkZJu3fOgpPQ4PA04HeDXG5e/6efdpbA1Oyrolx1MDAwNVx1MDAwMq7w8+9mr0735nnp8O+KXHUwMDEwTrRzd5B/kH9PNFx1MDAwYr3G36UnXHUwMDBiTXlgulxyfLx5bdBA6uyknbtcdTAwMWbk5dPLTvF23Lq4KZe6qW8jaNJthHq2tuRSWJzghCP47VLQXHUwMDFhgclo527cv4NvV1f4ZFDo3rZ6OX0sLuQ4n1x1MDAxYkRcdTAwMGXdZVfT+9zwVNnk5phcdTAwMWS+1FGLy9h8JcM0+VCYqOBcdTAwMDVcXGY/OS3UXHUwMDA2/VL872babU0zI2mFXHUwMDEwym9fa665RVx1MDAxOU86XHUwMDE23iq1TYlZJ/m13SQk0szBXHUwMDA2v3Vqe42v+PLUtrN9rnvJXHUwMDAzwnq1en6dsV5UXHUwMDA2/Uku91ouPlxyLnZJ96pbeuil3VjNjq5KULLSJWLuXHUwMDFhMYX4XHUwMDFmJzk7+7Ftn1x1MDAxMcc6XHUwMDE2Y/1EXHUwMDA3KfX+8+GF3Tm8Pjq5rNw/Z07axW5kXHUwMDA3edDVuqP19dXz4zGu3j537MxjNFx1MDAwN1x1MDAxOXrc32ijv8BcdTAwMWM0XHTuKGm2Z+BUbdD51/8ppdyWOVwilsJcdTAwMThDhOi3OzNXzNKCKI1cdTAwMTKdU94mSCVcdTAwMWHkmVZxVFv8Ln73M3LQazxSWnLQwTWOs87uZmFmZMM9yVx1MDAwZvltfjq+yVx1MDAxY1x1MDAxZr7ke+XTYbl6k3bDXHUwMDA1JWxJ4p5cdTAwMWV688LMXHUwMDEyRGieqFj+mFx1MDAxNyZcZilsylxmfys/jCvF12rv/tdFo2DXRuO93H6p2oxcXFmcmdyNc6ObfiFfrlx1MDAxNvtFKk6vruKohN7ev6/3w1x1MDAxMmh5I4Lbylx1MDAwZjNcdTAwMWQoqZlcIlx1MDAxMFvhXHJcblx1MDAxZf1cdTAwMWZS6q1ZW1x1MDAxMPtcbiyJX8EyV9RinoLIdLhhXGbCgWFB/kHx7yflitf4pU/JXHUwMDE1v9nUztQ+qu+codtcdTAwMGJcZlx1MDAwMNnv6Gpp1HZcXHXgVmhcdTAwMTAkWopxXG4hXHUwMDE311Sz1cXEpkbKdDGniigmXHUwMDE4Qd71lmy2Y6+imHNtVoT5YVx1MDAxMkzH7E6kXHUwMDE1fFFTR//oua1hi4NTlEKYzazNaOJvl/LVtFx1MDAxMuvmaNrcSmf9zZKqscPM3CFcdTAwMTNjYImURUp/bLbeUlx1MDAxMtNUe1x1MDAxYvuem6xcdTAwMDUvx9NaWj3VljRJXlxu8aZ0blx0neCGaJRbyPlyQHaDtdWrl1x1MDAxNG1btPmirFKxpG1J1UmzQTrsSNi5qtecfZZOXHUwMDEzZjYg4Fx1MDAxZc1pVk4r5GpcdTAwMWO6mMswMSRaWWyzMF5kuVdTfy+Wnlx1MDAxZnibxdJEQDDPuTdHaYgy2HRNplx1MDAwM+Nomctcck2XXG7saI35VUulvVx1MDAxOJz9bI6+5c89mji2JdKX+efCQWFA83uH+ZNC8dd1pVcoOIS6c1x1MDAxZDTlYFHgyrBpqs2W/PBjuZD63do8QFhZXCLtXHUwMDFkZ1x1MDAxY1x1MDAwYpXfWGRcYj73aUAvr1x1MDAwZvTO3uh8vJ9cdTAwMTP3dkRcdTAwMTaBp+GezXlrIy69nVx1MDAxOVx1MDAxNjRcIiyyon+dfVx1MDAxN1Yp6ZtD5lx1MDAwN96GQzglgDysPClcdTAwMTbzWWDtXHUwMDFmKEMsXHUwMDEx4DZK8d+mXHUwMDFjYnpJfT2HuFx1MDAwMGheO3Psxckgbrv8XHUwMDFje1x1MDAwZZ9JXHUwMDBmtWfOLOZXXHUwMDBlKVx1MDAxMLPcdr4wZ2KBYlxctVuHLDA7uDlcdTAwMTXDt0nPXHUwMDBmvFVcdTAwMGZcdTAwMTWzK1x1MDAwM2hcdTAwMDNvu7ZQXUAwaGOpI+1tsaFJb1pZkZBJe0E4+91cdTAwMWN+n6FcdTAwMGLCJ7xcdTAwMWS6YMdcYlx1MDAwM4o1XHUwMDA0uVx1MDAxNGFQbGhcdTAwMTlcdTAwMGU7hVx1MDAwMdZORvpcdTAwMDJlXHUwMDEw3vQhjElcdTAwMDBsXHUwMDE2w66IYN5Iz73sd9H6XHUwMDBlVENggEEtz3LhbyqZXHUwMDFmeFx1MDAxYiqRXFxcdTAwMDKPSOIpXHUwMDBmdI3F01x1MDAwZVx1MDAxMlNcdTAwMWWtXHUwMDFi5IZMYjJDW+2kXHUwMDEyK5P4gNC8lvD7XGYqXHTvcbBKJZhIiPrgXHUwMDA2gtIzOXbh+NqcSoiS2FxcXHUwMDEygzutJVx1MDAxNuGUsnVvKCfBISVcdTAwMTRjXHUwMDEynM5sn0PhZTjsRWbMQii0t0wofSlkvU+KudiLW2Z/dX/6wtRSvipcYjNhqW9cdTAwMTlcdTAwMTSnXGbCXHUwMDEwS2DtXZxgiC24k5zZTVOBhEogPcKVc7vlL2MvXHUwMDE3XHUwMDA2Z+8t0Jc26pJcXM/2YzGFK1hx7PjWXHUwMDFiRbDFXCKsN+Za00guSS1cdTAwMTTeUCqcTFbmat7SrKBXqau+bskj0lx1MDAxMspcdTAwMTU2OXZUXHUwMDAyXG76TrTGRiVqtv8u824oZrRO4OZcdTAwMDUgnFx1MDAxMFx1MDAwN1x1MDAwZYo0n70hk2zaXFwsKSbxotC8XHUwMDE2+EtcdTAwMWKZKMU0XHUwMDA1+WNmgThcdTAwMTJeMuEgld7mPL6cTMJcdTAwMTcyriNcdTAwMTOfil2qtcW1a7mqM65SOihFgym2OKKrlYHfnDI/8FahlWmMQrRvkiZ4jYjSiGlcdTAwMTJtOdamlFwiOFme9ytDKzdcdTAwMGXNa8dcdMHPoJXwxXs/nDM4wrRcdTAwMWU3XFyvkNltVDq+NFx1MDAwZq5AVc5cIsNowVWStFx1MDAxMr6OOZxWKKgxXHUwMDE3XHS8XHUwMDExXHUwMDBi9rZYXFxcdTAwMTJcdTAwMGKx3JWLjslcdTAwMWP3XHUwMDA2lN+kMj/wdu2zmcBC+U9cdPPAfFxyplx1MDAxMMxcdTAwMTLJXCJtd7AprWgst6rmiJlWPCg0r1x1MDAwNf5SxSggVFx1MDAxOMJm+1x1MDAwZW2qmDiT3nxccjfFVLOxR1x1MDAxMipBY1xu70KywnJcdTAwMTQhpszmTsBxZu9n7zw1syhcdTAwMTbMQdCJZW3Cmyaszdpoz+SV2aXdI3CWwZa2iHQ1m15ymNmQ7i119k1jMdFcdTAwMTgzrnSlrt8xg1x1MDAxNUxjmCBcdTAwMTD1zt30YqOxTVx1MDAxYmkkXHUwMDE1cHmRaF5OXGamjcokg1xikVx1MDAxMFxmgVx1MDAxN3VUYy5Yg1iC0Nl88teLo/C6/XBeUZ5F/G+z4sLyj7jMwkp3XHUwMDE13TKDY2ppuas45ptU5lx1MDAwN96KVIRiUlx1MDAxMuGbxeEhbdSpNOlgmkCpi+RcdTAwMDR/fbmcXHUwMDBmXHUwMDEwzWvHicHPIJXwXHUwMDEy+Fx1MDAxNS1idm82cyww7Nlj9ZNcIjRcdTAwMDZ5XHUwMDE0eTZcdTAwMGKZRdJcdTAwMWNcdTAwMTFqllTIXHUwMDAwdfRcdTAwMTnKKLxcdTAwMGJlOINcdTAwMTHLp9pXIFxicFx1MDAwM9PQRla7Q7hl1khLy7Oe9JvE5lx1MDAwN94uwCNGgWu/cn3H9t7u0lx1MDAxZdDrmiWSiJZS6a1cdTAwMTbjxFx1MDAxZt5xXHUwMDFmXHUwMDBlc0IwTlx1MDAwZVx1MDAwYq7aiyHSUliAn1wiQLyYUf8k9btBJs0m4etccsPZRFvSX1x1MDAxMGHLf+lcdTAwMDDYXHUwMDEy+JuA7daYxb4re2JVQ1RcIkGpt4jHXHUwMDAwLLhcdTAwMWRcdTAwMTjmXGI520XHRyRcdTAwMWIuPk2KSDxcdTAwMTg0r1x1MDAwNfpSpYNMYVx1MDAwZmZmXHUwMDBmZY7BojTn0o8sXHUwMDE0aF7DXHUwMDE2XHUwMDFmUkLhrVx1MDAwMlxcg0KmqFx1MDAwZptGSUgwR+mnY1CrnJxcdTAwMTiBhTfPXHUwMDBl3ziSWMqb62aIWVJgfzlcdTAwMDSch9x5cEdxors6+pvB5lx1MDAwN95qUl5KalDkR2A0bFKeXHUwMDEwRqVMYKNI00v962fQfEBoXlx1MDAwYvh9XHUwMDA2hUVcdTAwMTY7plxukFBcdTAwMDT3jzBgMMK1o9vHgi4wjJ1cdTAwMTFfwVx1MDAxM4m8XHUwMDBlhNqv0mK/XW9cdTAwMGVcdTAwMWWGZZy7pLXAqmtscu9YXCJcdTAwMTPzM66IN7jEyKJE01ktOTVcdTAwMDGoXHUwMDBmRuOlsfD+S6E0ZqqIfDZXMnkp4p7MW/T74Ja7rbVztVx1MDAwNv9exFx1MDAxOeNcIk4kXHUwMDE5RGfOjVwinY1ZXHUwMDAzeYxjXHT+XiSQltq4XHUwMDFmVzI85lx1MDAwNaF57Szw91x1MDAxOTxcdTAwMTZZ9Vx1MDAwMG9wjFx1MDAwNLBcdTAwMDbnRvQ4qk5cdTAwMWSiJ8oqzqChlPbyo/usuCxwXHUwMDFiN4qDnGr0ModBXHUwMDAyXGaeomaEmL0mXHUwMDE0Zz5cdTAwMDJcZsPNXHUwMDA1b4BcdTAwMTT8IVx1MDAxOENcZp9cdTAwMWOFrW2Jilx1MDAwMlFOXHUwMDA1k0DDXHUwMDFidIQ53Lt9PS2fjvBcdTAwMTMq73XOerlfey+T9DBKhC5cdTAwMTWGtSXSluBcdTAwMWNxYja7h4BrVYOavcFcdTAwMTiTcGBTdkJcdTAwMWQsXHUwMDExd5uYXHUwMDBmteR4ZzgqqdDkS3e5VEb7fHSXy226R/z09G7YuqdEfN1cIlx1MDAxY2hwtKl4s90/3nn0Z200Op/CXVxcUNLPR7v5tOvFz79as5ex/ZnlXHUwMDFiXHUwMDFia86c4N9//P1/RTp16iJ9 + + + + + SourceIcebergSink2023, bucket 12023, bucket 152023, bucket 16IcebergShuffleSourceIcebergSinkIcebergShuffleSourceIcebergSinkIcebergShuffle........ \ No newline at end of file diff --git a/rfcs/images/0077-iceberg-sink-shuffle/before_shuffle.svg b/rfcs/images/0077-iceberg-sink-shuffle/before_shuffle.svg new file mode 100644 index 00000000..432d2e69 --- /dev/null +++ b/rfcs/images/0077-iceberg-sink-shuffle/before_shuffle.svg @@ -0,0 +1,21 @@ + + + eyJ2ZXJzaW9uIjoiMSIsImVuY29kaW5nIjoiYnN0cmluZyIsImNvbXByZXNzZWQiOnRydWUsImVuY29kZWQiOiJ4nO19WVdcItuy9fv+XHUwMDE1NfZ9PeZdfXPepFx1MDAxMVFcdTAwMDRcdTAwMTVF5bt3OJC+b23wjvPfv1xitCTJJCmwwFrqdu9RZdGuXFw5o11cdTAwMTEx/++vXHUwMDFmP/6eTFx1MDAwN9W///3j7+pTudRpVkalx7//hY8/VEfjZr9cdTAwMDdPsdm/x/37UXn2ysZkMlx1MDAxOP/7v/97/lx1MDAwZa/c7768q9qpdqu9yVx1MDAxOF73/+DfP3783+xP3/eMquVJqVfvVGdvmD01/yqqbfDRbL83+1pLJZWKMfP2guY4XHUwMDAxXzepVuDZWqkzrs6fwYf+vsw8ZVPZXHUwMDFlzyRcdTAwMGUzJ9lcXLJ41clm599aa3Y6+cm083JNpXLjfuRb03gy6rerV83KpIHrXG48/va+cVx1MDAxZnZg/q5R/77e6FXH44X39Fx1MDAwN6VyczLFx1xieXv0ZVx1MDAxM/79Y/7IXHUwMDEz/ItcdTAwMGLqXHSt4Erl2+P4TiaER1xmI5z6nnhZTLzf6Y9wMf9Fq/jffDl3pXK7XHUwMDBla+pV3l4zXHUwMDE5lXrjQWlcdTAwMDQ3af66x9fLVNxjbPG7XHUwMDFi1Wa9MYHnpPaUMIJr39dXZ3vPONeGcDW/M/idg3RlhoL/nW/4qNStpvFcdTAwMWS9+07Hv2e9yuue/UTLXHUwMDFjL/z1kf/ML1xuX59cZuLMj7VcdTAwMDW8TapP8yv1oUNOn1x1MDAwZVx1MDAxZnrXz5e81lZcdTAwMTme7kpcdTAwMDXQf3vdf/61/GNf3pzNP1x1MDAxZmTFaNg5y5xTcqYvY6agXHUwMDE2v+Xn95dGo77/c19/m2/L/aBSelx1MDAwMTFV1lpOuGVKs7fnO81eO7hnnX65Pcf9X75cdTAwMDW/yduqq3z7nKWb9IpD7Vx0qo1cdTAwMTSc2Fx1MDAxOfBcdTAwMTbhqJhniVx1MDAxZig/MSSpx1x1MDAwNbfaXHUwMDE4Q1xyl1aEocRWXHTBtjD9p8X73TKw8MSriFFcdTAwMTA8XHUwMDA1XHUwMDFiatUypUlF8NGfSpNcdTAwMTGhObdKzuFcdTAwMTStNYOCtbCQ30fpXHUwMDFjdFxiNtjc/ItFme9NvzfJN5+rs+9YePSg1G12plx1MDAwYpuOXHUwMDFmst9p1nFcdTAwMDP+LsOKq6NcdTAwMDWDMmmCcXp7QbdZqXRcdTAwMTZANK7CimdfNd+8MnxVXHRcdTAwMWVcdTAwMWSl17FcdTAwMWb9UbPe7JU6XHUwMDE3XHUwMDExl4Off/hcdTAwMTPy1GNyqZiuZVx1MDAxNjlcct33n3eYXHUwMDFiTZg1dp1cdTAwMWL8olx1MDAxMkrjveH+PVx1MDAxOcjTh3R1b8Dqz5Pmoztys9wsXG5jwPzZRfvzYlx1MDAxN8FmXHRF7IJh2rJd1NSjWmpcdTAwMDO2l+DP/G6sYVx1MDAxZakwXHUwMDFhXHUwMDFlp+Zz2cdx47pTXHUwMDFmmuvz4vPDcHxycHyYzT2sa1x1MDAxZo9189ayaju+x8aJVlOfJ/VDxy37uNz0+bxepaNkzlx1MDAxOFx1MDAwNf5cdTAwMGXo1rVlbvlmui5zVnqEa1x1MDAwNL7mkkufXHUwMDEymkmeZFx1MDAxZdG7dUhBqpi2VnJcdTAwMDJKjlllw4InSFDgwCvhUoFDumVbXHUwMDFjXHUwMDEyrG1i9P1WXHUwMDEwNzFdrt5VR/X/6eXxi7ZkJFx1MDAwM+bwXHUwMDE3diNoXHUwMDBlX5f0Y3FFIaO4zCRzscKjXe16XHUwMDA31MpcIp6J8sChpeHYSnNPW2V02Js18MxcIsRcdTAwMWLz1S8+8Y8rXHUwMDFiXHUwMDEyn7C5YiFz9TOIpIpcdTAwMWHGl/m3zOfcXHUwMDA0NDFV8Fx1MDAxZSqlmS92e/4tXHUwMDA0PtZcdTAwMGbcdSV40G9cdTAwMDYt7/y3XHUwMDFmc3zM/vH2+//+a+mrQ1x1MDAwMMSfV+jN31x1MDAxYjKandJ4XHUwMDEy73e7zVx0XFzUKS4oXHUwMDE0WUxKo0ms2as0e/XFW/Waw1nHXHSeaabyPV4s8VxiVUzDXZQgYNoyrbnvZfXSXHUwMDAwb+VbXHUwMDBlI4SDaq8yX014pfsoz41qKYQ3eJ//Ob8/sVxcgVx1MDAxNIexJqnJ27Orm+ZzR/bs7bE8XHQrkCVcdTAwMGU5Klx1MDAxMVx1MDAwNYFvODvDufYsQFx1MDAxMcJiYiSn80t/0yRUac8wZVx0eIlWM/DZwyqFXHUwMDBiXHUwMDBm3Fxuqq2QknDO/1Euv+lcdTAwMGL/dMEhfqaESjLf87l6XHUwMDExJDJ8hvdxXHUwMDBli6U7XGKfNTdyru1+K8lzmshcZoZxdZmVTVrJ9Y5NpbN/XHUwMDE4RnS102lcdTAwMGXGQTxrXHUwMDA1ysRKqa0kyviSeDNcXFx1MDAwYutcdTAwMTFcIo1g4PNKoXz6+Vxy19ojjCjGqFx1MDAwNNG2hoZhTYVHLJ25zFxmU5uC/4PrrVx1MDAxOE1ANFx1MDAwMFx0tEXIQM4sKo/CNTj0klot1sml/1x0WEdcItZ3dToyPMNQXVx1MDAxM0xcbqxcdTAwMWSfncX3T8/3TDuWeCxcckVBtHLpi7w7WFpcdTAwMWWfwTV6VltKueKGXHUwMDEwMndWXkRcdTAwMTftXHJEbkqB5rNKqK2L2HZUwCuWLeVSUH9cImunXHUwMDEytFEw93GgJpGgpoxbq81cdTAwMDbnX/Ti+jh1VE09XfZVodTtXHUwMDFmjchNzXlQM+WBXHUwMDBmaZRSXFxKKmRcYtSaW1x1MDAwMb6l1EyAaXJcdTAwMWLUXHUwMDE0jKY1XFyLXHUwMDBmslx1MDAwYk6ielX2mlx0jfd6fVBcdTAwMWZcdTAwMWaXimp4c9a7vU/WVWVY6Z/Ec86DWlx1MDAxMk8ocOmpwsxcdTAwMTTjXHUwMDAxUFx1MDAxYo9oilx1MDAxZapm1PrcUTcxXHJOIURx/KN8XHUwMDFkNzFcdTAwMWRcdTAwMGUl3jQ1XHUwMDEzcJ8hXHUwMDEyXFxcdTAwMWbUh5KaYqtzMH4+mdzEZatbv61cdTAwMTScXHUwMDA3tVx1MDAxNp5cdTAwMDbYcquoXHUwMDEwgJZQ5Fx1MDAwMHG+0lRyS1x1MDAwMHCOg1x1MDAxYYJcdTAwMWJcYoBcYvvWoOaRXHUwMDExXHUwMDAzlVx1MDAxMChcblxihDc481x1MDAxOF+myzdcdTAwMTiY8lSV5/vDcjnVdlx1MDAxZdTWeJQzJjVcdTAwMWWcXHUwMDEzpkKghmjYXGJcIkG8XHJ3XHUwMDFj03BcdFqI3dTluFx1MDAwMOlcdTAwMTecla+qudrpdPrQ7mdv6vuZTDNxJdYpcYFcdTAwMTV4WoFJXHUwMDE2TFx1MDAxYcmCKT3J8V5DJCkkUYTb0E0y0lNKwuVQXHUwMDAwhDBCL71JzMB9sMRCtObP+f2T+FxigWxZ4Vx1MDAwYkS2klx1MDAxOLL0XFyAsshcdTAwMTSA0pIyStYywTtOcMyB+HowxtD/+3F3XHUwMDBmL5r8oMvLX6hcdTAwMDeYpIYzjVx0dW7Ewqt+VVx1MDAwZdOp1lx1MDAxNo+0XHUwMDE3z/km/cHySpj5PVxcPPpbuLrgOV/U5Wyx/EWoyFxmLUiVNsJfW/ErszQ871xcjp7bl637ZjOevLouXHUwMDBms1x1MDAxOe2O9Cw3S1x1MDAxMFxmeyBcYspSpbTVJnBcdTAwMTJcdTAwMGaaXGKgiVx0XHUwMDAyu3iItnW79FuHXHUwMDE4rzItXHUwMDAw0NJqvZNMz1x1MDAxYfUwLrhaTEe6WsxCkEjFXHUwMDA2XHUwMDE1XSllXHUwMDBlrnmuWy9Xe/f9XHUwMDAyPb7kpXPnIa21xzhCXHUwMDE2NFx1MDAxY9hRsVxiaas8K7mWSilcdTAwMGJ4YbuD9DZcXC0hmVCKki/raq1cdTAwMTU9RJpj0Fx1MDAwNcxcdTAwMDIk1lx1MDAwZlx1MDAxZc5Je79JXHUwMDBlxv3U3klNdWvDXHUwMDBi3my4XHUwMDBlaXBcdTAwMTU9qrmUklx1MDAxM1BwNlxiaVx08bLCNJBcdTAwMTHG/ejBUlx1MDAxME9pt1BF9YkhXHUwMDFkmY9cdTAwMTeGQHS4gZLm5JroaeH8qFx1MDAxMj+9quurdv9qr+o8olx1MDAxOdh7XG7bYGDXheUhQCujuaVcdTAwMTRcdTAwMDJiTq3rgLbgPFx0/6nnN0S0js7FXHUwMDEzY7lY/8y0XHUwMDE0601q/YN6+5ac39+NW0eVxp50XHUwMDFl0JJ6XHUwMDEws8+SO1xcmsCZKXpcdTAwMWSEWs6wIEJcdTAwMTjmuNfBiFx1MDAxNOBp0++NaFx1MDAxNZmJt8JSuNdcdTAwMWIk4mtH2cyeubpnnVx1MDAxMW8+dOpX2cRZyXlMa1xi/5Q0hjJcYv5cZlxyulx1MDAxZOBnK1wijeSMccpddzs4sVx1MDAxYW/at07E28jokFP4MZasj+lEtqXrXHUwMDA3fO8m3qi2XHUwMDA3+1fju2HZOo9paz3OIPazVFx1MDAwMKx9XHUwMDA1eD8xTYjSUnChKFj0T1x1MDAwMGpiqFJfXHUwMDFl1FHdNFx1MDAxME1EIVx1MDAxYZwyJbXaIDhcdTAwMTTjwm3i8K55IJ56z+XTRuO03iOuI9pI7VmpXGIoN1x1MDAwNjFgXHUwMDAw0ZxQRLQyXHUwMDAygi5jpd5cdTAwMTmgLfEgTlx1MDAxNVx1MDAxNlx1MDAwMlBJXHUwMDA192Xh1ji1eMUzV1x1MDAxNpynbVx1MDAxYyy9v8Hm3Xn4386z/1x1MDAxZMx6s4iOm03y8O/NuPu+e63eXHUwMDFhSlbK78o8vGTR58NsZrjpWiWlL1Kcvnx87lczrf1xsU1TjeKwfzNxXoqVUp7hVDMsu1UmYJY491xiSDLIjVx1MDAwNMGyu1x1MDAxM+Kt5OFcdTAwMTUhXHUwMDFjMK93kuH5JHl4zqN7i8BcdTAwMGY1hJJccnI815RddnOVys20WmxcdTAwMGX09fTxbDx1XHUwMDFl0oZ44N1cdTAwMTBAtZA4ZWBcdTAwMTHTOHdcdTAwMDTQTLTCXHUwMDE3XGLuekxcZnqIsm9ecFx1MDAxOVx1MDAxZVx1MDAxMzFcdTAwMDe1NVxmNnlcdTAwMDNvK75cdTAwMWYj3dP9Rnl4JGjm9iwlku2066BcdTAwMDa8elRcbomt91xm9HQgzyOYXHUwMDA3ek9cdTAwMTFcbvFcdTAwMDMoyF2emG4lfjBcdTAwMTDqXHUwMDE4or/3+ZKJnFxyXHUwMDA1+2M0gyBifVQnxrmD2qPMs8eMythpupo+lSPnUc3BeVx1MDAwN7hcblwi0M1QwVFcdTAwMTjMw1Z9gZUv4Nn7/XZcdTAwMTdRXHJv5GBgxW5cdTAwMWOQT4Jq4fPLQqg2goOXt9bwnlx1MDAxN1Q/Pj+Ui0RcdTAwMTfG59Xri0veiOvTccd5VEvhKUEkwVlTzIjAKZPgXHUwMDFlpVx1MDAxNky6pXBTnE/1YFxysaL6W1fHi+jKPE3orFx0Ym1Ii5tnkjlUncuz50fxmOye9U5Oj52HtDZcdTAwMWXEh1x1MDAxNvxcdTAwMGLM/Fx0XHUwMDFkgDSEZlxuvGpcdTAwMDVoZ9Q63vBcdTAwMDGOXHUwMDA32JpvnZBf0mE7r4xn3NJN5lI+1Jpn5dKtXHUwMDFh582Dzt3um3Tn8MJ1RFx1MDAxYsI8TrSe9XTYZYimRGnANDHEf6zqJKLhXHTGwVPiX77ZNHq6VWQqz1xuKTRsz/qAXHUwMDFl3HXjNbbfvEheZlKZh07mNn9z7TygpfKkMtidaZikPFBSK7SHY0zAWGlcdJ5cdNmd22GJx2etgdKAmtF8yYyrXybkXHUwMDE5IVpps5Vejy+Rkud/MCXv++6dp+RcdTAwMTWNNExSSyrhjqxcdTAwMWY9tHKjyzzlh/agOzrJxc9iuVxcec91OVbKelQwhZVoXHUwMDA0Lncxelx1MDAwMF/TkzYwT9nJbLyBXHUwMDEwSIFcdP3O2XhwhiPRbGBbjdzgfMlcdTAwMWWrQqzaP75cdTAwMWHY/qCtUoPmYYI5j2bDPMlcdTAwMTCxylCiXHUwMDE3z5eohkCZUaqZZZg4cT1riV7xN09aXG5cdTAwMTOZ3lx1MDAwMYMuNDpcdTAwMThrI/qpXcmOWrePXHUwMDE3jbtBjPX3z3qj1qnriNbE4uxcdTAwMDPJXHUwMDE5V2DOXHUwMDE1XHJAmntCgOZTXHUwMDAwIKKE2uGZ6XYwPWsslN87vaOjh1x1MDAxZihcdTAwMDHqifn351eoltWb+2brXCI2OKTC9ONcdTAwMTe11vXZvfOo5lx1MDAxY1x1MDAwMl7JwN9cdTAwMDC/3VpcdTAwMWRCNZ+d2HBrQFx1MDAwMbqes5RcdTAwMTDVQ9QnxDdcdTAwMDa1ZCvGXHUwMDAwQmBG7Sb9S/tT2+rfnT7cnpWb/XTXntb2XHUwMDA3zofEWmLTXHUwMDFkXFwsXHUwMDAxUFtcdTAwMTOYtI6gVkZpQVx1MDAxOEXQK8crXHUwMDAxwPtgZnZY9p1RvWL2kmBCS7bB7KWHWnvfJI9FP0M7XFxkb+WweVwinFx1MDAwN7XBuFtcdTAwMWJpOPhbRttcdTAwMDCohYfONFx1MDAwMaRB+LbLkq1tYFx1MDAxYeJGos33nlIjRWSQyCC65pZvMKRGnT09XcYvr59qYzN+uEyXz1x1MDAxM1x1MDAxNeebmFx1MDAwMKk4I09pxZB2SZtcdTAwMTCkQUtTjVU+XHUwMDFj3FXX9TRlhttcdTAwMDVF9EVBXHUwMDFklYznMrpcdTAwMDZcdTAwMDAtsbRErFx1MDAwZulrPd4/ilx1MDAxZrSKsXxqXFx7fi5cdTAwMGXERd95SEvjgW5cdTAwMDZVrKlhVFx1MDAwNf1pnMpEqFZWU60l392JqcWZqoZcdTAwMTJQtGAwpN5oqM/b4CVuwJ3eSl/el8jGUz9Bw0en4/1f/jv5+Fx1MDAxN+mqX+vTdr08Plx1MDAxN8dFxTuJaf7qyV//XHUwMDFmPW2KaVx1MDAwZvQgXHUwMDBlXHUwMDBlI7OocVx1MDAwMeGMIL9cdTAwMTFcdTAwMDJcdTAwMGYkwUBAXHUwMDE5pqKg0lNcdTAwMTCkSEVBTCjndFx1MDAxOSfFP/Om3ila8/Jho/FcdTAwMThu6cApXHUwMDEzedjCXGJcdTAwMDVpXHUwMDAzXHUwMDAx2Vx1MDAwMVx1MDAxMcVvT5zy8Md3wz7noKnAVaw3X+p1vqblqez1aMLPirVY6fZcXJp+Llx1MDAxMVx1MDAxNtooylx1MDAwN648XHUwMDE25nygknrguixnQeTwXHUwMDFl8ENcdTAwMTn1R9FzXHUwMDAyRM5nMck/9DG/fY71Jrdocf0j3321XHUwMDBl0YNp6GzmLl8vYliTWO0jXHUwMDE40FxmiKv0i8L7xyeuXugqPiWpZjlQoYKZXCLkU5KaW1x1MDAwNWE1p8L6lOlcdTAwMWKjXHUwMDFl8yjOpV9KNcs8sGBcdTAwMDLnMFxi7lx1MDAxZivyj4j8/OB3kUVgRCit1cvIXCJEtGnDsnoptUtcZqJbZVhaXHUwMDAyRfzZXHUwMDBioHD+MSH53Fx1MDAxYdnSaoq1XHUwMDFmP1mUuFx1MDAwN1x1MDAwMfNcdTAwMDLX5I9cdTAwMDVcdTAwMWUmi/VtmlrsiYDlh1x1MDAwMLFAtVx1MDAxNLWW1TbzbS10+Vx1MDAxMnBcdTAwMDVcdTAwMDJ2XHUwMDBmSyO5VlqEUbktVqe16mRY2DK82Vx1MDAwMJxOLY3d4CBWjpOpVrtbXHUwMDE3171ER3Vv++V0uuyONoliXHUwMDE2V1x1MDAxMNpcdTAwMDZcdTAwMTlcdTAwMWLxvZJD8E1cdTAwMTb1uFx1MDAxYtTi1EphXHUwMDE0lX9sVuRyXHUwMDEz/yvuVDvunVdHmZityMv65ShGJqVsf13uVF6KZzOxTGxCeoPnq5vbk9Re9XBbnoOAQJN/wLSH6IRcdTAwMTZcdTAwMDW3QPNccs6Hl2+m68ImmWfhWq3AmVoh+nSJkFx1MDAwZsZcdTAwMGVbJ0/diIn95yRLhZ3r/snJXHUwMDFmn7raXHUwMDE4pEtD27XJU4PE3VtlTf2FqdiYRNyHXHUwMDA2XHUwMDFmhfn7XHIjt9HBXHUwMDEx0+D64NyotWW1epOYkv2L2k2ZPI5I4YrSvWPn+89ARD1cdTAwMTn0qF5ccqP27E7N4u8xiyvGQNHQ3bQx7Mw6inZKXHUwMDFjXHJi/fMrU7ypJW/zzfTe87rWsXdcIlx1MDAwZs4qk8Pn6l5ekqfGyWS/kv1cXNaRi8jyKSaZ5EJsMlx1MDAwYmnpZjoucYB1T3JqQOSsXHQzi0vwVDXSfe9S8t5FLVx1MDAwZVxuUVqi2Vx1MDAxNlxupT6LdfxcYmrxX5iN3VOLR2S5/FxygJH9UiiugvipUX85XG54pYfvqMxcbspQYFx1MDAwMlI5XHUwMDEzV2U8XHQu/0JcXLltcd2AiH1eV4BJXHUwMDFktVxyV3bbdVx1MDAwNYJRJt4lw2tmkla7nT8+grZ7zazYhimgt6f/IMv6+73tXHUwMDE17Vx1MDAwMHTmauM+r61HZPouXj2+ml6f35a6ZMhLsnD3XHSoTIyH06ZcdTAwMTZjX3yv4lxmR1x1MDAxM1pcdTAwMDNGdkaivjNlspWGLcpcYrhrws9N5EzH1nZcdTAwMWTZVTwmNrpqXHUwMDFhXHUwMDE3QelcdTAwMDajXHUwMDFlpslsOnV/NyZ5mu+elHk+WVx1MDAxOFLn8Wwo+LJcXIEjqyXWRy9iWoBmYYAyi5VcXH5cIiMna/FQXHRcdTAwMTK2XHUwMDE1/3b7JvODIM3lXG5qT1x1MDAwM2ZSylxyXG5M2Z1I9sbXYlC7e+hcdTAwMWbEyvedk0rMdUxrYjyhOZOCKVx1MDAxZWJcdTAwMTZXgnlSUm6Zkcgs7ji1p9RcdTAwMTRcdTAwMTnkvjmmI2umjVx1MDAxMWax9+dXkD5SvVOV2EvtiWn7qXZcXM51K1x1MDAwN5+gYYt5lFx1MDAwMpg1XHUwMDBlw1x1MDAwZvKKI6SFwqo6tVx1MDAxMN07XHRoZVx1MDAwNLgu1kXWh49cdTAwMDP0irS1kswyTjegMnk2lcyksV+epjvJ7H1FdO7On/LOI1pKz0qL7ZaKKMJEXGLR4LtcdTAwMTJcdTAwMDW/YO+a6y2IXHUwMDA2XHQstJNcZmpcdTAwMWZcdTAwMDfqXHUwMDE1XHUwMDEzLlx0UsNv4Hc8JU7NsPnYeuzmTWF0PGrnXHUwMDBixSvnIY2jiJWUXHUwMDA2uSxDlOJcbkdcdTAwMTFzibWq5jNQilx1MDAwYklntOJcdTAwMGVcdTAwMGWZ+jBIXHUwMDBiXHUwMDFhXHUwMDE5XHUwMDFmWlxiNojYpKl2wPr1o5gpNPfrefp4XHUwMDEwf6LGOM8oXHUwMDBlXHUwMDEw8Vx1MDAxNLjR1lx1MDAwMJBCjOKz+NBQYVx1MDAxNbL3ON8orsUsW+JmSvUjzu5cdTAwMTiJXGZcdTAwMGVccuNa8/VcbiVf8FxcOYjLs55tTVx1MDAwYiaZfmJcdTAwMTNxxoXzM4iNYlx1MDAxZZNSa4FcZlwiQdZ0JfRcZs9MabLImr71XHUwMDAzgd9hX5/z8igrXHUwMDEw6r+P5z98nrelRq0/2ae1nTat9ZhMRGRcdTAwMWWeXHUwMDExsFlC+Fx1MDAwN6v9Mlx1MDAwZp9JP19cdTAwMTSbsXg7lty7fUyWs+Vk3HU5xrlpXFxcYkBcZlx1MDAxZX9rXHUwMDEzONNcdTAwMDNPXGbiKGNcdTAwMTVcdTAwMTOfgFJcdTAwMWOLjFx1MDAwNWhcdTAwMWOzkyTPJ8nFc1x1MDAxOelrMckgfmBmfV/ruXTIr2/6R3my/1x1MDAxOJP9XFw8e1x1MDAxZks4j2nDMNtcdTAwMGWhP1x1MDAxMzzEKVx1MDAwZbGyp7RcdTAwMTKUWfpcdDjFcTZcdTAwMTilajeQdsnXWlx1MDAxOT6syMVbcEftelx1MDAxZJcvmG6mRudcdTAwMTfSNGO1q0rncXhaOFx1MDAxOVx1MDAxY7mfuCTWQ4o8Ky0ybFx1MDAwNTiYpSWewFuiXHUwMDE0/1xmpOLgXHUwMDE4XHUwMDEybZycyfOBoI5MxiN7vFx1MDAxMVx1MDAxYlx1MDAxMZlcZlx1MDAxZnWx91DtXtZcdTAwMGKSpFx1MDAwZouDu9HU+V5cdTAwMTRcdTAwMWOfxolcdTAwMDb3w+LhKGdhUMOHXGLN7ScgXHUwMDE2XHUwMDE34LOQXdVcdTAwMDF8XHUwMDE2TMtV+XiISNgmZeTi+Lg+ysZGXHUwMDA3mXJ7T1xm7031UDjPo6YlUk5x9XIwXHUwMDFhpFx1MDAxNkdMa1x1MDAwMX5cdTAwMDdB1m7nqcUpx/dcdTAwMTPmXCI3z8eBWqxgnDKKwN1cXH942v1pv31aMFx1MDAwZmfkpk1cdTAwMGaLpjvNmZTzmDbUXHUwMDEzilx1MDAxMDxB0iFqcXSowZGGXGJR6k9ALW6Rb0VtZUb+54W0jq7XMuBtXHUwMDBia9T6XHUwMDBldStePdyfsMxtvDsoX13Xr1x1MDAwN6LRdFx1MDAxZNM4Pc3gfuOQbVxiXHUwMDE2aVxi05pZcD+knVGLu41ppIJcdTAwMTdf3/OITMeHh1x1MDAxOcwnp2G4xMhcdTAwMDZ1+fR6f9B8OipfpdLHiYHoXHUwMDE2TkaZrvNwVtxcdTAwMTNgqcHNkjTEKy6tRDgjY6Bxn1dcdTAwMWOuRVxiTbYxh/hLpOO/XHUwMDBisbifjjYgxqBcdTAwMWatUmSDqvi95iNPlE/rnf7d7eR2v9sr3zw5XHUwMDFmXHUwMDEwK808ziS4J1x1MDAxMCMow1x1MDAwMkaJaU9LiJRRn7lPLG5cdJfayt1UXFx+kmS8oJFccmOMYlPSJlx1MDAwN0zVUT5ROWqfd25j1+3MUF+eXHUwMDFk8IrzkDZgelxi1qdxTUK84jhzXHUwMDA0dCNX3Fx1MDAxOORcdTAwMTV3vObSWuSL/uZ5S71ijlx1MDAxYYfNpWZcdTAwMDNcdTAwMTbETK7SOrrNxW9sozSsNOMqkaxVXce0plxmm/E1M0RqZVx1MDAwMrl4rj1pLZNcdTAwMWFCXHUwMDA30I6OQ5pcdTAwMTKLU6/oNopcdTAwMWY+MaYj50JJQVx1MDAxOPKKr1x1MDAxZj+Qk+LkuHTT6LYv71x1MDAwN1x1MDAxZH7euqVPzvfjaZxuiU1cdTAwMDBcdTAwMTDxSvA9RFx1MDAxONNcdTAwMDTuXHTW+lxi7ng0LCyHXGLoW1x1MDAxN8avIHyg4JNs5krfPZb3Wkml88mrcqcxpPV2qVlwXHUwMDFl0Ip4lFx1MDAwYiVcdTAwMTRcdTAwMDA6RCeOgDY41Vx1MDAwZpk9P1x1MDAwMZ24NeB2SPqtXHUwMDFkXHUwMDBmXHUwMDE5XXRcdFFcYnLMyVxyukyNvag8ZtuVw7NJO2tcdTAwMDej6XODnThcdTAwMGZqIzyBSUlBcJ6YUFx1MDAwMVBbXHUwMDBm7lx1MDAwNbU4PcZ9QnFcblx1MDAxN8BccuLnO4OaR1x1MDAwNoiWXHUwMDEySvQmXHJM2YRKXHUwMDE2zzvVQVolr1x1MDAxM/X7o5PbhPNcdTAwMDemILeeIVx1MDAxNFx1MDAxOVx1MDAxZiyzRoRcdTAwMDNEwyxXOJzTfUpxbXGStCFf3pmOnmpcdTAwMTV9ssSxL1x1MDAxZVx1MDAwZlTWXHUwMDA29ONwWi/Vy6e1ulx1MDAxOTZqXHUwMDA3pPRwkHK+pFx1MDAxNkvjiUIuPVx1MDAwMSaJ8oArLZg341x1MDAwYlx1MDAwNJOFlOI7w/M2XGLFtUWula8wXHUwMDA28lx1MDAxZj7xn+teK1x1MDAxM28jrVx1MDAxMueaUWyuXVuIj0p8mM/nuvnM3eP0kd3nXHUwMDFlXG7yxnUhVkifXHUwMDA1XG6LQCyJ9OmLmXghXHImv3c66mpb02m0wCk633k6jSTRx8NcdTAwMDRDQ0U3XHUwMDE4RNzbP7HnZzbT3dPF4n6M7adKe8p5OFx1MDAxYoPeXHKlmlx1MDAxM1x1MDAxNSRcdTAwMTRcdTAwMTezXHUwMDFlXHUwMDEwbrCUekYo7ng0zJCHVZGv72StwrSKTvBcdTAwMTimXHUwMDA1kZtk4YvcXmbVxUX6oHLRo43nSf+BOX+yXHUwMDA0oVx1MDAxMU4+UJrBznOjXHUwMDAymDZcdTAwMWVV8Fx1MDAxYzN2xijuNqQlwEcoLb98XHTPKkjLyDpLXHUwMDEweGyL3qDO8i5+0WrSw7OhsVx1MDAxN0fXlctB/lY4XHUwMDFmOWhcYnYlXHUwMDAwgcFcdTAwMWbEWGtCkCZag3rGsyfn6cSlsFx1MDAxNj7CRTbxXHUwMDBmg7RcItFccqZcbu4yYVx1MDAxYlx1MDAxY//TyajDXHUwMDBmYqVcXLnRL+xVm0+STt2HNFx1MDAwNMNcdTAwMDK8U1xiMeH/4MxYhLRcdTAwMDBcdTAwMTfbKkS1NK5raVxyXHUwMDE3rPm3PiqVK0YuSS7BK2N2fUxnXHUwMDA3j/L+9pbEk+qITse99Enz8tR5TFx1MDAxYuVpSUE/W4W8XHUwMDFhIWdaSaKkNnxGJe44prXEMav+cu/vh2nFVtRcdTAwMGZjmzBGXHUwMDFja2P6cqIrT5V7nqJcdTAwMGYpVagnyomkXHUwMDFmtW5i2lDiIVx1MDAwNTdVXHUwMDEwN4TIxGdcdTAwMTFcIlx1MDAxNiQqI5FM3HHXXHUwMDAzZ7CgXHUwMDEz9eW96agsvL9cdTAwMTcgXGJoXHUwMDBiN5CwTVx1MDAxY48+P29VXHUwMDA2vc5+eW/4UL5J2l4qUXRcdTAwMWXQSniAXHUwMDAyoixcdTAwMTc8RCUuNPOMZIZjg1x1MDAxYlKJ71xm0NtgXHUwMDEyx4Qr6CGxjTa8L5GH/1x1MDAxMkziv+xqUZFJeMH5ZjPi43xQU9njh0GtUJTx6vT+YHjpfFx1MDAwZVx1MDAxZVDiKVx1MDAxY+hcdTAwMDJcdTAwMTLMQ1xc6aDEPJAroXBcXORcIlf61o3Sb3Guv5U7gPxwpbdBYvhZhThA3v1cdTAwMTGi+yu+8HdcYuxqJrVcdTAwMTWzSiBcZmaYjVxcX27vp2fpi/NE9aJz3Lfnd4fF4Sh57LrcKkEhPrI6TJYuhPRwWvgu+Vx1MDAwZTfnXFx/41FTaJ/ZXHUwMDFmm1x1MDAxZfX2ns9NeFx1MDAxNsmepFmkQaNcdTAwMTSgYUCrrO+Vrt5cdTAwMTFHRWNGkmStXCI2RDJcYs9gj1x1MDAxYsUpoVTY3dVYv4t0/WdkpVx1MDAxMSrEzXlcdTAwMTA7JlNaTU/2Y5F7XHUwMDFiYmQkXHUwMDFiwJmufnruIHN36IrXXCLlXm1cdTAwMTVcdTAwMTZXskDBXHIgY6HV0NBcIj5cdTAwMTNn07t525fqsNfDo+nRONE1k3yhcNTKllx1MDAwNpNKTpR8bstSx31m+cC6aFx1MDAxZVwiXHUwMDEw1VJ5XHUwMDAxYrSfwsjAWFx1MDAxMY5cdCZF5Fwi9/Yyet6fWmVGNsOkpeD4Ko35xq2riz+tQLfjXHUwMDFj/8xyXHUwMDAyXHUwMDE4JPKnLjFL1ERn/4TAXCKf9Vx1MDAwNnFcdTAwMDWV1cI6wupcbvvw36WuXHUwMDE2nGmfa7sxr+O7XFzvXHUwMDA1wuF3O+SrPPLf8L19U1x1MDAwMIPF1JxZpqXYYFJPl8p6L59M85y4O4yf1/TRnvP+XHUwMDA1XHUwMDE31kOALXq/+F5cYpE9Q8ROyVRcdTAwMTX41kwtdyxW8Fx1MDAxN2tBXHUwMDE4XHUwMDBlPnfK7f5cdTAwMTV98Wkrf1K+L41zpaurp7jYi1xyXHUwMDFlWXdd+uK753TmJlk7KzVZZ5SgTbVXzda35s2jrpN++dpJoTeNPNqmmuO8YL5B58Ly3XRd2KT0KPKLcmXEjFx1MDAxYXNB4ixcdTAwMDHIM7PIcLh1/mKKpXt4XG5pqOHS1/OzzH34mU9cdTAwMDY9oCyV26hcbn1/KmpTmL7f0OEu5vv3o7Jvo7fKXFz8XHUwMDBiW1x1MDAxMTSBwbWsl5RazVm8etA5jTy0V1x1MDAxMHRrtUmxoCpkeCl+d1VqtXNH1dZd8k5Y5/lcdTAwMTlcdTAwMDVOQDLBmu1cdTAwMTfDqL3dcoxr6lH9knNcIvgzx+xcdTAwMWHWXHUwMDExXHUwMDAyXHUwMDAzWPuOmuR2Zlx1MDAxZFx1MDAxZk354DZZPFaFQsyOXHUwMDBmYue8Mc6sa1x1MDAxZMexaem2MD3o31xc9KticFx1MDAxYTsqtTKfyzr6x1x1MDAwYoZcbr9w/qemXHUwMDFidEEt3UzHXHUwMDA1XHUwMDBlj2c0XHUwMDA386hcdTAwMTk2IPhtzUzqrPJCNOJbN45cdTAwMWG5yiHyJlx1MDAwNNlcctSSXHUwMDE2KFx1MDAxMTKOWmE3sd1GKe5nsY3pcvWuOqr/Ty+PX7RcdTAwMWJcdTAwMTP5XHUwMDBiq1x1MDAxMTSRr0v6sbiitVxmJV9tKCPT036KriCpj6JW202anlZ7+I7KrKDCXHUwMDEzOnhAM/NlqfEkXHLksLbO5Vx1MDAxM6Rf952ueoFmq5+TXHUwMDAytVKEOznqQ1x1MDAxOOKrotp+Onq1z/ljIVx0TFx1MDAxNcOTWlx091x1MDAxNXSb1tz3spckMPNmXHUwMDAxzJzoPlwiJb14XHUwMDAxnyhPXHUwMDFjXHUwMDAyXHUwMDE3/rzCav7ev/x/v3OIYXRcXMxcdTAwMDSnyKO8vlx1MDAxYTm+zrVcdTAwMWVost48OsmOep1hPdlcdTAwMTg5z1x1MDAxYq00gaB02SGXRWZcdTAwMTZcIuAvLDrl1PFcdIaMWFxyq5TmTznfXHUwMDFm5se+j7uRSqmZJWRcdTAwMDPW6GaHqzQ/qdZzbL9Ze6rVUlx1MDAwN1x1MDAxM+eLY1x1MDAxNSpcdTAwMGYsXHUwMDA0RjJ0XHUwMDFjYrBcYmrLkW5IWEu5kkI5PnZFcy1cZvXzMDhkMz9cbtM6ukhHKGZcdTAwMDHTXHUwMDFi1MfaSb11dlpkXHUwMDBm5dRx/yhObXdcdTAwMTTnrmNaU+IpiM00J8iHXHUwMDFl4KmwlnlSvnBYUGmV401cZji0jmOXkYt1XHRcdTAwMWZcdTAwMDfqaEIhKqk2Zlx1MDAwM0w/s1qXXHUwMDFjp1uVo+a40zeSXFyeXHUwMDE2x85jXHUwMDFhLTplyIRcdTAwMGWIoEKGMFxywk05x1x1MDAwZXjHtbQgXGbkzzjJUvFRgF6RtTaWId/ZXHUwMDA2XHLBXdrOqlb8oF27OVx1MDAxY+fOnvtCtdLOXHUwMDAzWlx1MDAxYVx1MDAwZlDEZ6PAXHUwMDE0YVwiXHUwMDA0aHBeiVKCITWL6105UktNdzOJ5LMgmkS2uOONhDu9wSCS1tOgmZGj/dpt9XR6c9xcdTAwMTN346r7KtqgXHUwMDEyhv028Fx1MDAxZii5oCtccijiXHUwMDEyXHUwMDBiy1xyNkfvsKR/K25cdTAwMDd8XHUwMDAysovspnThs2Car5j4RkFtUbVBwqN1Okw9pTLlequUtIVyOtU+Su+7XHUwMDBlakOkZ1x1MDAxOFx1MDAwZdE2XHUwMDAyRJipXHUwMDAwqPmMXGbdYo2L86NmxUthqpM97lx1MDAxZlPYwqKP7ig1uLVcdTAwMWJcZlx1MDAwM09ftlPtRMtMM1dPucc7k+CXOefP7rB3UuD8XHUwMDBlrjiXLHhcdTAwMTZg9Vxmz0xpXHUwMDAy+m+H08CN9JSSsFxmapC5SLynd1x1MDAxMnQ0XHUwMDE4VbtcdTAwMTVO8D98nrel3smP7b+K+u7dzzBUK1x1MDAxMvGAXHUwMDE2rTZpxNqbXFzk9NHecFJcdTAwMTK9brz9dHGfKVx1MDAxYtflWGnqSVx1MDAwMX4lxalcdTAwMWPaXHUwMDA05JhcdTAwMTNsazRcdTAwMTaL9Vxm4zvM8WwjXHUwMDE5z0HKmdjKLNLPm4tfwdtcYrsjJd9krn/hYsxcdTAwMDb7V4nHVuKpTI+zt1x1MDAwZqnpofOYNsKzXGZcdLLRPClcdTAwMTlcYolcdTAwMDX1lFZcdTAwMDJp4S2oSNf5dVx1MDAxNdosqb55XHUwMDAwXHUwMDEx6XBcdTAwMTnFtZBcdTAwMWJkecasevvcfn7MXHL6j6W8yFTuruiF65DWYNlcclx1MDAxNuRcbpw3q22gjFhcdTAwMTBcdTAwMGZdMYLEU8ZcdTAwMTjHXHUwMDEzl7NcdTAwMTFZ8DFfPn5YiejoTLxcdTAwMTRcdTAwMTSH06j1MX1aazZSXHUwMDA1U1H5Su+y98Cux52u81P9QUF7XHUwMDEyIFx1MDAwYspYg9HnLIxp+Fx1MDAxMKG5JXKhz9RFTHMmXHUwMDE0J3499FxyMVx1MDAxZE37JsFcdTAwMWVcdTAwMTNpzfqex2hgjtOTojo4yk3SmZP7xFUjVnNcdTAwMWXS0npcdTAwMWNnXHUwMDA2QeBrpFx0nphcdTAwMDKkNVb2XHUwMDAy3KQwzHXPQyOXu91Kw+XnxbSKTMdTXGJTKNZcdTAwMDKsXHUwMDBmatm9ZcWaObgxPdNcdTAwMWXTk+Zho/BcdFx1MDAxOIU4XHUwMDA0gVx1MDAxNlx1MDAwMislNTc07E6DXHUwMDFiXHLxodSMU+547lx1MDAxMq6DXHUwMDE5Q5ycZfhhmLaRIVwi7Cvsj/DTLP+SnpNcdTAwMWalpsPrvYcz3ddnh3fF4/tcdTAwMGLnh90jo1x1MDAxMIBcdFxmtqSKXHUwMDE5SkOY1sxcIouWVVSL3aUvt3NqalxiXTjo/qKQjkzHR1flg9cx63TeoHUtZSYkU5jmz/P1XFxhv/qcSeSSzsNZSU9cdFx1MDAxY9vIXHSV/sK0XHUwMDE3OEuEM8dcdTAwMDT54kiBXHUwMDFkXHUwMDEwXG5puOnWXHUwMDE4ZFwi5L45Zlx1MDAxYmTjkVxiWvgroL93Mp79wWS877t3noz311tcdTAwMDVcdTAwMWJQXHUwMDE1YZaITYri4893zYuLUa1bKVx1MDAwZupXXHUwMDFkXYnTO9elXHUwMDE4XHSFJMMhoFx1MDAwMENl2KJRMlp7XHUwMDFhaUo1IyBZ1vHCeMqY4Go3o/0/SSp+XHUwMDA1cSOmLJGCc/3QYVx1MDAxMK80zvKTSSxr9+4yXHUwMDBmhe5JLvPsPKKxcZpgcVx1MDAxYZ6xqsBcdTAwMWN0nDdcdTAwMDKqkSvwXcDLcp1cdTAwMDZcdTAwMWTZdSl3kn/l41wih+jJ/qCxhFx1MDAwNqlf39XKdm+aySltTeuPx7TPu1x1MDAwM5I6cH5KXHUwMDAwUlxuXHUwMDE5nFx1MDAxYqYghlCGXHUwMDA3XHUwMDEwrXF2XHUwMDFmk1x1MDAxYUmHrOuIXHUwMDA274RcdTAwMTmkv/7WkI5cdTAwMWNcYkWZMopcbrFBXHJAvS2f24mRfDRjmZTNo/vj/p5wXHUwMDFl0tx4wmjYeEMkyLFcYmNcdTAwMWGiKFx1MDAwYj49mHTteDCsuab+SaXfXHUwMDBm0WrVZH+JZbN0g8lcdTAwMTLXx8mpfa5p9nDZuGhmr8bjUtF93jfFPI6Eb1x1MDAwMlx1MDAxMI30jSFEXHUwMDFinPtcYvtcdTAwMDDYd73cUmrKqfxcdTAwMDb5nZX0K5FcdTAwMDVcdTAwMDBoaC0zXHUwMDFiXHUwMDA0h3yvclqfdp9cdTAwMDfli6tcdTAwMDTtjaaDXCLbc1x1MDAxZdJGeVxuq+GZxkliXCIwK8VYXHUwMDBm40aLQVx1MDAwNaO7XHUwMDFjdL9cdTAwMWT2XHUwMDE1XHUwMDBiMf1WyCo+L6Rl9Khr5O4japNcdTAwMDFAd1x1MDAwM1x1MDAxZD+8zlx1MDAxY/aaXHUwMDA3NJ+clm9cdTAwMGUnvZzrmEZGIUIosURcdGaNXGKHh1x1MDAwMFx1MDAxNK6olIaY4LJcdTAwMWODNFx1MDAwNLA40NmKL+9MR1x1MDAwZrSKPlZcdTAwMDJXWlOySZlWrHIyvD+iiWyiV9qviKfOgb7Uzlx1MDAwM1pcdI8pxlx1MDAwNVx1MDAxOGxccnBcYnjSlnnMSFx1MDAwZVx1MDAwNotJScTuSlos8bgknCpphGSaL5lq9cs8vDCzmZH/1MS/pMJ93fpcdTAwMWaehvd9987T8IauMEvEMlx1MDAwZVx1MDAwMfP68XCbqYeTQeE0UZrm9q/zw1jq8Nn5Xi2llVx1MDAwNyEkXHUwMDAx+SDgd8tAXHUwMDFlnlx1MDAxOC84JNLRXGY8RFx1MDAwZeBcdTAwMWFv4yTt0+bgV/E2SourkVx1MDAxYnhZpVx1MDAxNslcctqFXHUwMDE4K9iD6XWJXHUwMDFj9pPnbefhXGbWXHUwMDAwQlx1MDAwMpxpIHGswVwimqnF6cBcdTAwMDYn11x1MDAxOEJdJ0FcdTAwMTc4n1VY9p1Lh5WJzO9ozi3XZoMhhE/dQad+oFx1MDAxYsxo/ZgoNzTrJN2PhVx1MDAwMUuAXCJcdTAwMDOYXHUwMDE1mOVcdCDaeFQp0JpcdTAwMDYr4l1HNIVVSqRT/daxsI6uspSUXG5LN3E6MqmTXFzlapho5Mnz7Th/1j+uXVxcOY9pLFx1MDAwZbZcdTAwMTBAXGL4zVhrQqAmWiNcdTAwMDW6wOSA4ylLkD9cYnNcZv3WelqzyO5SLa2xhK/FXHUwMDE584rpxLDYS9hsbmpv0qonsrFn7fzxv4ZwWOGUU1x1MDAwYpi2IbpcdTAwMTHANDzMrUJYS+O6oiZESMHo10/wrMJ09LwlUE7MSLXB+f997Hl4cNx5LF62T1x1MDAwZi/syalpXHUwMDE0485j2lx1MDAxOFx1MDAwZmJcdTAwMDakhKaYXHUwMDFmXHR508i0JbWBm2J2yNC3XHUwMDFkSFsjOeP8e6tpXHUwMDExXHUwMDE5IWrKNZixXHKOlk7P4o27TsFcdTAwMTaKRdIhY84vplx1MDAxOefdabDTOFx1MDAxM49wXHS4XHUwMDA1TzSMaY7liMpIwSV13fVgglq4b1//uDQqXHUwMDBmL2X0+T+nhFus8VhcdTAwMWLRJHV5vZ+TXHUwMDE33cLgspqejmtW5Nw/WFLKk5pjp4/ljFx1MDAwNqbxXHUwMDFhxjyDXXqcoVx1MDAxYZe768OzzFx1MDAwM1x1MDAxYlx1MDAwMXuORb3C31x1MDAxZrl+XHUwMDFl3lxiQrndio7+XHUwMDEyiXjqay76+Ok0vi/fRio+sqXFRGbhXHUwMDA1XHUwMDAzR5pvkuPJkHqpdnStTolcdTAwMWFcdTAwMWZlr1x1MDAxZlx1MDAwN41O3HnmNJBcdTAwMTfPaG5BRFxyh2AyXHUwMDEwO0hcdTAwMTBhSlx1MDAwNJ4qXHUwMDFhs8PecInzb4RUmEin3M9ot0lHi4I3K7pccpP0WUU4wOj5XHUwMDExgvsrXHUwMDEy0XeI68qTM1x1MDAxMT2khFqAXHUwMDA1IeuRxL5cYu3F897UdGsk85A6Zufn/W7huPjgutCCP1x1MDAwNsGRsSbMb2ylXHUwMDE3ZI3ZtrByrjywk4vEpnOOQ87hps8o1oJcdTAwMDKKJprsat79u1x01D5cdTAwMWTTWSRtXHUwMDEy9VfvhLroYSV2k96u1TviqGRIPUvvXHUwMDFhZoLkgsibhOVwVDHFcVx1MDAwNOjO5GNcdFx1MDAwZrpPPFx1MDAxNsnQQzZMQNxFXHUwMDE3Zoc5XHUwMDE0Vu2YR2k1MdmPXHUwMDA1XHUwMDFlJYtVYLCJOM1cdTAwMTXX5XvVXHUwMDBijVx1MDAxMvdcZn9RkKGLXqBRilrMasOwuFx1MDAxOPBdXHUwMDE5LFx1MDAwM+JcdTAwMDfKIIafO1Fvq6GhRXwmxqYliMafvVx1MDAwMJjnXHUwMDFm85f/b1SQs1x1MDAwZv67NFx1MDAxOOQnXHUwMDAwqrdt//uhWX2MheXrv2qzXHUwMDFmVLKzXHUwMDFkQ21TnWH6P3/95/9cdTAwMDNcdTAwMTIqrlQifQ== + + + + + SourceIcebergSink2023, bucket 12023, bucket 22023, bucket 32023, bucket 16.....SourceIcebergSink2023, bucket 12023, bucket 22023, bucket 32023, bucket 16...........SourceIcebergSink2023, bucket 12023, bucket 22023, bucket 32023, bucket 16..... \ No newline at end of file From 8a594f6bc457123a7557f0443c3a15832d7045be Mon Sep 17 00:00:00 2001 From: Renjie Liu Date: Wed, 15 Nov 2023 13:25:55 +0800 Subject: [PATCH 2/4] Some complements --- rfcs/0077-iceberg-sink-shuffle.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/rfcs/0077-iceberg-sink-shuffle.md b/rfcs/0077-iceberg-sink-shuffle.md index 923780e4..b12a262e 100644 --- a/rfcs/0077-iceberg-sink-shuffle.md +++ b/rfcs/0077-iceberg-sink-shuffle.md @@ -45,9 +45,11 @@ There are two possible implementations for this feature: flowchart TD A[Source] --> B[IcebergPartitionOperator] B --> C[DispatcherExecutor] - C -->|partition_value| D[IcebergSink] + C -->|"partition_value"| D[IcebergSink] ``` +The `IcebergPartitionExecutor` will be a `StreamExecutor`, which calculates the partition value of each record and adds it to the record. The `DispatcherExecutor` doesn't need to change much, and do hash shuffle according to the partition value. + 2. Extend dispatcher executor to support iceberg partition shuffle. The plan is like following: ```mermaid @@ -56,11 +58,14 @@ flowchart TD B -->|IcebergDispatcher| C[IcebergSink] ``` +In this approach we need to add an extra `IcebergDispatcher` to dispatcher executor. The `IcebergDispatcher` will calculate the partition value of each record and do hash shuffle according to the partition value. + I prefer approach 1 since it's more extensible and does not change too much current shuffle implementation, e.g. other lakehouse sinks (delta lake) could have similar approach. ### Caveats When iceberg partition spec only contains range partitions(e.g. year, month, day), we don't need to do this shuffle, otherwise all traffic will go to same sink. +Also we need to reject upsert queries where partition columns is not a subset of the `stream_pk`. ## References From 17915082a4527e2cdd0536b5c331cec6674885da Mon Sep 17 00:00:00 2001 From: Renjie Liu Date: Thu, 16 Nov 2023 10:45:29 +0800 Subject: [PATCH 3/4] Add more details to explain cases better --- rfcs/0077-iceberg-sink-shuffle.md | 157 +++++++++++++++++++++++++++--- 1 file changed, 141 insertions(+), 16 deletions(-) diff --git a/rfcs/0077-iceberg-sink-shuffle.md b/rfcs/0077-iceberg-sink-shuffle.md index b12a262e..70d7a2e8 100644 --- a/rfcs/0077-iceberg-sink-shuffle.md +++ b/rfcs/0077-iceberg-sink-shuffle.md @@ -9,41 +9,171 @@ start_date: "2023/11/8" ## Motivation -Apache iceberg allows users to define partition spec[1] for a table. The partition spec defines how data is partitioned and stored in the table. For example, a table can be partitioned by date and hour. Also, it's required that each data file in apache iceberg can contain only one partition value. In our current iceberg sink implementation, we don't do any shuffle. This means that if the table is partitioned by bucket, we will have a lot of small files in the table. This is not good for performance. We should shuffle the data according to the partition spec. +Apache iceberg allows users to define partition spec[1] for a table. The partition spec defines how data is partitioned and stored in the table. For example, a table can be partitioned by date and hour. Also, **it's required that each data file in apache iceberg can contain only one partition value**. This means that if the table is partitioned by bucket, we will have a lot of small files in the table if we don't do any shuffle at all. Let's use following table as an example: + +```sql +CREATE TABLE prod.db.sample ( + id bigint, + data string, + category string, + ts timestamp) +USING iceberg +PARTITIONED BY (bucket(16, id), years(ts)) +``` + +If we don't do any shuffle, the writing process will be like following: + +![image](images/0077-iceberg-sink-shuffle/before_shuffle.svg) + +By shuffling values with same partition value to same sink, we can reduce the number of data files in the table: + +![image](images/0077-iceberg-sink-shuffle/after_shuffle.svg) ## Design -We will add a shuffle operator before sending data to iceberg sink. For example, let's assume the iceberg table is partitioned by following partition spec: +There are several cases to think about when doing shuffle: +```mermaid +flowchart TD + Start((start)) + End((end)) + Start --> append_only{Sink append only?} + + append_only -->|yes| iceberg_range_only_part{Range only partition?} + iceberg_range_only_part -->|yes| no_shuffle(No shuffle) + iceberg_range_only_part -->|no| append_only_shuffle(Shuffle by partition columns) + no_shuffle --> End + append_only_shuffle --> End + + append_only -->|no| upsert_iceberg_range_only_part{Range only partition?} + upsert_iceberg_range_only_part -->|yes| upsert_shuffle_by_stream_part_key(Shuffle by stream key + partition columns) + upsert_shuffle_by_stream_part_key --> End + upsert_iceberg_range_only_part -->|No| upsert_shuffle_by_part_key(Shuffle by partition columns) + upsert_shuffle_by_part_key --> End +``` + +Here we will use several examples to explain different cases. + +### Case 1: Append only table with range only partition + +Following iceberg table definition has only range partition: ```sql +## Iceberg CREATE TABLE prod.db.sample ( id bigint, data string, category string, ts timestamp) USING iceberg -PARTITIONED BY (bucket(16, id), years(ts)) +PARTITIONED BY (years(ts)) ``` -If we don't have the shuffle operator, the writing process will be like following: +And we want to sink following query to iceberg: +```sql +CREATE TABLE t0 (id bigint, data string, category string, ts timestamp) WITH ( + connector = 'kafka', + topic = 'kafka_1_csv_topic', + properties.bootstrap.server = 'message_queue:29092', + scan.startup.mode = 'earliest' +) FORMAT PLAIN ENCODE CSV (delimiter = ','); + +CREATE SINK s0 AS SELECT * FROM t0 WITH ( + connector = 'iceberg' +); +``` -![image](images/0077-iceberg-sink-shuffle/before_shuffle.svg) +In this case we don't need to do any shuffle, since it's range only partition. Otherwise it all traffic will go to same actor. -After we add the shuffle operator, the writing process will be like following: +### Case 2: Append only table with not range only partition -![image](images/0077-iceberg-sink-shuffle/after_shuffle.svg) +Following iceberg table definition has only range partition: +```sql +## Iceberg +CREATE TABLE prod.db.sample ( + id bigint, + data string, + category string, + ts timestamp) +USING iceberg +PARTITIONED BY (bucket(id, 16), years(ts)) +``` + +And we want to sink following query to iceberg: +```sql +CREATE TABLE t0 (id bigint, data string, category string, ts timestamp) WITH ( + connector = 'kafka', + topic = 'kafka_1_csv_topic', + properties.bootstrap.server = 'message_queue:29092', + scan.startup.mode = 'earliest' +) FORMAT PLAIN ENCODE CSV (delimiter = ','); + +CREATE SINK s0 AS SELECT * FROM t0 WITH ( + connector = 'iceberg' +); +``` + +In this case we need to shuffle by `(bucket(id), years(ts))` so that rows with same partition value will go to same actor. + +### Case 3: Upsert table with range only partition + +Following iceberg table definition has only range partition: +```sql +## Iceberg +CREATE TABLE prod.db.sample ( + id bigint, + data string, + category string, + ts timestamp) +USING iceberg +PARTITIONED BY (years(ts)) +``` + +And we want to sink following query to iceberg: +```sql +CREATE TABLE t0 (id bigint primary key, data string, category string, ts timestamp); + +CREATE SINK s0 AS SELECT * FROM t0 WITH ( + connector = 'iceberg' +); +``` + +In this case we need to shuffle by `(years(ts), id)` to avoid all traffic go to same actor. -This way we can reduce the number of data files in the table, which helps to improve the read performance of iceberg. + +### Case 4: Upsert table with not range only partition + +Following iceberg table definition has only range partition: +```sql +## Iceberg +CREATE TABLE prod.db.sample ( + id bigint, + data string, + category string, + ts timestamp) +USING iceberg +PARTITIONED BY (bucket(data, 16), years(ts)) +``` + +And we want to sink following query to iceberg: +```sql +CREATE TABLE t0 (id bigint primary key, data string, category string, ts timestamp); + +CREATE SINK s0 AS SELECT * FROM t0 WITH ( + connector = 'iceberg' +); +``` + +In this case we need to shuffle by `(bucket(data), years(ts))` so that rows with same will go to same actor. Notice that in iceberg we don't need to shuffle by `stream_key`, e.g. `id` since it only requires that insert/delete of same row will go to same partition. ### Implementation -There are two possible implementations for this feature: +There are two possible implementations for shuffling by iceberg partition columns: -1. Add a new `IcebergPartitionOperator`, which calculates the partition value of each record and adds it to the record, then asking the dispatcher executor to do hash shuffle according to the partition value. The plan is like following: +1. Add a new `IcebergPartitionExecutor`, which calculates the partition value of each record and adds it to the record, then asking the dispatcher executor to do hash shuffle according to the partition value. The plan is like following: ```mermaid flowchart TD - A[Source] --> B[IcebergPartitionOperator] + A[Source] --> B[IcebergPartitionExecutor] B --> C[DispatcherExecutor] C -->|"partition_value"| D[IcebergSink] ``` @@ -62,11 +192,6 @@ In this approach we need to add an extra `IcebergDispatcher` to dispatcher execu I prefer approach 1 since it's more extensible and does not change too much current shuffle implementation, e.g. other lakehouse sinks (delta lake) could have similar approach. -### Caveats - -When iceberg partition spec only contains range partitions(e.g. year, month, day), we don't need to do this shuffle, otherwise all traffic will go to same sink. -Also we need to reject upsert queries where partition columns is not a subset of the `stream_pk`. - ## References 1. https://iceberg.apache.org/spec/#partitioning \ No newline at end of file From 532d50c7f19ef940582390a17412e633e069449e Mon Sep 17 00:00:00 2001 From: Renjie Liu Date: Fri, 17 Nov 2023 16:25:08 +0800 Subject: [PATCH 4/4] Update case 3 according to discussion --- rfcs/0077-iceberg-sink-shuffle.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/rfcs/0077-iceberg-sink-shuffle.md b/rfcs/0077-iceberg-sink-shuffle.md index 70d7a2e8..0c1b8cfa 100644 --- a/rfcs/0077-iceberg-sink-shuffle.md +++ b/rfcs/0077-iceberg-sink-shuffle.md @@ -38,15 +38,13 @@ flowchart TD Start((start)) End((end)) Start --> append_only{Sink append only?} - append_only -->|yes| iceberg_range_only_part{Range only partition?} - iceberg_range_only_part -->|yes| no_shuffle(No shuffle) + iceberg_range_only_part -->|yes| no_shuffle(No shuffle) iceberg_range_only_part -->|no| append_only_shuffle(Shuffle by partition columns) no_shuffle --> End append_only_shuffle --> End - append_only -->|no| upsert_iceberg_range_only_part{Range only partition?} - upsert_iceberg_range_only_part -->|yes| upsert_shuffle_by_stream_part_key(Shuffle by stream key + partition columns) + upsert_iceberg_range_only_part -->|yes| upsert_shuffle_by_stream_part_key(Shuffle by stream key) upsert_shuffle_by_stream_part_key --> End upsert_iceberg_range_only_part -->|No| upsert_shuffle_by_part_key(Shuffle by partition columns) upsert_shuffle_by_part_key --> End @@ -137,7 +135,7 @@ CREATE SINK s0 AS SELECT * FROM t0 WITH ( ); ``` -In this case we need to shuffle by `(years(ts), id)` to avoid all traffic go to same actor. +In this case we need to shuffle by `id` to avoid all traffic go to same actor. Since in most cases upstream is already shuffled by `id`, we can avoid another shuffle. ### Case 4: Upsert table with not range only partition