From 13f1c11c283583303535a67db583618ab3d0cde3 Mon Sep 17 00:00:00 2001 From: Ryan Eakman <6326532+eakmanrq@users.noreply.github.com> Date: Sat, 31 Aug 2024 14:20:26 -0700 Subject: [PATCH] chore: add duckdb delta test (#164) --- ...3c-ad56-002876814e3a-c000.snappy.parquet.crc | Bin 0 -> 16 bytes ...16-b232-504d6194eb52-c000.snappy.parquet.crc | Bin 0 -> 20 bytes ...06-8e84-0d2a324c6894-c000.snappy.parquet.crc | Bin 0 -> 20 bytes ...75-acc6-b3c69c4ce80b-c000.snappy.parquet.crc | Bin 0 -> 20 bytes ...d5-8850-82ef3884db60-c000.snappy.parquet.crc | Bin 0 -> 20 bytes ...4e-9bcd-09ce8664f6e0-c000.snappy.parquet.crc | Bin 0 -> 20 bytes .../_delta_log/.00000000000000000000.json.crc | Bin 0 -> 36 bytes .../_delta_log/00000000000000000000.json | 8 ++++++++ ...f-4d3c-ad56-002876814e3a-c000.snappy.parquet | Bin 0 -> 646 bytes ...f-4b16-b232-504d6194eb52-c000.snappy.parquet | Bin 0 -> 1437 bytes ...b-4706-8e84-0d2a324c6894-c000.snappy.parquet | Bin 0 -> 1417 bytes ...e-4075-acc6-b3c69c4ce80b-c000.snappy.parquet | Bin 0 -> 1424 bytes ...3-4ad5-8850-82ef3884db60-c000.snappy.parquet | Bin 0 -> 1459 bytes ...f-4c4e-9bcd-09ce8664f6e0-c000.snappy.parquet | Bin 0 -> 1417 bytes .../engines/duck/test_duckdb_reader.py | 16 ++++++++++++++++ 15 files changed, 24 insertions(+) create mode 100644 tests/fixtures/employee_delta/.part-00000-e5965c7b-e58f-4d3c-ad56-002876814e3a-c000.snappy.parquet.crc create mode 100644 tests/fixtures/employee_delta/.part-00002-3fed7f18-370f-4b16-b232-504d6194eb52-c000.snappy.parquet.crc create mode 100644 tests/fixtures/employee_delta/.part-00004-143c5da1-d5ab-4706-8e84-0d2a324c6894-c000.snappy.parquet.crc create mode 100644 tests/fixtures/employee_delta/.part-00006-64f07e25-c30e-4075-acc6-b3c69c4ce80b-c000.snappy.parquet.crc create mode 100644 tests/fixtures/employee_delta/.part-00008-89ccad8d-df73-4ad5-8850-82ef3884db60-c000.snappy.parquet.crc create mode 100644 tests/fixtures/employee_delta/.part-00010-812b3382-8c7f-4c4e-9bcd-09ce8664f6e0-c000.snappy.parquet.crc create mode 100644 tests/fixtures/employee_delta/_delta_log/.00000000000000000000.json.crc create mode 100644 tests/fixtures/employee_delta/_delta_log/00000000000000000000.json create mode 100644 tests/fixtures/employee_delta/part-00000-e5965c7b-e58f-4d3c-ad56-002876814e3a-c000.snappy.parquet create mode 100644 tests/fixtures/employee_delta/part-00002-3fed7f18-370f-4b16-b232-504d6194eb52-c000.snappy.parquet create mode 100644 tests/fixtures/employee_delta/part-00004-143c5da1-d5ab-4706-8e84-0d2a324c6894-c000.snappy.parquet create mode 100644 tests/fixtures/employee_delta/part-00006-64f07e25-c30e-4075-acc6-b3c69c4ce80b-c000.snappy.parquet create mode 100644 tests/fixtures/employee_delta/part-00008-89ccad8d-df73-4ad5-8850-82ef3884db60-c000.snappy.parquet create mode 100644 tests/fixtures/employee_delta/part-00010-812b3382-8c7f-4c4e-9bcd-09ce8664f6e0-c000.snappy.parquet diff --git a/tests/fixtures/employee_delta/.part-00000-e5965c7b-e58f-4d3c-ad56-002876814e3a-c000.snappy.parquet.crc b/tests/fixtures/employee_delta/.part-00000-e5965c7b-e58f-4d3c-ad56-002876814e3a-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..3019ee4c51c64e17cc87d239f0a92963b7e2edad GIT binary patch literal 16 XcmYc;N@ieSU}E@ddBx{UUZ@WMC&vYB literal 0 HcmV?d00001 diff --git a/tests/fixtures/employee_delta/.part-00002-3fed7f18-370f-4b16-b232-504d6194eb52-c000.snappy.parquet.crc b/tests/fixtures/employee_delta/.part-00002-3fed7f18-370f-4b16-b232-504d6194eb52-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..649c76955d8f99b5e965f1f069d85acaf6c77c42 GIT binary patch literal 20 ccmYc;N@ieSU}BK0U9-#o*-Vq!>(@I106v)qa{vGU literal 0 HcmV?d00001 diff --git a/tests/fixtures/employee_delta/.part-00004-143c5da1-d5ab-4706-8e84-0d2a324c6894-c000.snappy.parquet.crc b/tests/fixtures/employee_delta/.part-00004-143c5da1-d5ab-4706-8e84-0d2a324c6894-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..9c93422705e731dfce4d6d5b9d12a23e778c80aa GIT binary patch literal 20 bcmYc;N@ieSU}8Aq?P9puQhKkp`JSZ!H);k> literal 0 HcmV?d00001 diff --git a/tests/fixtures/employee_delta/.part-00006-64f07e25-c30e-4075-acc6-b3c69c4ce80b-c000.snappy.parquet.crc b/tests/fixtures/employee_delta/.part-00006-64f07e25-c30e-4075-acc6-b3c69c4ce80b-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..fa8983d8770b961803221e5fd7d03e1291446bb6 GIT binary patch literal 20 bcmYc;N@ieSU}D(R7F2C&d;I>jb4=3!KdlFp literal 0 HcmV?d00001 diff --git a/tests/fixtures/employee_delta/.part-00008-89ccad8d-df73-4ad5-8850-82ef3884db60-c000.snappy.parquet.crc b/tests/fixtures/employee_delta/.part-00008-89ccad8d-df73-4ad5-8850-82ef3884db60-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..cf471a8919d726631396b5bb6df831208c86c33d GIT binary patch literal 20 bcmYc;N@ieSU}C7=zU8q@?7>hwr&$&NK)?p& literal 0 HcmV?d00001 diff --git a/tests/fixtures/employee_delta/.part-00010-812b3382-8c7f-4c4e-9bcd-09ce8664f6e0-c000.snappy.parquet.crc b/tests/fixtures/employee_delta/.part-00010-812b3382-8c7f-4c4e-9bcd-09ce8664f6e0-c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..814f962e994466116fae8bb8f7237a5e7604dfb9 GIT binary patch literal 20 bcmYc;N@ieSU}Css6!EnFH|IHR^F2!eL9z#! literal 0 HcmV?d00001 diff --git a/tests/fixtures/employee_delta/_delta_log/.00000000000000000000.json.crc b/tests/fixtures/employee_delta/_delta_log/.00000000000000000000.json.crc new file mode 100644 index 0000000000000000000000000000000000000000..8d605d99932758e7518ccfc5b5b7923255cef2ad GIT binary patch literal 36 scmYc;N@ieSU}7-c?0z|Tg`CAJUK@Z)Z zBh)H#U)fMcJb`cixYC7c;4|cO9S`F|C6t%kWT8z$`Ua{>iyU_XI;2+!_kTrH3rS6F ziyAal#XHSFbHO#opVo@#WeFrCl~#4EiAUxsNKujSaYZ&|nXkPT%zqo7<{A=EM#yU^ zc_dAwu>g;3-ATAwue~3artKYX@}+t5cf6U=|8Xx_12ztrKM})d6bv99P6A(yM}90guxkbbe=J1A`_7X&n*E19EQF@! G0>1#X0L6v? literal 0 HcmV?d00001 diff --git a/tests/fixtures/employee_delta/part-00002-3fed7f18-370f-4b16-b232-504d6194eb52-c000.snappy.parquet b/tests/fixtures/employee_delta/part-00002-3fed7f18-370f-4b16-b232-504d6194eb52-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..c2a02fa5eb58f15670606381f15f12892ee2a14e GIT binary patch literal 1437 zcmbW1PiqrF6u>8wq%LhKD4k&!RtQTQTZl{6BsL92P^3taP_#EIWs;qyp_@P5-6*B> zCVKE*LHYs2gL;r0Jc+2E!J}S02k|K4o7v6oih`}M%)WW^-tWD6@9nN%y=4(ZW{FBZ zJbLkSN+%e$6GTC1at`cOBrW}}Lz+{azcfo9qERY9p-kwib zox2`b0TbQc;r$)v+F{HmUtd0+Ntljf&Id97m6moEYidR$7`_PBC~Tx!Bm`=iSFBdX z8E9%GCcQ*t@RA9{UVn+9Q-!QTFr?|UPU@j_oFW^cI7&t5LmY?@$hk4?^_jc7Y^4><0Li0E6rLw@K-#)=SpXaHC@}x zD^OLfqM!-wH#Wf((`$4!udD=x3s#C2vRK-R^r~o5@4f_#o8qPX_ip>DNU}R z{0Q*l`->_?_Z|UUb~ji>uAk!AVsL&B{h)-C`kAnDK5b0-VLtdfZ z@hr?wK~;Nh%Vd29BVo$vFz<2KYxlZpwq#b!OGsV(!@}EhOcKA-q`1^fs)C|ve2Tk877~5^`*q%|lb6|)Wf}_N|!Fg!k?)tpNT~Krfj>8%bSOS6< zj4|D)9ULAOB8r^R*dOAxyRDPE%fA1=JM=YnK6UQnFXq&UyN5_!3rl~o&Tf>gYRM`t+m*&rsmz;|md6Ta}|gD$}T JbRB-ozX0YPHW&Z^ literal 0 HcmV?d00001 diff --git a/tests/fixtures/employee_delta/part-00004-143c5da1-d5ab-4706-8e84-0d2a324c6894-c000.snappy.parquet b/tests/fixtures/employee_delta/part-00004-143c5da1-d5ab-4706-8e84-0d2a324c6894-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..cafafd2f1865350280059e02a0abc2f69c5fa577 GIT binary patch literal 1417 zcmbW1O=uHA6vro%q^@mIgwC*wi-g677UGg^5}TG@>Y+#}A$YY?HrZ(!y2+-y8>N(< z1PflgD)gWik0M3HqZjeuRc~Gt1P?t3ig*&=d?eZ8#R|*J&YS=Hy_xskZhiTNLlBuF zI(hW!?uWA`!LS@78bafjF+wPv;v?8hk|R&PzI!{7ken+^Nd_dV!B!V!mxx7v96b1f z6SDJFDa(RvEog2t74h}M@6XR9bc2Y?G9o|Jn}cV+lMx4!6iAE<*w8Z^ttNsEIxzF& zfLWYsWX_84hA!u%p2q~e2tCqgL>ygAN^Me&tUwrKn6yS}>g_N^>gx44HQYbK@)cbD z>yuH>4YxO<<`(O?WE>GZjr9rE>G{DfV;gM`kx8&*H@j|!O`|mvVn~ahv~aJq+hWpR zi{c;@&J-J_v7Xl;pGHMPW5x+K!4%VLbS1AX$&|^Fq8X7uxq>S}WdMv2{hT3F^cN;J z%GXRXsgUw;0=7zP`5M4@hULg%qg4|2lf#~-IqWc%poxnK+8V9H){-iWngA#ZrPfL0 z{VbvMeVWXn{4pR0cjlQH-t`!OqM~x-=i~L2t0e3(huzEY^BwWVRks}`cStaTRdoT^pMv7M6bkuR8gli}3HP!%z4RbC)l2 literal 0 HcmV?d00001 diff --git a/tests/fixtures/employee_delta/part-00006-64f07e25-c30e-4075-acc6-b3c69c4ce80b-c000.snappy.parquet b/tests/fixtures/employee_delta/part-00006-64f07e25-c30e-4075-acc6-b3c69c4ce80b-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..35a4fb12056451257cf4bc7e88c45d8a33490e97 GIT binary patch literal 1424 zcmbW1&ubGw6vro%G^{lgh0d@G3x=f)EyN{jlK!HXdMHvQlwPfrNp>28n;+fXD5Vqx zZ|cFD2nF#XB8rzDr1sy?g9pKjc;?OAv8XP5klEa7FMvCq)1-BeR#i+l1$W;BnOh~EaV_7k}2}x=h5-G zlGZjrH@qJMj9uoNskpL<(RT+wQS0pGG%Kh@mWA+9JHNtj(3b z7KD9YI#Xqie_X9)d^k+dIMmD==U6%q`xq+ zQK?~)LWESwN!S`~lo|jNHCC&DC8sp(w}8FM3fKWm*@z}zOwt;31Fn|i&cshZM!kwH zY3!3RLg}Y0nM0)$F}zWdqB^7ZJprifs#-;kqQ~l#hP@Q9$2pPXAx>bU0*{D?7>Hsn z)GCO{DNQ^U#3MnZU!eRYjsC!sFgImg@B3|=4H%4qEvLf1%l)9!@9ASzyKY}W`s^u1 zu;bYjrREIo zBYmeA@;3Lu&>MOlYkHsr1TR}L-CEk)-!G3SayAoxh}Y@0|J_~o`~TgcuZi=SbC182 zGb3*8BYiC@{pGG-jCxgcY^Q2F#hK=?9A!%TE8$CHMN<15e1>qaf}d3r!ugDtw_en;rC}hu;g;3CR}pha>uh>pMj%6Zjh6A zpFhg1YsH(D z2KUEUY$4QJ&Idi)f3z7iAF{ShMiIdiSiQvBJ+HgN*rw|sk_St6t7Es>1X?LT4q@?R z3y(_MEhgfXAnf{4FvVIy+b|RmPOB;?r~Sqzm}0s}mkgyQ1b+#-1Ym^d z(<~XI2Wes=qh28S7%7U2V0F4?)N6^>$rFIF`iVt`zTnU|X)=S1qm)FA{$NDCN7v!1 zCN)Xg0({s!$u6gOoh6i>rO6^P&LKsj1y^jK!8s&|IxSW%0`I#x$*>O`_A0|gIOcCc ztinL_0r40Ek3g2t@W2xgtfVJ&ePMTRuQZ@U^E!-&d~T=p@8MDJ z|34i1It;#W?aAA7VZ?(yq~49CzuESSaj%M&X;nO!h)7D(YD$hIB#%yJp nHLLTLvNJbZZaOxzW-K;co^zarJ*92H+b;i51-@~s@VWj4`=mhF literal 0 HcmV?d00001 diff --git a/tests/fixtures/employee_delta/part-00010-812b3382-8c7f-4c4e-9bcd-09ce8664f6e0-c000.snappy.parquet b/tests/fixtures/employee_delta/part-00010-812b3382-8c7f-4c4e-9bcd-09ce8664f6e0-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..8c5adb9571ea386eed7c0fcb45fefd7ff5916ec4 GIT binary patch literal 1417 zcmbW1&ubGw6vro%O<8JDgwC*wtAxdd7UGg^5}TG@^iWDEA@pjcY_iiB-2CY7Mk%FN z4|?@x4j;K8Hf(MvrS@lxy2H$ReH(SsGj?9Q9-`@A>ry-nlRhD#8cCI)%> z^8H!PA{e$4L`P`yIz|X(GdWnnW|2JkaC~}-Q<94nDbYZ(I&Akrc8%EN?C|vqEhW2D zOGq1RmqjY#lf#dn^ptKC@v4l-lRA9%?akz0BtA&Y%h)tDo}PLTY|@dLrvqk@RbtMG z@TMW3X+4h#dJ#I&Ya)&z9!hOdovc6@WLmUN>gw$vLmKM!Br`rg!txbdgVm{M;Dz_M zqSigu^~fY5cm^96Sa%TgcNyF2_=rq_CAZ!4x@-omTM$E9g0zKuWxY0&{(2PmL*Y!Z zX_=b^UQe^CqY3jjHo+9r+jOO%FUyk2m7-ZuK)Hh}L1zGr5dEr=Y5Efr2NmiTnNmoZ zoPe#-dZ7+5USYX%Sh7pQesI{UEQcLrQZ#WfL0hK{I9gVXQ4avhMyY)o`8Y=?{hTH9 zsBi+v(Vcl^#&$qjd&CfQ6oxsog&5BzqyQM|H+FU4CSgX2NF8Q^EQp@+4d(&l=QqA|9 T-kiA!Uv>J2u4o7~;7|A$<5Mz= literal 0 HcmV?d00001 diff --git a/tests/integration/engines/duck/test_duckdb_reader.py b/tests/integration/engines/duck/test_duckdb_reader.py index eb4d570..026d7b3 100644 --- a/tests/integration/engines/duck/test_duckdb_reader.py +++ b/tests/integration/engines/duck/test_duckdb_reader.py @@ -98,3 +98,19 @@ def test_employee_extra_line_csv_multiple(duckdb_session: DuckDBSession): ), Row(**{"employee_id": 5, "fname": "Hugo", "lname": "Reyes", "age": 29, "store_id": 100}), ] + + +def test_employee_delta(duckdb_session: DuckDBSession): + df = duckdb_session.read.load( + "tests/fixtures/employee_delta", + format="delta", + ) + assert df.collect() == [ + Row(**{"employee_id": 1, "fname": "Jack", "lname": "Shephard", "age": 37, "store_id": 1}), + Row(**{"employee_id": 2, "fname": "John", "lname": "Locke", "age": 65, "store_id": 1}), + Row(**{"employee_id": 3, "fname": "Kate", "lname": "Austen", "age": 37, "store_id": 2}), + Row( + **{"employee_id": 4, "fname": "Claire", "lname": "Littleton", "age": 27, "store_id": 2} + ), + Row(**{"employee_id": 5, "fname": "Hugo", "lname": "Reyes", "age": 29, "store_id": 100}), + ]