o
    h                    @   sp  d dl mZ d dlmZ d dlZd dlZzd dlZW n ey%   dZY nw d dl	Z	d dl
Zd dlmZ d dlmZ d dlmZ dd Zdd	 Zd
d Zdd Ze	jjdd Zdd Zdd Ze	jjdd Zdd Zdd Zdd Ze	jdg de  fg de! fg de" fg d e# fg e$e% fd!d"gd#gge$e! fd$gdd%d&gge$e# fg d'e&e'd$e! e'd%e# gfgd(d) Z(e	jj)d*d+ Z*e	jj)d,d- Z+e	jj)d.d/ Z,e	jj)d0d1 Z-e	jj)d2d3 Z.e	jj)e	jj/d4d5 Z0d6d7 Z1d8d9 Z2d:d; Z3d<d= Z4d>d? Z5G d@dA dAZ6G dBdC dCZ7e	jdDe6e7gdEdF Z8dGdH Z9e	jdDe6e7gdIdJ Z:dKdL Z;dMdN Z<dOdP Z=dQdR Z>dSdT Z?dUdV Z@dWdX ZAdYdZ ZBd[d\ ZCd]d^ ZDd_d` ZEdadb ZFdcdd ZGdedf ZHdgdh ZIdidj ZJdkdl ZKdmdn ZLdodp ZMdqdr ZNdsdt ZOdudv ZPdwdx ZQe	jje	jdyg dzd{d| ZRe	jjd}d~ ZSe	jjdd ZTe	jjdd ZUe	jjdd ZVe	jjdd ZWe	jjdd ZXdd ZYdd ZZdd Z[dd Z\e	jj)dd Z]e	jjdd Z^dd Z_dd Z`dd Zae	jj)dd Zbe	jdejcejdgdd Zedd Zfdd Zgdd Zhdd Zidd Zjdd Zkdd Zldd Zme	jdejcejdgdd Zne	jdejcejdgdd Zoe	jdejcejdgdd Zpdd Zqe	jdejcejdgdd Zre	jdejcejdgdd Zsdd Ztdd Zue	jdejcejdgdd Zve	jdejcejdgddÄ Zwddń ZxddǄ ZyddɄ Zzdd˄ Z{dd̈́ Z|ddτ Z}ddф Z~e	jj)ddӄ ZddՄ Zddׄ Zddل Zddۄ Zdd݄ Zdd߄ Zdd Ze	jdejcejdgdd Ze	jdejcejdgdd Ze	jdejcejdgdd Ze	jjdd Ze	jdg dg dfefg dg dfejfdgddggdgddggfejfgdd Ze	jdejcejdgdd Ze	jdg dg dfejfdgddggdgddggfejfgdd Ze	jdg dg dfefg dg dfejfdgddggdgddggfejfgdd Ze	jdejcejdgdd Ze	jj)dd Ze	jj)dd Zd d Ze	jj)dd Zdd Zdd Zdd	 Zd
d Zdd Zdd Zdd Zdd Zdd Zdd Ze	jjdd Ze	jjdd Zdd Ze	jjdd Ze	jjd d! Ze	jjd"d# Ze	jje	jdejcejdgd$d% Ze	jjd&d' Ze	jjd(d) Ze	jjd*d+ Ze	jjd,d- Ze	jjd.d/ Ze	jjd0d1 Ze	jjd2d3 Ze	jjd4d5 Ze	jjd6d7 Ze	jdejcejdgd8d9 Ze	jdejcejdgd:d; Ze	jje	jd<ejejgd=d> Ze	jje	jd<ejejgd?d@ Ze	jjdAdB Ze	jdCdD Ze	jdEdF Ze	jdGdH Ze	jdIdJ Ze	jdKdL Ze	jdMdN Ze	jdOdP Ze	jdQdR Ze	jdSdT Ze	jdUdV Ze	jdWdX Ze	jdYdZ ZƐd[d\ Zǐd]d^ ZȐd_d` Zɐdadb Zʐdcdd ZdS (e      )OrderedDict)IterableN)from_dataframe)Versionc                  C   s  t jg t  d} | jt  ksJ |  g ksJ |   t jg dd}|jt  ks.J tt	 t g  W d    n1 sCw   Y  t g dg dg dg} t
| jts^J tdd | jD sjJ tdd |  D swJ t| jd	ksJ |  td
d |  D ksJ t| t| |   ksJ | jdksJ |   t| }| d usJ ~ | d u sJ d S )Ntypebinary                        	   c                 s       | ]
}t |tjjV  qd S N
isinstancepalib
Int64Array.0c r   L/var/www/vscode/kcb/lib/python3.10/site-packages/pyarrow/tests/test_table.py	<genexpr>4       z,test_chunked_array_basics.<locals>.<genexpr>c                 s   r   r   r   r   r   r   r    r!   5   r"   r   c                 s   s    | ]}|  V  qd S r   )get_total_buffer_sizer   r   r   r    r!   7   s    H   )r   chunked_arraystringr   	to_pylistvalidater   pytestraises
ValueErrorr   chunkslistall
iterchunkslenr#   sumsys	getsizeofobject
__sizeof__nbytesweakrefref)datadata2wrr   r   r    test_chunked_array_basics"   s@   

r<   c                  C   s  t g dg dg dg} | jt  ksJ t| dksJ t| jdks(J t g dg dg dg} | jt  ks?J t| dksGJ t| jdksPJ t jg dg dg dgt  d} | jt  kskJ t| dkssJ t| jdks|J t g dg g} | jt  ksJ t| dksJ t| jdksJ d	}tjt	|d
 t g sJ W d    n1 sw   Y  t jg t 
 djt 
 ksJ t g gjt  ksJ t jg gt 
 djt 
 ksJ d S )Nr	   r   r   r   r         @      @g      @r   r   z@cannot construct ChunkedArray from empty vector and omitted typematch)r   r%   r   int64r0   r,   int8r)   r*   r+   r&   null)arrmsgr   r   r    test_chunked_array_constructionD   sP    &rG   c                  C   sD   t ddg} t | | g}| }t g d}||s J d S )Nr
   r   r
   r   r
   r   )r   arrayr%   combine_chunksequals)rE   chunked_arrresexpectedr   r   r    test_combine_chunksq   s
   rO   c                   C   sl   t jg t  d t jg t  dksJ t jt jg t  dgt  d t jg t  dks4J d S )Nr   )r   r%   bool_rJ   rI   r   r   r   r    4test_chunked_array_can_combine_chunks_with_no_chunksz   s   rQ   c                  C   s\   t g dg dg g} t| }|  }t|tjsJ |jdks$J t||s,J d S )Nr	   r   )r   )	r   r%   npasarrayto_numpyr   ndarrayshapearray_equal)r9   arr1arr2r   r   r    test_chunked_array_to_numpy   s   
rZ   c                  C   s   d} t jt| d ttg dtg dg W d    n1 s%w   Y  t jt| d tjtg dgt d W d    d S 1 sLw   Y  d S )Nzchunks must all be same typer@   r	         ?       @      @r   )r)   r*   	TypeErrorr   r%   rI   float64)rF   r   r   r    !test_chunked_array_mismatch_types   s    "ra   c                  C   s:   t g dt g dg} t | } t| dksJ d S )Nr	   r   z<[
  [
    1,
    2,
    3
  ],
  [
    4,
    5,
    6
  ]
])r   rI   r%   strr9   r   r   r    test_chunked_array_str   s
   
rd   c                  C   sN  t g dt g dg} t | } | d  dksJ | d  dks'J | d  dks1J tt | d  W d    n1 sEw   Y  tt | d  W d    n1 s^w   Y  | td  dkspJ | dd	 }|	 d
d	gksJ | d	d }|	 dgksJ | dd }|j
| j
ksJ |	 g ksJ d S )Nr	   r   r
   r   r   iir   r   r   c   )r   rI   r%   as_pyr)   r*   
IndexErrorrR   int32r'   r   )r9   
data_slicer   r   r    test_chunked_array_getitem   s*   


rk   c                  C   s   t g dt g dg} t | } | t| }|j| jks"J | g ks*J | t| d }|j| jks;J | g ksCJ t jj| gdgd}|t|}t|dks\J t jj| gdgd}|t|d }t|dkswJ d S )Nr	   r   
   anamesr   )	r   rI   r%   slicer0   r   r'   Tablefrom_arrays)r9   rj   tabletable_slicer   r   r    test_chunked_array_slice   s    
ru   c                  C   sv   t dgt g dt g dt g dg} t | }ttd|D ]\}}|| ks1J q%t|ts9J d S )Nr   r	   r   r   rl   )r   rI   r%   ziprangerg   r   r   )r9   rE   ijr   r   r    test_chunked_array_iter   s   

rz   c                  C   s$  dd } dd }| t jg t  dt jg t  d |t jg t  dt jg t  d t jddgt  d}t jddgt  d}t jddgt  d}t jg d	t  d}| |g|g ||g|g | ||g||g | ||g|g |||g||g t jg t  dd rJ d S )
Nc                 S   st   t | tjr	| }nt| }t |tjr|}nt|}||s#J ||s*J ||ks0J |t|ks8J d S r   )r   r   ChunkedArrayr%   rK   rb   xarrsyarrsxyr   r   r    eq   s   

z%test_chunked_array_equals.<locals>.eqc                 S   sd   t | tjr	| }nt| }t |tjr|}nt|}||r#J ||r*J ||ks0J d S r   )r   r   r{   r%   rK   r|   r   r   r    ne  s   

z%test_chunked_array_equals.<locals>.ner   r   r   r   )r   r   r   r   )r   r%   ri   rB   rI   rK   )r   r   rm   br   dr   r   r    test_chunked_array_equals   s$    r   )r9   typ)TFTT)r
   r   r   r   )r\         @N)rm   Nr   r
   r   r   rm   r   r   ))r
   rm   )r   r   Nc                 C   st   g }| r| tj| d d |d | dd  } | stj||d}|  |||}|  ||s8J d S )Nr   r   )appendr   rI   r%   r(   loadsdumpsrK   )r9   r   pickle_modulearraysrI   resultr   r   r    test_chunked_array_pickle(  s   r   c                  C   s   dd l } tg dg}tj|dgd}|d}t|tjs!J | }t|| js-J |j	dks4J |d dks<J |j
dksCJ d S )Nr   r   r   rl   rm   rn   )r   r   )pandasr   rI   rs   columnr   r{   	to_pandasSeriesrV   name)pdr9   rs   colseriesr   r   r    test_chunked_array_to_pandasB  s   
r   c                  C   s   dd l } dd lm} tg dt| g dt| jdddt| jddddtg dtd	tg dtd	dfD ]&}td
|i}|	d

 }|jd
ksZJ | j|
 d
d}||| qCd S )Nr   r	   rm   r   rm   2012r   )periodszEurope/Brussels)r   tzmsr   )r   )r   pandas.testingtestingr   rI   Categorical
date_range	timestamprs   r   r   r   r   assert_series_equal)r   tmr9   rs   r   rN   r   r   r    *test_chunked_array_to_pandas_preserve_nameS  s   r   c                  C   s   dd l } | j| dddd}t|}| }|jdksJ |jdks&J |jdks-J |j	|js6J | j| dddd}t|}| }|jdksQJ |jdksXJ |jdks_J |j	|jshJ d S 	Nr   rl   r
   )index)rl   r   r   r   )r   r   )
r   	DataFrame
RangeIndexr   rs   r   num_rowsrV   r   rK   )r   r9   rs   r   r   r   r    .test_table_roundtrip_to_pandas_empty_dataframeg  s   

r   c                  C   s   dd l } | j| dddd}tj|}| }|jdks J |jdks'J |jdks.J |j	
|j	s7J | j| dddd}tj|}| }|jdksSJ |jdksZJ |jdksaJ |j	
|j	sjJ d S r   )r   r   r   r   RecordBatchfrom_pandasr   r   rV   r   rK   )r   r9   batchr   r   r   r    4test_recordbatch_roundtrip_to_pandas_empty_dataframe  s   r   c                  C   sl   dd l } dd lm} | ddgddgd}t|}|j  }|j	dks)J |
||jd d  d S )Nr   r
   r   g?g?rm   r   r   r   )r   r   r   r   r   rs   schemaempty_tabler   rV   assert_frame_equaliloc)r   r   dfrs   r   r   r   r    test_to_pandas_empty_table  s   
r   c                  C   sh  t dgt g dg} t | }t|}| g dks!J |jtdks+J tj|dd}| g dks<J t dd gt g dg} t | }t|}| }|d d	ks`J t|d siJ |d
d  g dksuJ |jtdksJ t j	t g dt g d}t ||g}t|}|jtdksJ | g dd
 ksJ d S )Nr   r	   )r   r
   r   r   rB   rb   dtype)0123r
   r\   r   r[   r`   )r   r
   r   r   r
   rm   r   r   r4   )rm   r   r   rm   r   )
r   rI   r%   rR   rS   tolistr   isnanDictionaryArrayrr   )r9   rL   np_arrelementsrE   r   r   r    test_chunked_array_asarray  s4   





r   c                  C   s  t t dt  t dt  g} t jg d| d}t |}| \}}|t t jg dt  ds:J |t t jg dt  dsMJ t jg | d}t |}| \}}|t t jg t  dspJ |t t jg t  dsJ d S )Nr   r   )r
   r   r         @)r         @r   )r
   r   r   )r   r   r   )	r   structfieldint16float32rI   r%   flattenrK   )tyrm   carrr   r   r   r   r    test_chunked_array_flatten  s   
&
"&r   c                  C   s   t t g d t g d g} | djt ddgs%J | djt ddgs5J |  } t g d}| dj|sKJ | dj|sVJ |  g d	ks`J d S )
NfoobarNr   )quuxNr   r   r   r   r
   r   r   r   r   )r   r   Nr   r   Nr   )	r   r%   rI   dictionary_encodechunk
dictionaryrK   unify_dictionariesr'   )rE   expected_dictr   r   r    %test_chunked_array_unify_dictionaries  s     r   c                   C   <   t jtdd t  W d    d S 1 sw   Y  d S )Nr   r@   )r)   r*   r_   r   r   r   r   r   r    test_recordbatch_dunder_init     
"r   c                  C   sx   G dd d} t g dt  }t |g}| |}t |}||ks&J t j|t  d}||t  ks:J d S )Nc                   @      e Zd Zdd ZdddZdS )z:test_chunked_array_c_array_interface.<locals>.ArrayWrapperc                 S   
   || _ d S r   )rI   )selfrI   r   r   r    __init__     
zCtest_chunked_array_c_array_interface.<locals>.ArrayWrapper.__init__Nc                 S      | j |S r   )rI   __arrow_c_array__r   requested_schemar   r   r    r        zLtest_chunked_array_c_array_interface.<locals>.ArrayWrapper.__arrow_c_array__r   __name__
__module____qualname__r   r   r   r   r   r    ArrayWrapper      r   r	   r   )r   rI   rB   r%   r   cast)r   r9   chunkedwrapperr   r   r   r    $test_chunked_array_c_array_interface  s   
r   c                  C   sn   G dd d} t g dg dg}| |}t |}||ks!J t j|t  d}||t  ks5J d S )Nc                   @   r   )zBtest_chunked_array_c_stream_interface.<locals>.ChunkedArrayWrapperc                 S   r   r   )r   )r   r   r   r   r    r     r   zKtest_chunked_array_c_stream_interface.<locals>.ChunkedArrayWrapper.__init__Nc                 S   r   r   )r   __arrow_c_stream__r   r   r   r    r     r   zUtest_chunked_array_c_stream_interface.<locals>.ChunkedArrayWrapper.__arrow_c_stream__r   r   r   r   r   r   r   r   r   r    ChunkedArrayWrapper  r   r   r	   )r   Nr   r   )r   r%   r   r   )r   r9   r   r   r   r   r    %test_chunked_array_c_stream_interface  s   
r   c                   @   r   )BatchWrapperc                 C   r   r   r   r   r   r   r   r    r   !  r   zBatchWrapper.__init__Nc                 C   r   r   )r   r   r   r   r   r    r   $  r   zBatchWrapper.__arrow_c_array__r   r   r   r   r   r    r      r   r   c                   @   r   )BatchDeviceWrapperc                 C   r   r   r   r   r   r   r    r   )  r   zBatchDeviceWrapper.__init__Nc                 K   s   | j j|fi |S r   )r   __arrow_c_device_array__)r   r   kwargsr   r   r    r   ,  s   z+BatchDeviceWrapper.__arrow_c_device_array__r   )r   r   r   r   r   r   r   r   r    r   (  r   r   wrapper_classc                 C   s   t jt jg dt  dgdgd}| |}t |}||ks!J t t dt  g}t j||d}t jt jg dt  dgdgd}||ksLJ d S Nr	   r   rm   rn   r   )r   record_batchrI   rB   r   r   ri   )r   r9   r   r   castable_schemarN   r   r   r    "test_recordbatch_c_array_interface0  s$   
r  c                  C   sn   t jt jg dt  dgdgd} tjtdd | jdd W d    n1 s*w   Y  | jd d}d S )	Nr	   r   rm   rn   z7Received unsupported keyword argument\(s\): \['other'\]r@   znot-none)other)r   r  rI   rB   r)   r*   NotImplementedErrorr   )r9   _r   r   r    =test_recordbatch_c_array_interface_device_unsupported_keywordF  s   r  c                 C   s   t jt jg dt  dgdgd}| |}t |}t j|g}||ks(J t t dt 	 g}t j||d}t dt jg dt 	 di}||ksQJ d S r   )
r   r  rI   rB   rs   rq   from_batchesr   r   ri   )r   r9   r   r   rN   r  r   r   r    test_table_c_array_interfaceV  s"   
r
  c                  C   s8  G dd d} t jt jg dt  dgdgdt jt jg dt  dgdgdg}| |}t |}t j|}||ks@J t j||d jd	}||ksPJ t t dt 	 g}t j||d	}||
|kslJ tjt jjd
d t j|t t dt t 	 gd	 W d    d S 1 sw   Y  d S )Nc                   @   r   )z4test_table_c_stream_interface.<locals>.StreamWrapperc                 S   r   r   )batches)r   r  r   r   r    r   o  r   z=test_table_c_stream_interface.<locals>.StreamWrapper.__init__Nc                 S   s"   t j| jd j| j}||S Nr   )r   RecordBatchReaderr	  r  r   r   )r   r   readerr   r   r    r   r  s   
zGtest_table_c_stream_interface.<locals>.StreamWrapper.__arrow_c_stream__r   r   r   r   r   r    StreamWrappern  r   r  r	   r   rm   rn   r   r   r  zField 0 cannot be castr@   )r   r  rI   rB   rs   rq   r	  r   r   ri   r   r)   r*   r   ArrowTypeErrorlist_)r  r9   r   r   rN   good_schemar   r   r    test_table_c_stream_interfacem  s(   "
"
"r  c                  C   s   t jtdddt jg dddg} t | ddg}g }| D ]}|| q |j|ks/J |t j||jdks;J |t j|d	d  |jd	d  dksOJ ||ksUJ d S )
Nr   r   r   r   r   r   Nrl   ri   c0c1rn   r
   )r   rI   rw   r  itercolumnsr   columnscolumn_names)r9   r   r  r   r   r   r    test_recordbatch_itercolumns  s   (r  c                  C   s  t jtdddt jg dddg} t g dt dgdgd	ggg}d
dg}t | |}|t | |ks8J |t | |sCJ |t ||ksMJ |t ||rXJ t j| |ddid}||siJ |j|ddrrJ |d ryJ |dksJ d S )Nr   r   r   r  ri   r   r   efr  r  keyvaluero   metadataTcheck_metadatar   )r   rI   rw   r  rK   )data1r:   r  r   
batch_metar   r   r    test_recordbatch_equals  s&   r%  c               	   C   s   t t g dt g dgddg} | t ddg| dds'J | t dd gt t dd gt dd ggddgsHJ d S )Nr
   r   r   Nr   rm   r   r   r   r  f1f2r   r   r   )r   r  rI   takerK   rp   r   r   r   r    test_recordbatch_take  s   &r+  c                  C   2   t jt g dgdgd} | d jdksJ d S Nr
   r   r   r   a0rn   r   )r   r  rI   _name)rbr   r   r    )test_recordbatch_column_sets_private_name     r2  c               	   C   sl   t ddg} t t dt t  g}tt t j	| g|d W d    d S 1 s/w   Y  d S )Nr
   r   f0r  )
r   rI   r   r   r  utf8r)   r*   r  r  )rE   r   r   r   r    ,test_recordbatch_from_arrays_validate_schema  s
   "r6  c                  C   sd   t dgt g dt dgg} tt t | g d W d    d S 1 s+w   Y  d S )Nr
   )tokyolikehappyderek)idtagsr   )r   rI   r)   r*   r+   r  rc   r   r   r    -test_recordbatch_from_arrays_validate_lengths  s   
"r=  c                  C   s<   t g g } t| dksJ | jdksJ | jdksJ d S r  )r   r  r0   r   num_columnsr   r   r   r    test_recordbatch_no_fields  s   r?  c                  C   s   t tdt g dg} tt t j| g dd W d    n1 s'w   Y  tt t j| dgd W d    d S 1 sEw   Y  d S Nr   r   r   rn   rm   )r   rI   rw   r)   r*   r+   r  rc   r   r   r    *test_recordbatch_from_arrays_invalid_names  s   "rA  c                  C   s@   t tdt g dg} t | ddg}|jjd u sJ d S )Nr   r   r  r  )r   rI   rw   r  r   r   r9   r   r   r   r    test_recordbatch_empty_metadata  s
   rC  c                 C   s   t jtdddt jg dddg}t dt  t dt  g}t j|dd	id
}t j||d}| | 	|}|
|sAJ |j|ksHJ d S )Nr   rC   r   r   r   intsfloats   foo   barr   r  )r   rI   rw   r   rC   r   r   r  r   r   rK   )r   r9   fieldsr   r   r   r   r   r    test_recordbatch_pickle  s   rJ  c                  C     t tdt g dt tddg} t jj| dd}|d|jds+J |d|jds9J t	t
 |d W d    n1 sNw   Y  t	t |d  W d    n1 shw   Y  t	t |d	 W d    d S 1 sw   Y  d S 
Nr   r   rl   r   rn   rm   r   r   r   )r   rI   rw   r   rr   r   rK   r   r)   r*   KeyErrorr_   rh   rB  r   r   r    test_recordbatch_get_field      "rN  c                  C      t tdt g dt tddg} t jj| dd}|d|ds*J tjt	dd	 |d
 W d    n1 sAw   Y  tt
 |d  W d    n1 s[w   Y  tt |d W d    d S 1 svw   Y  d S Nr   r   rl   r   rn   rm   r   z"Field "d" does not exist in schemar@   r   r   )r   rI   rw   r   rr   r   rK   r)   r*   rM  r_   rh   rB  r   r   r    test_recordbatch_select_column'  s"   "rR  c                  C   \  t g d} t g d}t ddgddgddgd d	d
gg}t | ||gg d}|dg}t | gdg}||s@J |ddg}t ||gddg}||sXJ |dg}t | gdg}||smJ |ddg}t ||gddg}||sJ |ddi}|ddg}d|jjv sJ tj	t
dd |dg W d    n1 sw   Y  tj	tdd |dg W d    n1 sw   Y  |ddg}t ||gddg}||sJ t | ||gg d}tj	t
dd |dg W d    n	1 sw   Y  |dg}t |gdg}||s,J d S Nr&  r'  r
   r   r   r   r   r   r   rl   )r(  r)  f3r(  rU  r)  r   rm   test   azField "f5" does not existr@   f5zindex out of bounds)r(  r)  r(  zField "f1" exists 2 times)r   rI   r  selectrK   replace_schema_metadatar   r   r)   r*   rM  rh   )a1a2a3r   r   rN   batch2r   r   r    test_recordbatch_select<  F   $r_  c                   C   sF   t t tjttd W d    d S 1 sw   Y  d S )Nr   )r)   r*   r_   r   r   from_struct_arrayrI   rw   r   r   r   r    *test_recordbatch_from_struct_array_invalido  s   "rb  c               	   C      t jddiddigt dt  fdt  fgd} t j| }|t jt jdd gt  dt jd dgt  dgddgsBJ d S NrD  r
   rE  r\   r   )	r   rI   r   ri   r   r   ra  rK   rr   struct_arrayr   r   r   r    "test_recordbatch_from_struct_arrayt     
rg  c               
   C   s   t jt jdd gt  dt jd dgt  dgddg} |  }|t jddiddigt dt  fdt  fgds@J d S Nr
   r   r\   rD  rE  )	r   r   rr   rI   ri   r   to_struct_arrayrK   r   )r   r   r   r   r     test_recordbatch_to_struct_array  s   rk  c                   C   sJ   t jtdd tjttd W d    d S 1 sw   Y  d S )Nz*Argument 'struct_array' has incorrect typer@   r   )r)   r*   r_   r   rq   ra  rI   rw   r   r   r   r    $test_table_from_struct_array_invalid  s   "rl  c               	   C   rc  rd  )	r   rI   r   ri   r   rq   ra  rK   rr   re  r   r   r    test_table_from_struct_array  rh  rm  c               	   C   s   t jddiddiggt dt  fdt  fgd} t j| }|t jt j	dd gt  dt j	d dgt  dgddgsCJ d S rd  )
r   r%   r   ri   r   rq   ra  rK   rr   rI   )chunked_struct_arrayr   r   r   r    *test_table_from_struct_array_chunked_array  s   
ro  c               
   C   s   t jt jdd gt  dt jd dgt  dgddg} |  }|t jddiddiggt 	dt  fdt  fgdsAJ d S ri  
r   rq   rr   rI   ri   r   rj  rK   r%   r   rs   r   r   r   r    test_table_to_struct_array  s   rr  c               
   C   s   t jt jdd gt  dt jd dgt  dgddg} | jdd}|t jddigddiggt 	dt  fdt  fgdsDJ d S )Nr
   r   r\   rD  rE  max_chunksizerp  rq  r   r   r    -test_table_to_struct_array_with_max_chunksize  s   ru  c                 C   sN   |  |sJ | j|ksJ | j|ksJ | j|jksJ | j|jks%J d S r   )rK   sizer   rV   strides)tensorexpected_tensorr   rv  r   r   r    check_tensors  s
   rz  typ_str)
uint8uint16uint32uint64rC   r   ri   rB   r   r`   c              	   C   sL  t | }g d}g d}g d}tjtj|t|dtj|t|dtj|t|dgg d}|jdd}t |||gj	|dd	}tj
|}t||t|d
 | }t |||gj	|dd	}tj
|}t||t|d
 |d}	g d}g d}g d}|	jdd}t |||gj	|dd	}tj
|}t||t|d |	 }t |||gj	|dd	}tj
|}t||t|d |dd}
g d}g d}g d}|
jdd}t |||gj	|dd	}tj
|}t||t|d |
 }t |||gj	|dd	}tj
|}t||t|d d S )N	r
   r   r   r   r   r   r   r   r   	rl         (   2   <   F   P   Z   	d   r  r  r  r  r  r  r  r  r   r   F	row_majorForder   Cr
   )r   r   r   r   r   r   r   r   )r  r  r  r  r  r  r  r  )r  r  r  r  r  r  r  r     r   )r   r   r   r   r   )r  r  r  r  r  )r  r  r  r  r     )rR   r   r   r   rr   rI   from_numpy_dtype	to_tensorcolumn_stackastypeTensor
from_numpyrz  rp   )r{  r   rX   rY   arr3r   r   r   rN   batch1r^  r   r   r    'test_recordbatch_to_tensor_uniform_type  sV   

r  c               	   C   s
  g d} g d}g d}t jt jtj| tjdt  dt jtj|tjdt  dt jtj|tjdt  dgg d}|jdd}t| ||gjtjd	d
}t j	
|}t||t  d | }t| ||gjtjdd
}t j	
|}t||t  d d S )Nr  r  r  r   r   r   Fr  r  r  r  r  )r   r   rr   rI   rR   float16r  r  r  r  r  rz  rX   rY   r  r   r   r   rN   r   r   r    +test_recordbatch_to_tensor_uniform_float_16  s$   r  c               	   C   s2  g d} g d}dddt jdddd	d
g	}tjtj| t dtj|t dgddg}|jdd}t 	| |gj
t jdd}tj|}t||t d | }t 	| |gj
t jdd}tj|}t||t d tjtj| t dtj|t dtj|t dgg d}|jdd}t 	| ||gj
t jdd}tj|}t j| | |jdksJ |jt ksJ |j|jksJ |j|jksJ | }t 	| ||gj
t jdd}tj|}t j| | |jdksJ |jt ksJ |j|jksJ |j|jksJ d S )Nr  r  r     ,    X         r   rm   r   Fr  r  r     r  r   r  )rR   nanr   r   rr   rI   r}  r   r  r  r  ri   r  r  rz  r   r`   r   assert_equalrT   rv  r   rV   rw  r  r   r   r    %test_recordbatch_to_tensor_mixed_type*  sP   r  c                  C   s   g d} g d}g d}t jt j| t  dt jtj|tjdt  dt j|t  dgg d}tj	t
dd |  W d    d S 1 sJw   Y  d S )	Nr  r  )	r  r  r  i  r  r  r  r  r  r   r   r   z.Casting from or to halffloat is not supported.r@   )r   r   rr   rI   r}  rR   r  r   r)   r*   r  r  )rX   rY   r  r   r   r   r    >test_recordbatch_to_tensor_unsupported_mixed_type_with_float16^  s    
"r  c               	   C   s   ddddt jddddg	} d	d
dddddt jdg	}tjtj| t dtj|t dgddg}|jdd}t | |gj	t jdd}tj
|}t j| | |jdksZJ |jt kscJ |j|jkskJ |j|jkssJ d S )Nr
   r   r   r   r   r   r   r   rl   r  r  r  r  r  r  r  r   rm   r   Fr  r  r  r  )rR   r  r   r   rr   rI   r   r  r  r  r  r  r   r  rT   rv  r   rV   rw  rX   rY   r   r   r   rN   r   r   r    test_recordbatch_to_tensor_nanr  s    r  c                  C   sV  g d} g d}t jt j| t  dt j|t  dgddg}tjt jdd |	  W d    n1 s8w   Y  |j	dd	d
}t
| |gjt
jdd}t j|}t
j| | |jdksgJ |jt  kspJ |j|jksxJ |j|jksJ t jt j| t  dt j|t  dgddg}|j	dd	d
}t
j| | |jdksJ |jt  ksJ |j|jksJ |j|jksJ t jt j| t  dt j|t  dgddg}|j	dd	d
}t
| |gjt
jdd}t j|}t
j| | |jdksJ |jt  ksJ |j|jks J |j|jks)J d S )N)	r
   r   r   r   Nr   r   r   r   )	rl   r  r  r  r  r  r  Nr  r   rm   r   z-Can only convert a RecordBatch with no nulls.r@   TF)null_to_nanr  r  r  r  )r   r   rr   rI   ri   r   r)   r*   r  r  rR   r  r  r`   r  r  r   r  rT   rv  r   rV   rw  rC   r  r   r   r    test_recordbatch_to_tensor_null  s`   
r  c                  C   s   t jt jg t  dt jg t  dgddg} |  }tg g gjtjdd}t j	
|}|j|jks8J |jt  ksAJ |j|jksIJ |jdksPJ d S )Nr   rm   r   r  r  )r   r   )r   r   rr   rI   r   r  rR   r  r  r  r  rv  r   rV   rw  )r   r   r   rN   r   r   r     test_recordbatch_to_tensor_empty  s   r  c                  C   s   g d} g d}t jt j| t  dt j|t  dgddg}tjt jdd |	  W d    d S 1 s9w   Y  d S )Nr  )	rm   r   r   rm   r   r   rm   r   r   r   rm   r   zDataType is not supportedr@   )
r   r   rr   rI   ri   r5  r)   r*   r  r  )rX   rY   r   r   r   r    &test_recordbatch_to_tensor_unsupported  s   
"r  c                 C   s  t tdt g dg}ddg}| ||d}|d}|jdks$J | dd	 |D |d}||s6J |dd}| d
d	 |D |d}||sNJ |d|sXJ t|t|dkseJ tt	 |d W d    n1 szw   Y  |dd|d d sJ |dd|d d sJ |dd|dd sJ |dt|d |dd  sJ |t|d d|dd  sJ |t|d d|dd sJ d S )Nr   r   r  r  rn   r   r   c                 S   s   g | ]}| d qS r   rp   r   r   r   r   r    
<listcomp>      z+_table_like_slice_tests.<locals>.<listcomp>c                 S   s   g | ]}| d d qS r  r  r  r   r   r    r        r   re   r   )
r   rI   rw   rp   r   rK   r0   r)   r*   rh   )factoryr9   ro   objslicedrN   sliced2	expected2r   r   r    _table_like_slice_tests  s.   
&&*r  c                   C   s   t tjjS r   )r  r   r   rr   r   r   r   r    test_recordbatch_slice_getitem  r   r  c                   C   s
   t tjS r   )r  r   rs   r   r   r   r    test_table_slice_getitem  r   r  c                  C   sZ   t dt jg t ddi} | dd}|  t dt jg t  di} |   d S )Nrm   usr   r   )r   rs   rI   r   rp   r   r%   r&   )rs   rt   r   r   r    test_slice_zero_length_table  s
   r  c                  C   s   t jdgdd} t jddgdd}tt| gdg}tt|gdg}ttj tj||g W d    d S 1 sAw   Y  d S )Nr
   r~  r   r>   r?   r`   r  )	rR   rI   r   r  r)   r*   ArrowInvalidrq   r	  )r[  r\  r  r^  r   r   r    "test_recordbatchlist_schema_equals   s   "r  c                  C   r,  r-  )r   rs   rI   r0  tr   r   r    #test_table_column_sets_private_name+  r3  r  c                  C   sf   t jjg g d} | | sJ | d rJ t jjg g ddid}| j|ddr*J | |s1J d S )Nrn   r  r  r  Tr!  )r   rq   rr   rK   )rs   r  r   r   r    test_table_equals1  s   r  c                  C   sR  t t dt  t dt  g} t jt dgt dggddgd}t j|g| }|j	| s6J |
dt dggksDJ |
dt dggksRJ t t dt  g}tt j t j|g| W d    n1 sww   Y  t t dggdg}tt j t j|g|  W d    d S 1 sw   Y  d S )Nrm   r   r
   gQ	@rn   r   )r   r   r   rB   r`   r  rI   rq   r	  rK   r   r%   r)   r*   r  )r   r   rs   incompatible_schemaincompatible_batchr   r   r    "test_table_from_batches_and_schema=  s&   "r  c            
      C   s`  ddl m}  dd l}|dttdi}|dttddi}tjj|dd}tjj|dd}tj	
|||g}|j|||gdd	}| }t|d
ksPJ | tj	
| | |jdd}ttt|g dksnJ | | | | tj	
| | tj	
t|||g}	||	sJ tt |jdd W d    d S 1 sw   Y  d S )Nr   )r   rm   rl   r  F)preserve_indexT)ignore_indexr   r  rs  )rl   r  r   rl   )r   r   r   r   r-   rw   r   r   r   rq   r	  concat
to_batchesr0   r   mapiterrK   r)   r*   r+   )
r   r   df1df2r  r^  rs   expected_dfr  table_from_iterr   r   r    test_table_to_batchesR  s0   "r  clsc           	   	   C   s   t jtdddt jg dddg}| j|dd}|  |jjr#J t|dks+J |jdks2J |j	t|ks;J |j
dksBJ | d	ksJJ |jd	ksQJ t|t||  ksaJ | }|td
g dfdg dfgkswJ t|ts~J || j||jdksJ tt |d  W d    n1 sw   Y  g }| D ]c}| t ju rt|t ju sJ | D ]}|d usJ qtt |d W d    n1 sw   Y  tt ||j W d    n1 sw   Y  nt t|t j!sJ |"| q|j#|ksJ || j||j$dks"J || j|dd  |j$dd  dks7J ||ks>J t jt j%dt & ddidt %dt ' gddid}| j||d}|j|kshJ t()|}| d usuJ ~| d u s~J d S )Nr   r   r   r  ri   r   rn   r   r      rm   )r   r
   r   r   r   r   r  r   re   r
   r  r  r  rH  r  rF  rG  )*r   rI   rw   rr   r(   r   r   r0   r   r>  rV   r#   r6   r2   r3   r4   r5   	to_pydictr   r   dictfrom_pydictr)   r*   rh   r  rq   r   r{   r/   r   
num_chunks
issubclassArrayr   r  r  r   r   ri   r7   r8   )	r  r9   rs   pydictr  r   r   r   r;   r   r   r    test_table_basicsu  sv   	




*
r  c                   C   r   )Nrq   r@   )r)   r*   r_   r   rq   r   r   r   r    test_table_dunder_init  r   r  c                  C   s   t ddg} t ddg}t jdt  tdddd	}t jd
t  dd}t jj| |gt ||gd}d|djv s?J |dj	du sIJ d S )Nr
   r   r   r   field1ABr   rH  field2Fnullabler  rW  r   )
r   rI   r   rB   r  rq   rr   r   r   r  )arr0rX   field0r  rs   r   r   r    0test_table_from_arrays_preserves_column_metadata  s   r  c                  C   s   t tdt g dg} tt t jj| g dd W d    n1 s(w   Y  tt t jj| dgd W d    d S 1 sGw   Y  d S r@  )r   rI   rw   r)   r*   r+   rq   rr   rc   r   r   r    $test_table_from_arrays_invalid_names  s   "r  c                  C   s   t tdg dg} tj| ddgd}tjj| ddgd}||s$J ttdt	 tdt
 g}tj| |d}tjj| |d}||sMJ d S )Nr   r   rm   r   rn   r  )r-   rw   r   rs   rq   rr   rK   r   r   r}  rB   )r9   r   rN   r   r   r   r    test_table_from_lists  s   
r  c                 C   s   t jddgddggt  dt jg dgt  dg}t jt dt  t dt  gd	d
id}t jj||d}| | 	|}|
  ||sMJ d S )Nr
   r   r   r   r   )somestringsN rD  strsrF  rG  rH  r  )r   r%   r~  r&   r   r   rq   rr   r   r   r(   rK   )r   r9   r   rs   r   r   r   r    test_table_pickle  s   r  c                  C   rK  rL  )r   rI   rw   rq   rr   r   rK   r   r)   r*   rM  r_   rh   r9   rs   r   r   r    test_table_get_field  rO  r  c                  C   rP  rQ  )r   rI   rw   rq   rr   r   rK   r)   r*   rM  r_   rh   r  r   r   r    test_table_select_column  s"   "r  c                  C   st   t jt g dt g dt g dgg dd} tjtdd | d W d    d S 1 s3w   Y  d S )	Nr	   r   r   r   rn   z"Field "a" exists 2 times in schemar@   rm   )r   rs   rI   r)   r*   rM  r   rs   r   r   r    !test_table_column_with_duplicates+  s   "r  c                 C   s   t tdt g dt tddg}| j|dd}t d|d j}|d||d }|||d }| j||d g d	d}||sIJ ||sPJ |d
||d }| j|d g| dd}||slJ d S )Nr   r   rl   r   rn   r   r
   r   )rm   r   r   r   r   )r   rm   r   r   )	r   rI   rw   rr   r   r   
add_columnappend_columnrK   )r  r9   rs   	new_fieldt2t3rN   t4r   r   r    test_table_add_column6  s$   	r   c                 C   s   t tdt g dt tddg}| j|dd}t d|d j}|d||d }t|}|d |d< | j|d	d}||sFJ d S )
Nr   r   rl   r   rn   r   r
   r   )r   r   r   )	r   rI   rw   rr   r   r   
set_columnr-   rK   )r  r9   rs   r  r  expected_datarN   r   r   r    test_table_set_columnT  s   	r  c           	      C   s   t td}t g d}t tdd}| j|||gdd}|ddg}|d}| j|gdd}||s;J | j||gd	d}||sKJ tjtd
d |dg W d   dS 1 sdw   Y  dS )z& drop one or more columns given labelsr   r   rl   r   rn   rm   r   r   r   r   Column 'd' not foundr@   r   N)	r   rI   rw   rr   drop_columnsrK   r)   r*   rM  )	r  rm   r   r   rs   r  r  exp_t2exp_t3r   r   r    test_table_drop_columnsm  s   	
"r
  c                  C   s   t td} t g d}t tdd}t jj| ||gdd}|ddg}|d}t jj|gdd}||s=J t jj||gd	d}||sNJ tjt	d
d |dg W d   dS 1 sgw   Y  dS )z, verify the alias of drop_columns is workingr   r   rl   r   rn   rm   r   r  r  r  r@   r   N)
r   rI   rw   rq   rr   droprK   r)   r*   rM  )rm   r   r   rs   r  r  r  r	  r   r   r    test_table_drop  s   
"r  c                 C   sr   t tdt g dt tddg}| j|dd}|d}|  | j|dd  dd}||s7J d S )	Nr   r   rl   r   rn   r   r
   r  )r   rI   rw   rr   remove_columnr(   rK   )r  r9   rs   r  rN   r   r   r    test_table_remove_column  s   	
r  c                 C   sx   t tdg}| j|dgd}|d}|  t|t|ks#J |d|d|d }|  |	|s:J d S )Nr   rm   rn   r   )
r   rI   rw   rr   r  r(   r0   r  r   rK   )r  r9   rs   r  r  r   r   r    test_table_remove_column_empty  s   

r  c                  C   sT   g } ddg}d}t jt|d tjj| |d W d    d S 1 s#w   Y  d S )Nrm   r   z?Length of names [(]2[)] does not match length of arrays [(]0[)]r@   rn   )r)   r*   r+   r   rq   rr   )r9   ro   messager   r   r    test_empty_table_with_names  s   "r  c                  C   s4   t g } | jg ksJ | t jg g sJ d S r   )r   rs   r  rK   rq   rr   r  r   r   r    test_empty_table  s   
r  c                 C   s
  t tdt g dt tddg}| j|g dd}|jg dks'J | j|g dd}|g d}|  |jg dksDJ ||sKJ |d}|  |jg dks]J ||sdJ d}tj	t
|d |d	 W d    d S 1 s~w   Y  d S )
Nr   r   rl   r   rn   )ehbeeseaz.names must be a list or dict not <class 'str'>r@   z
not a list)r   rI   rw   rr   r  rename_columnsr(   rK   r)   r*   r_   )r  r9   rs   rN   r  r  r  r   r   r    test_table_rename_columns  s&   	
"r  c                 C   s  t tdt g dt tddg}| j|g dd}|jg dks'J | j|g dd}|ddd	}|  ||ksBJ | j|g d
d}| j|g dd}|ddd	}|  ||ksfJ tjt	dd |ddd W d    d S 1 sw   Y  d S )Nr   r   rl   r   rn   )r  r   r  r  r  )rm   r   )rm   rm   r   )r  r  r  r  r@   )rm   r   )
r   rI   rw   rr   r  r  r(   r)   r*   rM  )r  r9   rs   rN   t1r  r   r   r    !test_table_rename_columns_mapping  s$   	"r  c                  C   s   t t dt  t dt  g} t t d| g}t jddg| d}t jddg|d}t jd	d
gt  d}t jj|||gg dd}|	 }|
  t jjt jddgt  dt jddgt  dt jddg| d|gg dd}||s}J d S )Nr   r   nestr   r   r   )   g      )@)   g      -@FTr   rn   r
   r   r   r   r  r  )za.xza.yzb.nestr   )r   r   r   r   r   rI   rP   rq   rr   r   r(   rK   )ty1ty2rm   r   r   rs   r  rN   r   r   r    test_table_flatten  s&   r!  c                  C   s   t jt dgt dggddgd} t jt dgt dggddgd}t j| |g}| }|  ||s=J |jD ]	}|j	dksIJ q@d S )Nr
   rm   r(  r)  rn   r   r   )
r   r  rI   rq   r	  rJ   r(   rK   r  r  )r  r^  rs   combinedr   r   r   r    test_table_combine_chunks1  s   
r#  c                  C   s
  t jt g d t g d t g dgg dd} t jt g d t g d t g dgg dd}t j| |g}|d	d
i}|ddj	
t ddgs`J |ddj	
t ddgssJ |ddj	
t g dsJ |ddj	
t ddgsJ |t  }t g d}t g d}|ddj	
|sJ |ddj	
|sJ |ddj	
|sJ |ddj	
|sJ | g dg dg ddksJ |jjd	d
iksJ d S )Nr   ){     r%    )TFNNr   rn   )r   r   Nr   )r%  r&  r&  N)FNNTs   key1s   value1r   r   r   r
   r   )r$  r%  r&  r%  r&  r   )r   r   Nr   r   r   Nr   )r$  r%  r%  r&  r%  r&  r&  N)TFNNFNNT)r   r  rI   r   rq   r	  rZ  r   r   r   rK   r   default_memory_poolr  r   r   )r  r^  rs   expected_dict_0expected_dict_1r   r   r    test_table_unify_dictionaries>  sR   r*  c                  C      t tdg dg} t tddg dg}tjjdd | D dd}tjjd	d |D dd}t||g}|  t|dksBJ tjjd
d t| |D dd}|	|sYJ d S )Nr   g      $g      g        r?   g      $@rl   r\   r]   r^   r>   r?   c                 S      g | ]}t |qS r   r   rI   r  r   r   r    r  m  r  z&test_concat_tables.<locals>.<listcomp>r   rn   c                 S   r.  r   r/  r  r   r   r    r  o  r  c                 S      g | ]\}}t || qS r   r/  r   r   r   r   r   r    r  v      )
r-   rw   r   rq   rr   concat_tablesr(   r0   rv   rK   r9   r:   r  r  r   rN   r   r   r    test_concat_tablesc  *   
r5  c                  C   sx   t jjttdgdd} t jjtdgdd}tjt jdd t j| |gdd}W d    d S 1 s5w   Y  d S )	Nrl   rm   rn   r   z@Unable to merge: Field a has incompatible types: int64 vs stringr@   
permissivepromote_options)	r   rq   rr   r-   rw   r)   r*   r  r3  )r  r  r  r   r   r    test_concat_tables_permissive}  s   "r;  c                  C   s`   t jjttdgdd} tjtdd t j| | gdd W d    d S 1 s)w   Y  d S )Nrl   r7  rn   z Invalid promote options: invalidr@   invalidr9  )	r   rq   rr   r-   rw   r)   r*   r+   r3  r  r   r   r    !test_concat_tables_invalid_option  s   "r=  c                   C   <   t t td g W d    d S 1 sw   Y  d S r   )r)   r*   AttributeErrorr   r3  r   r   r   r    test_concat_tables_none_table     "r@  c                  C   s   dd l } ttdt tdt g}td}| ||d}| tjgd |d}tj	j
||dd}tj	j
||dd}|j|jsJJ |jj|jd	d
rUJ t||g}|jj|jd	d
sgJ |j|jspJ d S )Nr   rm   r   abcdefghr   r   F)r   r  Tr!  )r   r   r   r   r&   r-   r   rR   r  rq   r   rK   r3  )r   r   valuesr  r  table1table2table3r   r   r    1test_concat_tables_with_different_schema_metadata  s   rG  c               	   C   s  t jt jddgt  dgdg} t jt jddgt  dgdg}tt t j	| |gdd	}W d    n1 s=w   Y  |
t jt jg d
t  dt jg dt  dgddgsdJ t jt jddgt  dgdg} t jt jddgt  dgdg}tjt jdd/ tt t j	| |gdd	 W d    n1 sw   Y  W d    d S W d    d S 1 sw   Y  d S )Nr
   r   r   int64_fieldr\   r]   float_fieldT)promoter
   r   NNNNr\   r]   r  z Schema at index 1 was different:r@   F)r   rq   rr   rI   rB   r   r)   warnsFutureWarningr3  rK   r*   r  r  r  r   r   r   r    &test_concat_tables_with_promote_option  s8   

"rP  c               	   C   s  t jt jddgt  dgdg} t jt jddgt  dgdg}t j| |gdd	}|t jt jg d
t  dt jg dt  dgddgsOJ t jt jddgt  dgdg}t j| |gdd	}|t jt jg dt  dgdgsJ d S )Nr
   r   r   rH  r\   r]   rI  defaultr9  rK  rL  r8  rH   )	r   rq   rr   rI   rB   r   r3  rK   ri   )r  r  r   r  r   r   r    !test_concat_tables_with_promotion  s0   


rR  c                  C   s   t jt jddgt  dgdg} t jt jddgt  dgdg}tjt jdd t j	| |gdd W d    d S 1 sAw   Y  d S )	Nr
   r   r   r  zUnable to merge:r@   rQ  r9  )
r   rq   rr   rI   rB   r   r)   r*   r  r3  r  r  r   r   r    'test_concat_tables_with_promotion_error  s   "rT  c                  C   s
  t tdt g dt g dt g dg} t jj| tdd}|d |d s/J |d	 |d
 s:J |d |d sEJ |d |d sPJ tt	 |d  W d    n1 sdw   Y  tt	 |d  W d    d S 1 s~w   Y  d S )Nr   r   r-  abbccddeefabcdrn   re   r   r  r   r
   r  r   r   r   )
r   rI   rw   rq   rr   tuplerK   r)   r*   rh   r  r   r   r    test_table_negative_indexing  s    

"r^  c                  C   r+  )Nr   r,  rl   r-  c                 S   r.  r   r/  r  r   r   r    r    r  z'test_concat_batches.<locals>.<listcomp>r   rn   c                 S   r.  r   r/  r  r   r   r    r    r  c                 S   r0  r   r/  r1  r   r   r    r    r2  )
r-   rw   r   r   rr   concat_batchesr(   r0   rv   rK   r4  r   r   r    test_concat_batches  r6  r`  c                  C   s   t jt jddgt  dgdg} t jt jddgt  dgdg}tjt jdd t 	| |g W d    d S 1 s?w   Y  d S )Nr
   r   r   r  z$not match index 0 recordbatch schemar@   )
r   r   rr   rI   rB   r   r)   r*   r  r_  rS  r   r   r    $test_concat_batches_different_schema  s   "ra  c                   C   r>  r   )r)   r*   r?  r   r_  r   r   r   r     test_concat_batches_none_batches  rA  rb  c                 C   s  t tdt g dg}| j|tdd}t t dt  t dt  g}t t dt  g}| t j	u r>d}nd	}d
| d}t
jt|d || W d    n1 s]w   Y  t
jt|d || W d    d S 1 szw   Y  d S )Nr   r   rV  rn   r  r   rm   rs   zrecord batchz1Target schema's field names are not matching the z's field names:.*r@   )r   rI   rw   rr   r]  r   r   ri   r   rq   r)   r*   r+   r   )r  r9   rs   target_schema1target_schema2cls_namer  r   r   r    &test_table_cast_to_incompatible_schema#  s.   	
"rf  c              
   C   s*  t jtdt  dt jg dt  dt jg dt  dt jg dt  dg}| j|tdd}t jtdt  dt jg dt 	 dt jg dt  dt jg dt  dg}| j|tdd}t 
t d	t  t d
t 	 t dt  t dt  g}||}||sJ d S )Nr   r   r   r-  rU  r[  rn   r
   r   r   r   r   rm   r   r   r   )r   rI   rw   rB   ri   r`   r&   rr   r]  r   r   r   r   rK   r  r9   rs   r  expected_tabletarget_schemacasted_tabler   r   r    test_table_safe_castingF  s(   	
rl  c              
   C   sh  t jtdt  dt jg dt  dt jg dt  dt jg dt  dg}| j|tdd}t jtdt  dt jg dt 	 dt jg dt  dt jg dt  dg}| j|tdd}t 
t d	t  t d
t 	 t dt  t dt  g}tjt jdd || W d    n1 sw   Y  |j|dd}||sJ d S )Nr   r   r   )g?g@gffffff
@g@g      @rU  r[  rn   rg  rm   r   r   r   	truncatedr@   F)safe)r   rI   rw   rB   ri   r`   r&   rr   r]  r   r   r   r)   r*   r  r   rK   rh  r   r   r    test_table_unsafe_castingi  s.   	ro  c                  C   s   t jddgt jd} t }tj| |dtj| dd  |dg}ttjj tjj	|ddgd W d    d S 1 s=w   Y  d S )Nr   r
   r   r   r[  r\  rn   )
rR   rI   r|  r   r)   r*   r   r  rq   rr   )rI   u8r   r   r   r    test_invalid_table_construct  s   $"rq  zdata, klassr  r   r   r   r   Nr  r   r   r   r?   c                 C   s  || d || d g} t dt  fdt  fg}t jj| |d}|jdks*J |jdks1J |j|ks8J t dt  fg}t	t
 t jj| |d W d    n1 sZw   Y  t dt  fdt  fg}t jj| |d}t j|djsJ |jdksJ |jdksJ |j|ksJ t dt  fdt dfg}t	ttf t jj| |d W d    n1 sw   Y  t	t
 t jj| |ddgd	 W d    n1 sw   Y  t	t
 t jj| |d
did W d    d S 1 sw   Y  d S )Nr   r
   r  rE  r  r   r   s)r   ro   rF  rG  r   r   )r   r   r5  r   rq   rr   r>  r   r)   r*   r+   types
is_float32r   r   r   r  r_   r  r9   klassr   rs   r   r   r    test_from_arrays_schema  s4    $rz  c              
   C   s  |  i }|jdksJ |jdksJ |jtg ksJ | i ks%J tdt fdt fg}tdg dfdg dfg}|  |}|jdksNJ |jdksUJ |j|ks\J | |ksdJ dd	i}|	|}| j ||d
}|j|ks{J |jj
|ksJ | |ksJ | j ||d}|j|ksJ |jj
|ksJ | |ksJ tt | j |||d W d    n1 sw   Y  tt | j dg ditdt fgd W d    n1 sw   Y  tjtdd( | j g dg ddtdt fdt fdt fgd W d    n	1 s!w   Y  tt | j dg dii d W d    d S 1 sCw   Y  d S )Nr   r  rE  rr  rs  r   r   rF  rG  rH  r  ru  r  )r   r
   r   zdoesn't contain.* c, dr@   r	   )r   r   r   r   rm   r   r   )r  r>  r   r   r   r  r5  r`   r   with_metadatar   r)   r*   r+   r_   r&   rM  rB   ri   r   )r  rs   r   r9   r   r   r   r    test_table_from_pydict  sZ   






$r|  c                 C   s  t d|| d fd|| d fg} tdt fdt fg}tj| }|jdks.J |jdks5J |j|ks<J tjj| |d}|jdksKJ |jdksRJ |j|ksYJ tdt fdt	 fg}tjj| |d}tj
|djs|J |jdksJ |jdksJ |j|ksJ tdt fdtdfg}tttf tjj| |d W d    d S 1 sw   Y  d S )	Nr  r   rE  r
   r   r   r  rt  )r   r   r   r5  r`   rq   r  r>  r   r   rv  rw  r   r   r   r)   r*   r  r_   rx  r   r   r    #test_table_from_pydict_arrow_arrays	  s(   $ "r}  c                 C   s   t d|| d fd|| d fg} tdt fdt fdt fg}tjtdd tj	j
| |d W d    n1 s@w   Y  tdt fg}tj	j
| |d}|jdks^J |j|kseJ |jdgksmJ d S )Nr  r   rE  r
   rD  r@   r  )r   r   r   r5  r`   rB   r)   r*   rM  rq   r  r>  r  rx  r   r   r    test_table_from_pydict_schema#	  s   $
r~  c                 C   s  |  g }|jdksJ |jdksJ |jtg ksJ | g ks%J tdt fdt fg}ddddddd	d dg}|  |}|jd
ksNJ |jdksUJ |j|ks\J | |ksdJ ddi}||}| j ||d}|j|ks{J |jj	|ksJ | |ksJ | j ||d}|j|ksJ |jj	|ksJ | |ksJ t
t | j |||d W d    n1 sw   Y  t
t  | j ddiddidd
igtdt fgd W d    n1 sw   Y  tdt fdt fdt fg}| j dddd
dddddg|d}dd d dd
d d ddd d dg}|j|ks3J | |ks<J t
t | j ddidd
iddigi d W d    n	1 s]w   Y  dddddidd	ig}dddd ddd	d dg}|  |}|jd
ksJ |jdksJ | |ksJ ddiddddd ig}ddiddidd ig}|  |}|jdksJ |jdksJ | |ksJ d S )Nr   r  rE  r  r   )r  rE  r   r   r   r   r   rF  rG  rH  r  ru  r  r
   rm   r   r   r   r   )rm   r   r   )from_pylistr>  r   r   r   r'   r5  r`   r{  r   r)   r*   r+   r_   r&   rB   ri   r   )r  rs   r   r9   r   r:   r   r   r    test_table_from_pylist;	  s   







"

r  c                  C   s  dd l } | tdg dfdg dfg}tdt fdt fg}tjj||d}tj	
|djs8J |j |ksAJ tdt fdtdfg}tttf tjj||d W d    n1 skw   Y  tdt fdt fdt fg}tjtdd	 tjj||d W d    n1 sw   Y  tdt fg}tjj||d}|jd
ksJ |j |ksJ |jdgksJ d S )Nr   r  rr  rE  rs  r  rt  rD  r@   r
   )r   r   r   r   r   r5  r   rq   r   rv  rw  r   r   remove_metadatar   r)   r*   r  r_   r`   rB   rM  r>  r  )r   r   r   rs   r   r   r    test_table_from_pandas_schema	  s.   
 
r  c                  C   s  dd l } tdg dfdg dfg}tjg dddtjg dddd	}tdt fdt fg}| |}t|}tj	
|}||sKJ tj||d
}tj	j
||d
}||saJ t|}tj	|}||ssJ tj||d
}tj	j||d
}||sJ d S )Nr   r   r   rm   r	   r&   r   ri   )r   rm   r  )r   r   r   rI   r   ri   r&   r   rs   rq   r   rK   r  )r   r   
d_explicitr   r   rD  rE  r   r   r    test_table_factory_function	  s&   


r  c                  C   s   t t tjdg didgd W d    n1 sw   Y  tdt fg} tdtjg dt di| }|	dj
t ksHJ tjg dddg}dg}t||}|j|ksbJ tdt fg} t|| }|j|ksyJ d S )Nrm   r	   rn   r   rB   )r)   r*   r+   r   rs   r   ri   rI   rB   r   r   r  )r   rs   r9   ro   r   r   r     test_table_factory_function_args	  s   "r  c                  C   s   dd l } tt tj| dg didgd W d    n1 s#w   Y  tt tj| dg diddid W d    n1 sHw   Y  tdt fg}t| dg di|}|	dj
t kspJ d S )Nr   rm   r	   rn   rF  rG  rH  )r   r)   r*   r+   r   rs   r   r   ri   r   r   )r   r   rs   r   r   r    'test_table_factory_function_args_pandas	  s    "r  c                   C   sv   t jtdd td W d    n1 sw   Y  t jtdd td W d    d S 1 s4w   Y  d S )Nz!Expected pandas DataFrame, pythonr@   zinvalid inputzExpected pandas DataFrame)r)   r*   r_   r   rs   r  r   r   r   r    $test_factory_functions_invalid_input	  s   "r  c                  C      t jt jdt  ddidt dt  gddid} t jt jg dd	d
t jg ddd
g| d}t|dks:J |jdddksDJ |jdddksNJ |jdddksXJ d S )Nr  r  r  rH  r  rF  rG  r.  r   r   rl   r  r  r  ri   r  zJpyarrow.Table
c0: int16
c1: int32
----
c0: [[1,2,3,4]]
c1: [[10,20,30,40]]Tshow_metadatazhpyarrow.Table
c0: int16
  -- field metadata --
  key: 'value'
c1: int32
-- schema metadata --
foo: 'bar'r   preview_colsr
   z:pyarrow.Table
c0: int16
c1: int32
----
c0: [[1,2,3,4]]
...)	r   r   r   r   ri   rs   rI   rb   	to_stringr   tabr   r   r    test_table_repr_to_string
  s    	r  c                  C      t jt jdt  ddidt dt  gddid} t jt jg dd	 d
dt jg dd	 ddg| d}t|dks>J d S )Nr  r  r  rH  r  rF  rG  r.  rl   r   r   r  ri   r  zppyarrow.Table
c0: int16
c1: int32
----
c0: [[1,2,3,4,1,...,4,1,2,3,4]]
c1: [[10,20,30,40,10,...,40,10,20,30,40]])r   r   r   r   ri   rs   rI   rb   r  r   r   r    "test_table_repr_to_string_ellipsis-
     r  c                  C   r  )Nr  r  r  rH  r  rF  rG  r.  r   r   r  ri   r  zLpyarrow.RecordBatch
c0: int16
c1: int32
----
c0: [1,2,3,4]
c1: [10,20,30,40]Tr  znpyarrow.RecordBatch
c0: int16
  -- field metadata --
  key: 'value'
c1: int32
-- schema metadata --
foo: 'bar'r   r  r
   z>pyarrow.RecordBatch
c0: int16
c1: int32
----
c0: [1,2,3,4]
...)	r   r   r   r   ri   r  rI   rb   r  r   r   r   r   r     test_record_batch_repr_to_string?
  s    	r  c                  C   r  )Nr  r  r  rH  r  rF  rG  r.  rl   r   r   r  ri   r  zpyarrow.RecordBatch
c0: int16
c1: int32
----
c0: [1,2,3,4,1,2,3,4,1,2,...,3,4,1,2,3,4,1,2,3,4]
c1: [10,20,30,40,10,20,30,40,10,20,...,30,40,10,20,30,40,10,20,30,40])r   r   r   r   ri   r  rI   rb   r  r   r   r    )test_record_batch_repr_to_string_ellipsisj
  r  r  c                  C   s   d} d}t |g df| g dfg}t| t f|t fg}tj||d}|d dtjg ddds;J |d	 dtjg dd
dsNJ d S )Nu   ääähu   öööfr   r	   r  r   ri   r   r
   r&   )	r   r   r   ri   r&   rs   r   rK   rI   )col_acol_br   r   r   r   r   r    "test_table_function_unicode_schema|
  s   &*r  c                  C   sR   t t g dt g dgddg} | t ddg| dds'J d S )Nr&  r'  r(  r)  r   r   )r   rs   rI   r*  rK   rp   r  r   r   r    %test_table_take_vanilla_functionality
  s   *r  c                  C   sr   t t g dt g dgddg} t t dd gt dd ggddg}| t dd g|s7J d S )Nr&  r'  r(  r)  r
   rm   r   r   rs   rI   r*  rK   )rs   result_with_null_indexr   r   r    test_table_take_null_index
     "r  c                  C   sr   t t g dt g dgddg} t t dd gt ddggddg}| t dd	g|s7J d S )
Nr&  r'  r(  r)  r   r   r   r
   r   r  )rs   result_non_consecutiver   r   r    test_table_take_non_consecutive
  r  r  c                  C   rS  rT  )r   rI   rs   rY  rK   rZ  r   r   r)   r*   rM  rh   )r[  r\  r]  rs   r   rN   rE  r   r   r    test_table_select
  r`  r  c                  C   s  dd } t jt g dt g dt g dt g dgg dd}|d	d
g}| | g dg ddks?J |d	d
dg}| | g dg dg ddks]J |d	dg}| | g dg ddkswJ |d	ddg}| | g dg dg ddksJ |d	ddg}| | g dg dg ddksJ |d	dgdg}| | g dg dg ddksJ |d	dddg dfdg}| | g dg dg dg dg dg dd ksJ t jt g d!t g d"gd	d#gd}|d	gd#d$tjd%d&fg}| d'gd(gd)ks2J |d	gd#d$tjd*d&fg}| d'gd+gd)ksQJ |d	gd#d$tjd,d&fg}| d'gd-gd)kspJ |d	gg dfd#d$tjd,d&fg}| d'gd(gd-gd.ksJ |d	gg dfg}| d'gd(gd/ksJ t g d0td1d2}t j	
|jd(d3}|d	dg}| | d'd4gd1d5gdksJ d S )6Nc                 S   s   d| v rt t| d | d }n| d }t|}dt| d i}| D ] }|dkr)q"tt|| | }|D ]}||g ||  q4q"|S )Nkeys2keys)r]  rv   sortedr  
setdefaultr   )r   r  sorted_keyssorted_dentryrC  kr   r   r    sorted_by_keys
  s   z+test_table_group_by.<locals>.sorted_by_keysrm   rm   r   r   r   )Xr  YZr  rg  )rl   r  r  r  r  )r  r  rC  	bigvaluesrn   r  )rC  hash_sumr   )r   r   r   )r  
values_sum)rC  
hash_count)r   r   r
   )r  r  values_count)rC  r1   )rC  max)r  r1   )r   r   r   )r  r  r  )r  
values_maxbigvalues_sum)r  r  )r  r  r  )r  r  bigvalues_maxr  )rm   r   r   r   )r  r  r  r  )r   r   r   r   )r  r  r  	count_all)r  r  r  r  r  r  )rm   rm   rm   )r
   NNrC  countr.   )moderm   r   )r  r  	only_nullr   
only_validr
   )r  r  r  )r  r  )rm   r   rm   r   rm   r   r   r  rC  rs  r   r   )r   rs   rI   group_by	aggregater  pcCountOptionsrw   rq   r	  r  )r  rs   rtable_with_nullstable_with_chunksr   r   r    test_table_group_by
  s   












	

r  c                  C   s   t g dddgd d} t g dddgd d}t | |g}tt |ddg W d    n1 s<w   Y  |jddddg}t ddgd	dgd
}||s_J d S )Nr.  rm   r   r   r   )rm   firstF)use_threadsr
   )r   a_first)	r   rs   r3  r)   r*   r  r  r  rK   )rD  rE  rs   r   rN   r   r   r    test_table_group_by_first  s   r  c                  C   sp   t jdg di} |  }| j|jksJ | | ksJ | jdd}| jdks-J | jdks6J d S )Nr   r	   r   rs  r
   )r   rq   r  	to_readerr   read_allread_next_batchr   )rs   r  r   r   r    test_table_to_recordbatchreader  s   r  c                  C   s   t g dg dd} t g dg dd}| |dd}| t g dg dg d	d
ks4J | j|dddd}| dt g dg dg dd
ksUJ d S )Nr
   r   r   rm   r   r  colAcol2rf   r   r
   r  r  r  )colBcol3r  r  r  r  Nr  r  r  
full outer	join_typer
   r   r   rf   rm   r   r  Nr  r  Nr  r   rs   joinrJ   sort_byrO  r   r   r    test_table_join  s(   
r  c                  C   s   t g dg dd} t g dg dd}| |d}| t g dg dg dd	ks3J | j|dd
dd}| dt g dg dg dd	ksTJ d S )Nr  r  r  r  r  )r  r  r  r  r  r  _r)r  right_suffixr  r  r  r  rO  r   r   r    test_table_join_unique_key  s(   
r  c                  C   s   t g dg dg dd} t g dg dg dd}| j|dd	d
}| dt jg dg dg dg dg dgg ddksGJ d S )Nr  rl   r  r  r  r  r  colValsr  rf   r  rl   r  r  r  r  r  )rl   r  r  Nr  )rl   r  Nrf   r  )r  r  r  r  r  rn   r  rO  r   r   r    test_table_join_collisions  s(   r  c                 C   s\   |  g dg dg dd}|tddk }||  ddgd	d
gddgdks,J d S )N)r
   r   r   r   )rl   r  Nr  )rm   r   r   r  r  r  r  r
   r   rl   r  rm   r   )r  filterr  r   )r  r  r   r   r   r    test_table_filter_expression  s   r  c                  C   s   t g dg dg dd} t g dg dg dd}t | |g}|tdd	k }| t g d
g dg ddksDJ d S )Nr  r  r  r  r  r  r  r  rl   )r
   r   r   r   r
   )rl   r  r  r  rl   )rm   r   r  r  r  )r   rs   r3  r  r  r   rJ   )r  r  r  r   r   r   r    #test_table_filter_expression_chunks  s"   
r  c                  C   s   t g dg dd} t g dg dg dg dg dg dd}| |dd}| t g dg dg d	g d	g d	g d	g d	d
ksLJ | j|dddd}| dt g dg dg dg dg dg dg dd
ksyJ d S )Nr  r  r  r  r  )r  r  col4col5col6col7r  r  r  )r  r  r  r  r  r  r  r  r  r  r  r  r  rO  r   r   r    test_table_join_many_columns  s@   	

r  c                  C   s~   t jg dg dd} t jg dg dg dd}| j|dd	d
ddd}| t g dg dg ddks=J d S )N)r
   r
   r   r   r   )rm   r   rm   r   r  r  )r   r   r  )rm   r   g)r\   r^   r?   )r  r  colCr  r  r
   r  r  onby	toleranceright_onright_by)r\   NNNN)r  r  r  )r   rq   r  	join_asofrJ   rs   )r  r  r  r   r   r    test_table_join_asof6  s$   
r  c                  C   s   t g dg dg dd} t g dg dg dg dd	}| j|d
ddgdd}|dt g dg dg dg ddksEJ d S )Nr  r  r	   r  r  r  r  r  r  r   r   r   )r  r  r  r  r  r  r  r
   r  r  r  )Nr  Nr  r  r  r  )r   rs   r  r  rO  r   r   r     test_table_join_asof_multiple_byN  s(   r  c                  C   s`   t dg di} t g dg dd}| j|dg dd}|t g dg dd	ks.J d S )
Nr  r	   r  r  )r  r  r
   r   )r  r  r  )r  r  )r   rs   r  rO  r   r   r    test_table_join_asof_empty_byh  s   r  c               	   C   s   t g dg dg dg dd} t g dg dg dg d	g d
d}d}tjt|d | j|dddgddddgd W d    d S 1 sKw   Y  d S )Nr  r  r	   r  r  r  r  r  r  r  r  r  )r  r  colUniqr  r  zXColumns {'colVals'} present in both tables. AsofJoin does not support column collisions.r@   r  r  r  r
   r  )r   rs   r)   r*   r+   r  r  r  rF   r   r   r    test_table_join_asof_collisions|  s*   	"r  c               	   C   s   t g dg dg dd} t g dg dg dg dd	}d
}tjt jj|d | j|dddgdddgd W d    d S 1 sFw   Y  d S )Nr  r  r	   r  r  r  r  r  r  r  r  r  z)inconsistent size of by-key across inputsr@   r  r  r  r
   r  r   rs   r)   r*   r   r  r  r  r   r   r    'test_table_join_asof_by_length_mismatch  s$   "r
  c               	   C      t g dg dd} t g dg dg dg dd}d	}tjt jj|d
 | j|ddgdddgd W d    d S 1 sBw   Y  d S )Nr  r	   r  r  r  r  )g     X@r]   r\   r  r  zCExpected by-key type int64 but got double for field colA in input 1r@   r  r  r
   r  r	  r  r   r   r    %test_table_join_asof_by_type_mismatch  "   
"r  c               	   C   r  )Nr  r	   r  r  r  r  )r]   r^   r>   r  zAExpected on-key type int64 but got double for field on in input 1r@   r  r  r
   r  r	  r  r   r   r    %test_table_join_asof_on_type_mismatch  r  r  c                 C   s   |  d dgd dgd}ttjddddtjddd	dg}tt || W d    n1 s4w   Y  |  d dgd	dgd}||j|ksOJ d S )
Nr
   Tr   rm   rB   r  r   boolF)r  r   r   r   r)   r*   r+   r   )r  rs   
new_schemar   r   r    test_table_cast_invalid  s   	r  c                 C   s"  | j tg dtg dgddgd}|d g dg ddks'J |d	g g d
g ddks:J | j tjg dt dtg dgddgd}|dg}| }|d g dksgJ |d g dksqJ |dg}| }|d g dksJ |d g dksJ d S )N)r   r
   r   r   r   )r   rm   r   rm   r   rC  r  rn   r  rg  r  )rC  
descending)r   r   r   rm   rm   )r   r   r   r   r
   )r   r   r   #   r   )r   carr   foobarrm   r   )rm   r  )r  r   r   r   )r  r  r   r   )rm   	ascending)rr   r   rI   r  r  rB   )r  rs   r  
sorted_tabsorted_tab_dictr   r   r    test_table_sort_by  s8   r  constructorc                 C   sL  | g dg dgddgd}t |}t jddgdd	gd
dggdd}t j|| t j|dd}t j|| |jdks?J |g }t |}t d}t j|| |jdks\J t j|dd}t j|| |jdksqJ |dd}t |}t d}t j|| |jdksJ t j|dd}t j|| |jdksJ d S )Nr	   r=   rm   r   rn   r
   r   r   r   r   r   r`   r   ri   )r   r   r   r   )	rR   rS   rI   r   assert_allcloser   rY  emptyrp   )r  rs   r   rN   rE  rF  r   r   r    test_numpy_asarray  s.   
 




r  c                 C   s   | g dg dgddgd}t jddgdd	gd
dggdd}tt jtdk r7t j|dd}t j|| d S tjtdd t j|dd W d    d S 1 sQw   Y  d S )Nr	   r=   rm   r   rn   r
   r   r   r   r   r   r`   r   z
2.0.0.dev0F)copyzUnable to avoid a copyr@   )	rR   rI   r   __version__r   assert_array_equalr)   r*   r+   )r  rs   rN   r   r   r   r    test_numpy_array_protocol6  s    "r"  c                  C   s   d} t jt| dd t| D d}t jt| dd t| D d}tt jj}|j|ddd	 W d    n1 s@w   Y  d
}|t	|j
v sPJ tt jj}|j|ddd	 W d    n1 sjw   Y  |t	|j
v sxJ d S )Nr  c                 S   s   g | ]}d d t dD qS )c                 S      g | ]}|qS r   r   )r   zr   r   r    r  K      z;test_invalid_non_join_column.<locals>.<listcomp>.<listcomp>r   )rw   r  r   r   r    r  K  s    z0test_invalid_non_join_column.<locals>.<listcomp>)r;  array_columnc                 S   r#  r   r   r  r   r   r    r  O  r%  )r;  r  r;  innerr  zOData type list<item: int64> is not supported in join non-key field array_column)r   rq   r  rw   r)   r*   r   r  r  rb   r  )	NUM_ITEMSr  r  excinfoexp_error_msgr   r   r    test_invalid_non_join_columnF  s$   r+  c                  C   s   t d} | dS )Nzpyarrow.cudar   )r)   importorskipContext)cudar   r   r    cuda_context_  s   

r/  c                   C   s&   t t dt  t dt  gS )Nr  r  )r   r   r   ri   r   r   r   r    r   e  s   &r   c                 C   s0   t g d| djt g d| djgS )Nrg  r   r  r
   )r   rI   r   r   r  r   r   r    
cpu_arraysj  s   r0  c                    s    fdd|D S )Nc                    s   g | ]}|  jqS r   copy_tomemory_manager)r   rE   r/  r   r    r  r  r  zcuda_arrays.<locals>.<listcomp>r   )r/  r0  r   r4  r    cuda_arraysp     r5  c                 C      t | }|jdu sJ |S )NTr   r%   is_cpu)r0  r%   r   r   r    cpu_chunked_arrayu     
r:  c                 C   r7  NFr8  )r5  r%   r   r   r    cuda_chunked_array|  r;  r=  c                 C   s    t | | }|jdu sJ |S r<  r8  )r0  r5  r%   r   r   r    cpu_and_cuda_chunked_array  s   r>  c                 C   s   t j| |dS Nr  )r   r  )r0  r   r   r   r    cpu_recordbatch  s   r@  c                 C   s   | | jS r   r1  )r/  r@  r   r   r    cuda_recordbatch  s   rA  c                 C      t j||g| dS r?  r   rs   )r   r:  r   r   r    	cpu_table  r6  rD  c                 C   rB  r?  rC  )r   r=  r   r   r    
cuda_table  r6  rE  c                 C   s   t j||g| dS r?  rC  )r   r:  r=  r   r   r    cpu_and_cuda_table  r6  rF  c                 C   s^  |j |j ksJ | | ksJ t|t|ksJ t|t|v s&J |  tt |jdd W d    n1 s@w   Y  tt |j W d    n1 sXw   Y  tt |j	 W d    n1 spw   Y  tt |
  W d    n1 sw   Y  tt |d  W d    n1 sw   Y  tt |  W d    n1 sw   Y  tt |  W d    n1 sw   Y  tt |  W d    n1 sw   Y  tt |d W d    n	1 sw   Y  tt ||k W d    n	1 s"w   Y  tt |  W d    n	1 s<w   Y  tt |  W d    n	1 sVw   Y  tt |  W d    n	1 spw   Y  tt |  W d    n	1 sw   Y  tt |  W d    n	1 sw   Y  tt |  W d    n	1 sw   Y  tt |  W d    n	1 sw   Y  tt |  W d    n	1 sw   Y  tt |  W d    n	1 sw   Y  tt |g d W d    n	1 s)w   Y  tt |d W d    n	1 sDw   Y  |dd tt |dg W d    n	1 sfw   Y  tt |  W d    n	1 sw   Y  tt |  W d    n	1 sw   Y  tt |  W d    n	1 sw   Y  |j|jksJ t |j!t |j!ksJ |"d}|j#t$j%j&ksJ tt |'  W d    n	1 sw   Y  tt |(  W d    n	1 sw   Y  tt |)  W d    d S 1 s(w   Y  d S )NTfullr   TFTFTr   r   r
   )*r   lengthrb   reprr(   r)   r*   r  
null_countr6   r#   is_nullis_nanis_valid	fill_nullr   rT   	__array__r   r   r   rJ   uniquevalue_countsr  r   rp   r*  	drop_nullsortr   r  r0   r,   r   device_typer   DeviceAllocationTypeCUDAr'   r   
__reduce__)r/  r:  r=  r>  r   r   r   r    test_chunked_array_non_cpu  s   





















$rZ  c                 C   s   |    | jtjjksJ | jdu sJ | jt|jksJ | j	|jks&J t
| t| v s0J | jD ]}|jtjjks>J q3| j|ksFJ d S r<  )r(   rV  r   rW  rX  r9  r>  r0   ro   r  rb   rK  r  r   )r   expected_schemar   r   r   r    verify_cuda_recordbatch0  s   
r\  c              
      s  t | d |j}|jdksJ t|jdksJ |j|jfD ]G}tg dt	 
| j}|dd|}t | tdt	 d d}	tjt|	d |dddg|  W d    n1 saw   Y  q|d}t | dd |d	g}t | dd |d
d	g}
t|
jdksJ |
jtjjksJ |d
g}t | dd ttd
t td	t g}tt || W d    n1 sw   Y  tjdg| g dd
| j}|dd|}tt |  W d    n	1 sw   Y  tt |dg|  W d    n	1 s#w   Y  tt |dg W d    n	1 s?w   Y  tt |d
 W d    n	1 sZw   Y  |d dkslJ |d dksyJ |
| j}tt | |du sJ W d    n	1 sw   Y  tj!"|d
d	g}t | d |
t#  |sJ tj!$|d |d d}t | d |
t#  |sJ  fddt%t j&D }tj'j"||d}tt tj!(| W d    n	1 sw   Y  tt |j)sJ W d    n	1 s$w   Y  tt |* s6J W d    n	1 sAw   Y  tt |+  W d    n	1 s[w   Y  tt |,  W d    n	1 suw   Y  tt |-  W d    n	1 sw   Y  tt |.  W d    n	1 sw   Y  tt |/  W d    n	1 sw   Y  tt |0  W d    n	1 sw   Y  |1dd}t | d |jdksJ |
t# }||1ddksJ |2ddi}t | d |jj3ddiks J |4ddg}ttd dj5td dj5g}t ||d |6  tt |j6dd W d    n	1 s_w   Y  tt |7  W d    n	1 syw   Y  tt |8  W d    n	1 sw   Y  tt |9  W d    n	1 sw   Y  tt t:|;  W d    d S 1 sw   Y  d S )Nr[  r  r   )r  re   r   r
   r   c2z`Got column on device <DeviceAllocationType.CPU: 1>, but expected <DeviceAllocationType.CUDA: 2>.r@   r
   r  r  r   rI  maskTr  r  c                       g | ]}  |qS r   r   r   rx   r  r   r    r    r  z,test_recordbatch_non_cpu.<locals>.<listcomp>rI  r      key   valuecol0col1rG  )<r\  r   rV   r0   r  r  r  r   rI   rC   r2  r3  r   r   r)   r*   r_   r  remover  rV  rW  rX  rY  r   rB   r  r   rT  r  r*  r  rK   r   rr   default_cpu_memory_managerr  rw   ro   StructArrayra  r6   r#   r  r'   r   r  rj  	serializerp   rZ  r   r  r   r(   rQ  r   r   r   __dataframe__)r/  r@  rA  r5  r   Nfnr   	new_batcherr_msgempty_batchr  null_colcuda_recordbatch_with_nullsrI  rf  	cpu_batchr[  r   r  r    test_recordbatch_non_cpu<  s   
&









$rw  c                 C   s   |    | jdu sJ | jt|jksJ | j|jksJ t| t| v s'J | jD ] }|jdu s3J |	 D ]}|jdu s@J |j
tjjksIJ q7q*| j|ksRJ d S r<  )r(   r9  r>  r0   ro   r  rb   rK  r  r/   rV  r   rW  rX  r   )rs   r[  r   r   r   r   r    verify_cuda_table  s   
rx  c              
      s  t | d |j}|jdksJ t|jdksJ |j|jfD ]S}tdg| t	 }|
| j}	|dd|	}
t |
 tdt	 d |dd|}
|
jdu sTJ |
djdu s^J |
djdu shJ |
djdu srJ q|d}
t |
 dd |d	g}
t |
 dd |d
d	g}
t|
jdksJ |
jsJ |d
g}
t |
 dd ttd
t td	t g}tt || W d    n1 sw   Y  tjdg| dg| d
| j}|dd|}tt |  W d    n	1 sw   Y  tt |dg|  W d    n	1 s,w   Y  tt |dg W d    n	1 sHw   Y  tt |d
 W d    n	1 scw   Y  |d dksuJ |d dksJ tt ||sJ W d    n	1 sw   Y  tj |d
d	g}
t |
 d tj!|d |d d}
t |
 d  fddt"t j#D }tj$j ||d}tt tj%| W d    n	1 sw   Y  tj&||g }
t |
 d tt |j'sJ W d    n	1 sw   Y  tt |( s,J W d    n	1 s7w   Y  tt |)  W d    n	1 sQw   Y  tt |*  W d    n	1 skw   Y  tt |+  W d    n	1 sw   Y  tt |,  W d    n	1 sw   Y  |j-dd}|D ]}tt. t/| d W d    n	1 sw   Y  q|j0dd}|D ]}tt. t/| d W d    n	1 sw   Y  q|1dd}
t |
 d |
jdksJ |2ddi}
t |
 d |
jj3ddiks J |4ddg}
ttd dj5td dj5g}t |
|d |6  tt |j6dd W d    n	1 s_w   Y  tt |7  W d    n	1 syw   Y  tt |7  W d    n	1 sw   Y  tt |8  W d    n	1 sw   Y  tt |9d
 W d    n	1 sw   Y  tt |:|d
 W d    n	1 sw   Y  tt |;|d
d
d W d    n	1 sw   Y  tt |<  W d    n	1 sw   Y  tt |=  W d    n	1 s6w   Y  tt t>|?  W d    n	1 sRw   Y  tt |@  W d    d S 1 smw   Y  d S )Nr]  )rl   r   r   r
   r^  Fr   Tr  r  r_  ra  c                    rb  r   rc  rd  r  r   r    r  9  r  z&test_table_non_cpu.<locals>.<listcomp>re  r   rs  r   rf  rg  rh  ri  rG  )Arx  r   rV   r0   r  r  r  r   rI   rC   r2  r3  r   r   r9  r   r  rj  r  rY  r   rB   r)   r*   r  r   rT  r  r*  r  rK   rq   rr   r  rw   ro   rl  ra  r	  r6   r#   r  r'   r   rj  r  AssertionErrorr\  r  rp   rZ  r   r  r   r(   r   r   r  r  r  rQ  r   r   rn  rY  )r/  rD  rE  r5  rA  r   ro  rp  cpu_colcuda_col	new_tabler  rt  cuda_table_with_nullsrI  rf  r  r   r  r[  r   r  r    test_table_non_cpu  s  

&"










$r~  )collectionsr   collections.abcr   r2   r7   numpyrR   ImportErrorr)   pyarrowr   pyarrow.computecomputer  pyarrow.interchanger   pyarrow.vendored.versionr   r<   rG   rO   rQ   markrZ   ra   rd   rk   ru   rz   r   parametrizerP   rB   r`   r&   r  r|  r   r   r   r   r   r   r   r   r   nopandasr   r   r   r   r   r   r   r   r  r  r
  r  r  r%  r+  r2  r6  r=  r?  rA  rC  rJ  rN  rR  r_  rb  rg  rk  rl  rm  ro  rr  ru  rz  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  rq   r   r  r  r  r  r  r  r  r  r  r   r  r
  r  r  r  r  r  r  r  r!  r#  r*  r5  r;  r=  r@  rG  rP  rR  rT  r^  r`  ra  rb  rf  rl  ro  rq  r-   rI   r%   rz  r|  r}  r~  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  aceror  r  r  r  r  r  r  r  r  datasetr  r  r  r  r
  r  r  r  r  rs   r  r  r"  r+  fixturer/  r   r0  r5  r:  r=  r>  r@  rA  rD  rE  rF  rZ  r\  rw  rx  r~  r   r   r   r    <module>   s  "-	


0$





(

'	
37

3


<
%




"
F







%






	 
#
9 
 

U
 
*+	3 &   !