o
    hC                     @   s  d Z ddlZddlZddlZddlZzddlZW n ey%   ej	jZ
Y nw edZe ZedkZej	je de dZdZdZdd Zd	d
 Zdd Zej	dg ddd Zdd ZdYddZej	dg ddd Zej	dg ddd Zdd Zej	dg ddd Zej	dg ddd Zej	dg dd d! Z ej	dg dd"d# Z!ej	d$d%d&gej	dg dd'd( Z"ej	dg dd)d* Z#d+d, Z$d-d. Z%d/d0 Z&d1d2 Z'd3d4 Z(d5d6 Z)d7d8 Z*d9d: Z+d;d< Z,d=d> Z-d?d@ Z.dAdB Z/eej	dg ddCdD Z0dEdF Z1dGdH Z2dIdJ Z3dKdL Z4ej	ddMdNgdOdP Z5dQdR Z6dSdT Z7dUdV Z8dWdX Z9dS )Zz
UNTESTED:
read_message
    Nzpyarrow.cudazlinux-x86_64z'CUDA IPC not supported in platform `%s`)reasonc                 C   s&   t d| _t t j d | _d S )Nr      )cudaContextglobal_contextget_num_devicesglobal_context1module r   K/var/www/vscode/kcb/lib/python3.10/site-packages/pyarrow/tests/test_cuda.pysetup_module1   s   r   c                 C   s   | ` d S N)r   r	   r   r   r   teardown_module6   s   r   c                  C   s   t j dks	J tjdksJ tjt j d ksJ tj} | jr$J dt| v s,J tj	}|| j	ks6J |jr;J |j
dksBJ |jtjjksKJ tjtdd t t j  W d    d S 1 sfw   Y  d S )Nr   r   z)<pyarrow.MemoryManager device: CudaDevicez5device_number argument must be non-negative less thanmatch)r   r   r   r   device_numberr   memory_manageris_cpureprdevice	device_iddevice_typepaDeviceAllocationTypeCUDApytestraises
ValueError)mmdevr   r   r   test_Context:   s    

"r!   size)r   r     c                 C   sj   t | }tj|tjd}d|| d d|  d < | }tj|tjd}tj|| |j| ks3J d S )Ndtyper         )	r   new_host_buffernp
frombufferuint8copytestingassert_equalr"   )r"   bufarrarr_cparr2r   r   r   test_manage_allocate_free_hostP   s   
r3   c                  C   s6   t j} t d}t j| d ksJ ~t j| ksJ d S )N   )r   bytes_allocated
new_buffer)r5   cudabufr   r   r   test_context_allocate_del[   s
   
r8   hostc                 C   s   |dkrS| dks
J t | }|j| ksJ tj|tjd}|j| ks%J tjjdd| tjd|dd< | dks@| dks@J tj|tjd}tj	
|| ||fS |dkrwt| dd	\}}t| }|j| kskJ |j|d| d
 ||fS td)z5Return a host or device buffer with random data.
    r9   r   r$   r      )lowhighr"   r%   Nr   targetpositionnbyteszinvalid target value)r   allocate_bufferr"   r)   r*   r+   randomrandintsumr-   r.   make_random_bufferr   r6   copy_from_hostr   )r"   r>   r/   r0   arr_dbufr   r   r   rF   c   s$   

rF   c           
      C   s  t | \}}t|}|j| ksJ tj| tjd}tj	|| t
t t| W d    n1 s7w   Y  t|}|j| ksHJ tj| tjd}tj	|| t| }|j| ksgJ tj| tjd}tj	|| |d|j}|j| ksJ tj| tjd}tj	|| | dkr||| d d   tj| tjd}tj	t|| d d  || d d  f| ||d | d   t|}|j| ksJ tj| tjd}tj	|| ||| d d   tj| tjd}tj	|| |d|jd }|j| ksJ tj| tjd}tj	|| || d | d }|j| | d  ksDJ tj| tjd}tj	|| d d  | | d }d|  d }tj|||d}|j|ksuJ tj| tjd}tj	||||  | t|j||d}|j|ksJ tj| tjd}tj	||||  | tj|||d}|j|ksJ tj| tjd}tj	||||  | t||||  }|j|ksJ tj| tjd}tj	||||  | tj| ||d}|j|ksJ tj| tjd}tj	||||  | t| }|j| ks<J t|}|||}|j|ksOJ tj| tjd}tj	||||  | t| }tj|tjd}	||	d d < t|}|j| ksJ tj| tjd}tj	|| tj|||d}|j|ksJ tj| tjd}tj	||||  | t|j||d}|j|ksJ tj| tjd}tj	||||  | d S )	Nr$   r   r      
   r&   )offsetr"   rL   length)rF   r   buffer_from_datar"   r)   r*   copy_to_hostr+   r-   r.   r   r   BufferError
memoryviewtobytesslicerG   concatenater6   r   r(   )
r"   r0   r/   r7   r2   cudabuf2arr3soffsetssizerH   r   r   r   test_context_device_bufferz   s   


*






rZ   c                 C   s@  t }t| dd\}}|j}t| |jj }|tj||dd d < ||}|j	|j	ks.J tj|
 |d}tj|| ||}|j	|j	ksKJ tj|
 |d}tj|| tjtjdd |td W d    n1 svw   Y  tjtjdd |tg d W d    d S 1 sw   Y  d S )	Nr   r=   r$   z$buffer is not backed by a CudaBufferr   s   123z8cannot create device buffer view from .* 'numpy.ndarray')r   rJ   r'   )r   rF   r%   r   r(   itemsizer)   r*   buffer_from_objectr"   rP   r-   r.   r   r   r   ArrowTypeError	py_bufferarray)r"   ctxr0   cbufr%   hbufcbuf2r2   r   r   r   test_context_from_object   s.   

"rd   c                  C   s   t } ttj}d}t||j }t|}| 	|j
|j|}t||d ks+J ~t||ks5J | 	|j
|j|}~|  t||j }| 	|j
|j}~tjtjdd |  W d    d S 1 slw   Y  d S )NrK   r   zCuda error r   )r   r)   r%   r+   r   r(   r[   sysgetrefcountforeign_bufferaddressr"   rP   r   r   r   ArrowIOError)r`   r%   r"   rb   rcfbufr   r   r   test_foreign_buffer  s(   

"rl   c                 C   s(  t | \}}| | ksJ t|}|j| ksJ |jr!J | | ks+J | dkr6|jdks6J t| D ]}|| || ksFJ q:t	d t	| d | d fD ]}||  ||  ksdJ qT|	| d | d }|j
|ksvJ tjtdd t  W d    d S 1 sw   Y  d S )Nr   r&   rJ   z-Do not call CudaBuffer's constructor directlyr   )rF   rS   
to_pybytesr   rO   r"   r   rh   rangerT   parentr   r   	TypeErrorr   
CudaBuffer)r"   r0   r/   ra   issbufr   r   r   test_CudaBuffer5  s*   


"ru   c                 C   s,  t | \}}| | ksJ t| }|tj|tjdd d < |j| ks(J |j	s-J | | ks7J t
| D ]}|| || ksGJ q;td t| d | d fD ]}||  ||  kseJ qU|| d | d }|j|kswJ ~tjtdd t  W d    d S 1 sw   Y  d S )Nr$   r&   rJ   z-Do not call HostBuffer's constructor directlyr   )rF   rS   rm   r   r(   r)   r*   r+   r"   r   rn   rT   ro   r   r   rp   
HostBuffer)r"   r0   r/   rb   rr   rs   rt   r   r   r   test_HostBufferQ  s*   


"rw   c           	      C   s
  t d}| |j }tj|dd}t|tjsJ t|tjr!J t j	||d}|j
| ks/J t| |d d < t j	||d}t j|| t|}t|tjsRJ t|tjsZJ |j
|ksaJ |jrfJ |j|d|d |jd|d}t j	||d}t j|| d S )Nuint16T)	resizabler$   r   r?   )r)   r%   r[   r   rB   
isinstanceBufferr   rq   r*   r"   rn   r-   r.   r   r6   r   rG   rP   )	r"   dtrA   r/   r0   rH   device_bufferbuf2r2   r   r   r   test_copy_from_to_hostl  s&   



r   c              	   C   sF  t | dd\}}| }|jsJ tj|tj|tjd |j| d d}|js+J tj|| d d  tj|tjd |j| d | d d}|jsNJ tj|| d | d | d   tj|tjd |j| d dd}|jsuJ |jdks|J | d	 d
fd| d dfdfD ]"\}}t	j
tdd |j||d W d    n1 sw   Y  qd| d f| d	 | d d	 d f| dffD ]"\}}t	j
tdd |j||d W d    n1 sw   Y  qt| d }|j|d tj|d | d  tj|tjd | dk rd S |j|dd tj|dd| d   tj|tjd |j|dd tj|d d tj|tjdd d  |j|ddd tj|dd tj|tjdd d  d| d fd| d fd| d	 f| d | d d ffD ]%\}}t	j
tdd |j|||d W d    n	1 sw   Y  q{d S )Nr   r=   r$   r&   r@      r?   r   rJ   r   r   r   !position argument is out-of-ranger   8requested more to copy than available from device buffer)r/      )r/   r@   )r/   rA      )r/   rA   r@      rK      z,requested copy does not fit into host buffer)r/   r@   rA   )rF   rP   r   r)   r-   r.   r*   r+   r"   r   r   r   r   rB   )r"   r0   rI   r/   r@   rA   r   r   r   test_copy_to_host  sp   

&
 
&&
*r   dest_ctxsameanotherc              	      s   t |dd\} | }| dkrt}  jj| jkrtd n j} | | fdd}| |ks5J |dkr||d d|d |d  |d | d   ksSJ | |ksZJ |d	|d
 d|d d	 |d |d
   |||d
  d	  d   ksJ |d
 dfd|d	 dfdfD ]!\}}tjt	dd |||d W d    n1 sw   Y  qd|d	 ffD ]!\}}tjt	dd |||d W d    n1 sw   Y  q|dk rd S |d
 |d	 d
 d	 ffD ]"\}}tjt	dd |||d W d    n	1 sw   Y  qd S )Nr   r"   r>   r   znot a multi-GPU systemc                     4   j  g| R i |  }tj|tjd S Nr$   )copy_from_devicerP   r)   r*   r+   tolistargskwargsrbufr/   rI   r   r   put     z"test_copy_from_device.<locals>.putr&   r   r   rJ   r?   r   r   r   r   r   r   r   6requested more to copy than available in device buffer)
rF   r   r   contextr   r   skipr6   r   r   )r   r"   r0   lstr   r@   rA   r   r   r   test_copy_from_device  sX   

44
r   c              	      s  t | dd\} | }t|  fdd}| |ksJ | dkri|| d d|d | d  |d |  d   ks=J | |ksDJ |d| d d	|d d |d | d   || | d  d  d   ksiJ | d d
fd| d dfdfD ]!\}}tjtdd |||d	 W d    n1 sw   Y  qwd| d ffD ]!\}}tjtdd |||d	 W d    n1 sw   Y  q| dk rd S | d | d d d ffD ]!\}}tjtdd |||d	 W d    n1 sw   Y  qd S )Nr9   r   c                     r   r   )rG   rP   r)   r*   r+   r   r   r   r   r   r     r   z test_copy_from_host.<locals>.putr&   r   r   rJ   r?   r   r   r   r   r   r   z6requested more to copy than available from host bufferr   )rF   r   r   r6   r   r   r   )r"   r0   r   r   r@   rA   r   r   r   test_copy_from_host  sL   
44
r   c                  C   s   t d} | jtjjksJ t| jtjsJ t| j	tj
s J | js%J | jjs+J | jt jks5J | j	js;J tddd\}} | jtjjksLJ t| jtjsUJ | jtj	jks^J t| j	tj
sgJ | jrlJ | jjrrJ | j	jrxJ d S )NrK   r   r   )r   r(   r   r   r   	CUDA_HOSTrz   r   Devicer   MemoryManagerr   default_cpu_memory_managerrF   r   r   )r/   _r   r   r   test_buffer_device'  s    


r   c                     s  dd  d fdd	} d\}}| || | |||d   d\}}| tjdtjd	 |d
tjdtjd	 | tjdtjd	 |  t| tj}tj	|d d
 tjd
tjd	 tj	|d
d tjdtjd	 tj	|dd  tjdtjd	 d S )Nc                 S   s   t | }t|}||fS r   )r   r6   r   BufferWriter)r"   ra   writerr   r   r   allocate>  s   

z#test_BufferWriter.<locals>.allocater   c                    s   | \}}t | dd\}}|dkr||_| }|dksJ ||j|d | |ks0J |d | }|dks?J || k r\t|| | }||j||d ||7 }|| k sC|  |j| ksgJ |j	
  | }	|j	
  |	j| ks|J tj|	tjd}
tj||
 d S )Nr9   r   r   rN   rM   r$   )rF   buffer_sizetellwriterT   seekminflushr"   r   synchronizerP   r)   r*   r+   r-   r.   )
total_size	chunksizer   ra   r   r0   r/   r@   bytes_to_writer~   r2   r   r   r   test_writesC  s0   


z&test_BufferWriter.<locals>.test_writes)i   r#      d   r$   2      K   )r   )
r   r)   aranger+   writeatr   r*   rP   r-   r.   )r   r   
chunk_sizera   r   r0   r   r   r   test_BufferWriter=  s   
""&r   c                  C   sL  d} t | }t|}t| dd\}}|jdksJ d|_|jdks%J ||jdd | dks6J ||jdd d|_|jdksIJ |j	dksPJ ||jdd	d
 |j	dksaJ ||jddd
 |j	dksrJ ||jddd
 ||jddd
 |
  | }|j| ksJ tj|tjd}tj|| d S )Nr#   r9   r   r   r   r   rK      i,  rM   i6  i  i  i  r$   )r   r6   r   r   rF   r   r   rT   r   num_bytes_bufferedr   rP   r"   r)   r*   r+   r-   r.   )r"   ra   r   r0   r/   r~   r2   r   r   r   test_BufferWriter_edge_casesp  s0   

r   c                  C   s  d} t | dd\}}t|}|d | dksJ |d}t|dks)J | dks1J |d tjdtj	d}|
|}|d	ksIJ | dksQJ tj|dd  |d d	  |d
 | d
ksmJ | }tj| tj	d}tj|| d S )Nr#   r   r   i  r   r   i  r$   r   r   )rF   r   BufferReaderr   r   readlenr)   zerosr+   readintor-   r.   read_bufferr*   rP   )r"   r0   ra   readerdatar2   nr~   r   r   r   test_BufferReader  s&   





r   c                  C   sz   t ddd\} }t|}|d | }t|dksJ | dks&J | }tj	|
 tjd}tj| | d S )Nr   r   r   r$   )rF   r   r   r   r   r   r   r   r)   r*   rP   r+   r-   r.   )r0   ra   r   r   r~   r2   r   r   r   test_BufferReader_zero_size  s   

r   c                 C   st   t t dt  t dt  g}t tjjdd| tjd}t tjjdd| tjd}t j||g|d}|S )Nf0f1r   r:   )r"   r%   schema)	r   r   fieldint16r_   r)   rC   rD   record_batch)rN   r   a0a1batchr   r   r   make_recordbatch  s   r   c                  C   s   t d} |  }t| t}t|| j}t|tj	sJ | j|jks%J | j
|j
ks-J | j|jks5J | }||s@J tj|| j}|| sQJ | j
|j
ksYJ | j|jksaJ | d|dsnJ | |suJ d S )NrK   r   )r   	serializer   serialize_record_batchr   read_record_batchr   rz   r   RecordBatchnum_columnsnum_rowsrP   equalsipccolumn)r   rb   ra   cbatchr/   batch2r   r   r   test_batch_serialize  s    r   c            
      C   s   t jg dt  d} t jddgdgg d gt t  d}t jg dt dt  fdt  fgd}t jj	g d	t d
dgd}t jj	g d|d}t jj	g d|d}| |||||g}t 
dd t|D }t j||d}t j|g}	|	S )N)r   r   *   Ntyper   r   rJ   ))abT)cdeF)NNNstrsbools)r   r   r   Nfoobar)indices
dictionary)rJ   r   rJ   N)rJ   r   r   Nc                 S   s    g | ]\}}d  ||jfqS )zf{})formatr   ).0rr   r0   r   r   r   
<listcomp>  s    zmake_table.<locals>.<listcomp>r   )r   r_   r   list_int32structutf8bool_DictionaryArrayfrom_arraysr   	enumerater   Tablefrom_batches)
r   r   a2a3a4a5arraysr   r   tabler   r   r   
make_table  s4   &


r  c                  C   s   t  } t }tj|| j}||  W d    n1 sw   Y  t| 	 }t
t|}|j|t|d tjt| }|| ||fS )N)rA   )r  r   BufferOutputStreamr   
new_streamr   write_tabler^   getvaluerm   r   r6   r   rG   open_streamr   r   read_all)htablesinkoutrb   rI   dtabler   r   r   make_table_cuda  s   r  c                  C   sr   t  \} }}}|j|jksJ |j|jksJ |j|jksJ | | s(J |tj| 	 s7J d S r   )
r  r   r   r   r   rP   r   r   r
  r  )rb   r  rI   r  r   r   r   test_table_deserialize  s   
r  c                  C   s   t  \} }}}tj|j|j}|j|jksJ |j|jks J |j|jks(J | 	|
 s1J |	tj|
  s@J d S r   )r  r   r   r   columnscolumn_namesr   r   r   r   rP   r   r
  r  )rb   r  rI   r  dtable2r   r   r   %test_create_table_with_device_buffers  s   
r  c                 C   sx   t jd}t jj| }||}|j  | }|j	|j	ks+J t
|j	|j	ftj||jd}tj|| d S )Nr   r$   )r   r   r   IpcMemHandlefrom_bufferopen_ipc_bufferr   r   rP   r"   r   r)   r*   r%   r-   r.   )handle_bufferexpected_arrother_context
ipc_handleipc_bufr/   r0   r   r   r   other_process_for_test_IPC'  s   

 r  c                 C   sf   dd l }|d}t| dd\}}| }| }|jt||fd}|  |  |j	dks1J d S )Nr   spawnr   r   )r>   r   )
multiprocessingget_contextrF   export_for_ipcr   Processr  startjoinexitcode)r"   r   r`   r0   ra   r  r  pr   r   r   test_IPC2  s   
r(  c                  C   s  t ddd\} }|j}||jfD ]v}tg d}||}| d jr'J | d jtj	j
ks4J | d j|jks@J |t }||sNJ td|i}||}|d  d }	|	jrgJ |	jtj	j
kspJ |	j|jksxJ |t }
|
|sJ qd S )NrK   r   r   )r   r   rJ   r   col)rF   r   r   r   r_   copy_tobuffersr   r   r   r   r   r   r   )r   r/   mm_cudadestr0   arr_cudaarr_roundtripr   
batch_cudabuf_cudabatch_roundtripr   r   r   test_copy_toA  s&   


r3  c                  C   s  t d} | j}|d}t|d|}|d}t|d|}tt }tj	dgddgg|d}t
d	d
d\}}	|	j}
||
}|| |jdksQJ |jtjksYJ |j	jdksaJ ~tj||}|jtt ksuJ |t }||sJ ~t jtdd tj|| W d    n1 sw   Y  ||
}||| ~tj||}|jtt ksJ |t }||sJ ~t jtdd tj|| W d    d S 1 sw   Y  d S )Npyarrow.cffistruct ArrowSchema*	uintptr_tstruct ArrowDeviceArray*r   rJ   r   r   rK   r   r   !Cannot import released ArrowArrayr   "Cannot import released ArrowSchema)r   importorskipffinewintcastr   r   r   r_   rF   r   r*  _export_to_c_devicer   r   r   r   rN   Array_import_from_c_devicer   r   r   r   r   )cffir;  c_schema
ptr_schemac_array	ptr_arraytypr0   r   r/   r,  carrcarr_newarr_newr   r   r   test_device_interface_arrayZ  sF   





"rK  c                  C   s  t d} | j}|d}t|d|}|d}t|d|}td}|j}t	|t
}t||}	|	| |jdksAJ |jt
jksIJ |jjdksQJ ~	tj||}
|
j|ks`J |
t }||snJ ~
t jtdd tj|| W d    n1 sw   Y  t||}	|	|| ~	tj||}
|
j|ksJ |
t }||sJ ~
t jtd	d tj|| W d    n1 sw   Y  t | t jtd
d tj|| W d    d S 1 sw   Y  d S )Nr4  r5  r6  r7  rK   rJ   r8  r   r9  z%ArrowSchema describes non-struct type)r   r:  r;  r<  r=  r>  r   r   r   r   r   r   r?  r   r   r   r_   rN   r   r   rA  r*  r   r   r   r   r   _export_to_c)rB  r;  rC  rD  rE  rF  r   r   ra   r   
cbatch_new	batch_newr   r   r   !test_device_interface_batch_array  sP   



"rO  c                  C   s   t d} t| t}t|| j}| d }|d }t|t|ks#J t d} t| t}t|| j}| d }|d }t|t|ksFJ d S )NrK   r   r   r   r   r   r   r   r   str)r   ra   r   r0   rH  r   r   r   test_print_array  s   rR  rK   r   c                 C   sr   t | d }tj|tjd}t| |d d < tt| t }tj	t | d |g}t
|t
|ks7J d S )Nr   r$   )r   r(   r)   r*   int64rn   r   r_   r@  from_buffersrQ  )r"   r/   np_arrr0   rH  r   r   r   test_print_array_host  s   rV  c                 C   sh   g }g }t |D ]!}t| }t|t}t||j}||d  ||d  qt	|t	|fS )Nr   )
rn   r   r   r   r   r   r   appendr   chunked_array)n_elements_per_chunkn_chunksarrscarrsr   r   ra   r   r   r   r   make_chunked_array  s   r]  c                  C   sH   t dd\} }t|t| ksJ t dd\} }t|t| ks"J d S )NrK   r'   r      )r]  rQ  )r0   rH  r   r   r   test_print_chunked_array  s   r_  c                  C   sp   t d} t| t}t|| j}t|t| ksJ t d} t| t}t|| j}t|t| ks6J d S )NrK   r   rP  )r   ra   r   r   r   r   test_print_record_batch  s   r`  c                  C   s&   t  \} }} }t|t|ksJ d S r   )r  rQ  )r   r  ctabler   r   r   test_print_table  s   rb  )r9   ):__doc__re   	sysconfigr   pyarrowr   numpyr)   ImportErrormark
pytestmarkr:  r   get_platformplatformhas_ipc_supportskipifcuda_ipcr   r   r   r   r!   parametrizer3   r8   rF   rZ   rd   rl   ru   rw   r   r   r   r   r   r   r   r   r   r   r   r  r  r  r  r  r(  r3  rK  rO  rR  rV  r]  r_  r`  rb  r   r   r   r   <module>   s   









?1
+3$	58

