o
    Ûñhú¦  ã                   @   sê  d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	zd dl
ZW n ey7   dZY nw d dlZd dlmZmZ zd dlmZ d dlZW n	 eyY   Y nw G dd„ dƒZG dd„ deƒZG d	d
„ d
eƒZG dd„ deƒZejdd„ ƒZejdd„ ƒZejdd„ ƒZejejdddejdddgddd„ ƒZdd„ Z dd„ Z!dd„ Z"ej# $dd d!„ d"d!„ g¡d#d$„ ƒZ%d%d&„ Z&ej#jd'd(„ ƒZ'd)d*„ Z(d+d,„ Z)ej#jej#j*d-d.„ ƒƒZ+ej#jd/d0„ ƒZ,d1d2„ Z-ej# $d3ej. /¡ ej.j/d4d5g¡d6d7„ ƒZ0d8d9„ Z1ej# $d3ej. /¡ ej.j/d4d5g¡d:d;„ ƒZ2d<d=„ Z3ej#jd>d?„ ƒZ4ej#jd@dA„ ƒZ5ej# $dBd4dCg¡dDdE„ ƒZ6ej#j7dFdG„ ƒZ8dHdI„ Z9dJdK„ Z:ej# $d3ej. ;¡ ej.j;dCdLej.j;dCdMej.j;ej.j<j=dNej.j;dCej.j<j=dOg¡dPdQ„ ƒZ>dRdS„ Z?dTdU„ Z@dVdW„ ZAdXdY„ ZBdZd[„ ZCej#jd\d]„ ƒZDejd^d_„ ƒZEd`da„ ZFdbdc„ ZGddde„ ZHej#jIdfdg„ ƒZJdhdi„ ZKdjdk„ ZLdldm„ ZMG dndo„ doejNƒZOG dpdq„ dqeƒZPejdrds„ ƒZQej#jRdtdu„ ƒZSej#jRdvdw„ ƒZTej#jdxdy„ ƒZUej#jdzd{„ ƒZVej#jej# $d|d}d~g¡dd€„ ƒƒZWdd‚„ ZXej#jdƒd„„ ƒZYej#jd¥d…d†„ƒZZej#jd‡dˆ„ ƒZ[ej#jd‰dŠ„ ƒZ\ej#jd‹dŒ„ ƒZ]ej#jddŽ„ ƒZ^ej#jdd„ ƒZ_ej#jd‘d’„ ƒZ`ej#jd“d”„ ƒZad•d–„ Zbd—d˜„ Zcd™dš„ Zdd›dœ„ Zeddž„ ZfdŸd „ Zgd¡d¢„ Zhd£d¤„ ZidS )¦é    )ÚUserListN)Úchanged_environÚinvoke_script)Úassert_frame_equalc                   @   s:   e Zd ZdZdd„ fdd„Zdd„ Zdd	„ Zddd„ZdS )Ú
IpcFixtureNc                   C   ó   t  ¡ S ©N©ÚioÚBytesIO© r   r   úJ/var/www/vscode/kcb/lib/python3.10/site-packages/pyarrow/tests/test_ipc.pyÚ<lambda>/   ó    zIpcFixture.<lambda>c                 C   s   || _ |  ¡ | _d S r   )Ú_sink_factoryÚget_sinkÚsink)ÚselfÚsink_factoryr   r   r   Ú__init__/   s   zIpcFixture.__init__c                 C   s   |   ¡ S r   )r   ©r   r   r   r   r   3   s   zIpcFixture.get_sinkc                 C   s
   | j  ¡ S r   )r   Úgetvaluer   r   r   r   Ú
get_source6   ó   
zIpcFixture.get_sourceé   Fc           
      C   s´   d}t  dt  ¡ fdt  ¡ fg¡}|  | j|¡}g }t|ƒD ]}t jdd„ t|ƒD ƒg d¢g|d}| |¡ q|rFt j	 
|¡}	| |	¡ n
|D ]}| |¡ qH|j| _| ¡  |S )Nr   ÚoneÚtwoc                 S   s   g | ]}t   ¡ ‘qS r   )Úrandom©Ú.0Ú_r   r   r   Ú
<listcomp>B   ó    z,IpcFixture.write_batches.<locals>.<listcomp>)ÚfooNÚbarÚbazbazÚqux©Úschema)Úpar(   Úfloat64Úutf8Ú_get_writerr   ÚrangeÚrecord_batchÚappendÚTableÚfrom_batchesÚwrite_tableÚwrite_batchÚstatsÚwrite_statsÚclose)
r   Únum_batchesÚas_tableÚnrowsr(   ÚwriterÚbatchesÚiÚbatchÚtabler   r   r   Úwrite_batches9   s(   ÿýzIpcFixture.write_batches)r   F)Ú__name__Ú
__module__Ú__qualname__r5   r   r   r   r?   r   r   r   r   r   ,   s    r   c                   @   s&   e Zd ZdZdZdd„ Zddd„ZdS )	ÚFileFormatFixtureTNc                 C   s   t jj||| jdS )N©Úoptions)r)   ÚipcÚnew_filerE   ©r   r   r(   r   r   r   r,   X   ó   zFileFormatFixture._get_writerFc                 C   sÀ   | j |d}t |  ¡ ¡}tj |¡}|jt|ƒksJ ‚t|ƒD ]\}}| 	|¡}||  
|¡s2J ‚|j 
|d j¡s=J ‚q t|jtjjƒsHJ ‚t| jtjjƒsRJ ‚t|jƒt| jƒks^J ‚d S )N©r8   r   )r?   r)   ÚBufferReaderr   rF   Ú	open_fileÚnum_record_batchesÚlenÚ	enumerateÚ	get_batchÚequalsr(   Ú
isinstancer4   Ú	ReadStatsr5   Ú
WriteStatsÚtuple)r   r8   r;   Úfile_contentsÚreaderr<   r=   r   r   r   Ú_check_roundtrip[   s   
z"FileFormatFixture._check_roundtrip©F)r@   rA   rB   Úis_filerE   r,   rX   r   r   r   r   rC   S   s
    rC   c                   @   s    e Zd ZdZdZdZdd„ ZdS )ÚStreamFormatFixtureFNc                 C   s   t jj||| j| jdS )N)Úuse_legacy_formatrE   )r)   rF   Ú
new_streamÚuse_legacy_ipc_formatrE   rH   r   r   r   r,   v   s   üzStreamFormatFixture._get_writer)r@   rA   rB   r^   rE   rZ   r,   r   r   r   r   r[   n   s
    r[   c                   @   s   e Zd Zdd„ ZdS )ÚMessageFixturec                 C   ó   t  ||¡S r   ©r)   ÚRecordBatchStreamWriterrH   r   r   r   r,      ó   zMessageFixture._get_writerN)r@   rA   rB   r,   r   r   r   r   r_      s    r_   c                   C   ó   t ƒ S r   )r   r   r   r   r   Úipc_fixture…   ó   re   c                   C   rd   r   )rC   r   r   r   r   Úfile_fixtureŠ   rf   rg   c                   C   rd   r   )r[   r   r   r   r   Ústream_fixture   rf   rh   zFile Format)ÚidzStream Format)Úparamsc                 C   s   |   | j¡S r   )ÚgetfixturevalueÚparam)Úrequestr   r   r   Úformat_fixture”   s   rn   c                  C   sH   d} t  tj¡ tj t | ¡¡ W d   ƒ d S 1 sw   Y  d S ©Nó    )ÚpytestÚraisesr)   ÚArrowInvalidrF   rL   rK   ©Úbufr   r   r   Útest_empty_file¢   s   "ÿrv   c                 C   ó   | j dd d S )NFrJ   ©rX   ©rg   r   r   r   Útest_file_simple_roundtrip¨   ó   rz   c                 C   rw   )NTrJ   rx   ry   r   r   r   Útest_file_write_table¬   r{   r|   r   c                   C   r   r   r	   r   r   r   r   r   ±   r   r   c                   C   r   r   )r)   ÚBufferOutputStreamr   r   r   r   r   ²   r   c                 C   sP   t | ƒ}| ¡ }t | ¡ ¡}tj |¡}| ¡ }tj 	|¡}| 
|¡s&J ‚d S r   )rC   r?   r)   rK   r   rF   rL   Úread_allr0   r1   rQ   )r   Úfixturer;   rV   rW   ÚresultÚexpectedr   r   r   Útest_file_read_all°   s   r‚   c           	      C   s®   |   ¡  |  ¡ }tj |¡}tj t |¡¡}t |¡}| ¡ }| ¡ }| ¡ }| |¡s/J ‚| |¡s6J ‚|j	}|j
dks@J ‚|jdksGJ ‚|j	|ksNJ ‚|j	|ksUJ ‚d S ©Né   r   )r?   r   r)   rF   rL   rK   ÚRecordBatchFileReaderr~   rQ   r4   Únum_messagesrM   )	rg   ÚsourceÚreader1Úreader2Úreader3Úresult1Úresult2Úresult3Úst1r   r   r   Útest_open_file_from_bufferÁ   s   
r   c                 C   sT   dd„ |   ¡ D ƒ}t |  ¡ ¡}tj |¡}| ¡ }t |¡j	dd}t
||ƒ d S )Nc                 S   ó   g | ]}|  ¡ ‘qS r   ©Ú	to_pandas©r   r=   r   r   r   r!   Ú   r"   z)test_file_read_pandas.<locals>.<listcomp>T©Údrop)r?   r)   rK   r   rF   rL   Úread_pandasÚpdÚconcatÚreset_indexr   )rg   ÚframesrV   rW   r€   r   r   r   r   Útest_file_read_pandasØ   s   r›   c                 C   sŽ   |   ¡  |  ¡ }| d¡j}t|dƒ}| |¡ W d   ƒ n1 s#w   Y  tj t	 
|¡¡ ¡ }tj t |¡¡ ¡ }| |¡sEJ ‚d S )Nú
file.arrowÚwb)r?   r   ÚjoinÚstrpathÚopenÚwriter)   rF   rL   ÚpathlibÚPathr~   ÚOSFilerQ   )rg   Útmpdirr‡   ÚpathÚfÚt1Út2r   r   r   Útest_file_pathlibä   s   ÿrª   c                  C   sH   t  d¡} t tj¡ tj | ¡ W d   ƒ d S 1 sw   Y  d S ro   )r
   r   rq   rr   r)   rs   rF   Úopen_streamrt   r   r   r   Útest_empty_streamò   s   
"ÿr¬   c              	   C   s¸   t  ¡ }t  t  d|¡g¡}|  d¡j}t  |d¡3}t j ||¡}t j	dg|d}t  
|g|¡}| |¡ W d  ƒ n1 sAw   Y  W d  ƒ n1 sPw   Y  td|ƒ dS )zËARROW-15783: Verify to_pandas works for interval types.

    Interval types require static structures to be enabled. This test verifies
    that they are when no other library functions are invoked.
    Únumsrœ   r   ©é   é   é   ©ÚtypeNzread_record_batch.py)r)   Úmonth_day_nano_intervalr(   Úfieldrž   rŸ   r¤   rF   rG   Úarrayr.   r¡   r   )r¥   Úmdn_interval_typer(   r¦   r   r:   Úinterval_arrayr=   r   r   r   Ú"test_read_year_month_nano_intervalø   s   ý€ÿr¹   c              	   C   s¦   t  tj d¡t jdtjdddgddgdddœ¡}tj 	|¡}|  
| j|j¡}| |¡ W d   ƒ n1 s8w   Y  tj t |  ¡ ¡¡ ¡ }t| ¡ |ƒ d S )Nr   r#   r$   T©Ú
categoriesÚordered©r   r   )r—   Ú	DataFrameÚnpr   ÚrandnÚCategoricalÚnanr)   ÚRecordBatchÚfrom_pandasr,   r   r(   r3   rF   r«   rK   r   r~   r   r’   )rh   Údfr=   Úwrr>   r   r   r   Ú!test_stream_categorical_roundtrip  s   
þþÿÿrÇ   c           	      C   sÄ   |   ¡  |  ¡ }tj |¡}tj t |¡¡}t |¡}| ¡ }| ¡ }| ¡ }| |¡s/J ‚| |¡s6J ‚|j	}|j
dks@J ‚|jdksGJ ‚|j	|ksNJ ‚|j	|ksUJ ‚t|ƒt| jƒks`J ‚d S rƒ   )r?   r   r)   rF   r«   rK   ÚRecordBatchStreamReaderr~   rQ   r4   r†   rM   rU   r5   )	rh   r‡   rˆ   r‰   rŠ   r‹   rŒ   r   rŽ   r   r   r   Útest_open_stream_from_buffer  s    
rÉ   rE   F©Úuse_threadsc                 C   sd   |   ¡  |  ¡ }tjj||d}| ¡  |j}|jdksJ ‚|jdks%J ‚t	|ƒt	| j
ƒks0J ‚d S ©NrD   r„   r   )r?   r   r)   rF   r«   r~   r4   r†   rM   rU   r5   )rh   rE   r‡   rW   Ústr   r   r   Útest_open_stream_options6  s   rÎ   c                 C   óP   |   ¡  |  ¡ }t t¡ tjj|dd W d   ƒ d S 1 s!w   Y  d S ©NTrD   )r?   r   rq   rr   Ú	TypeErrorr)   rF   r«   )rh   r‡   r   r   r   Ú#test_open_stream_with_wrong_optionsH  ó
   "ÿrÒ   c                 C   sN   |   ¡  |  ¡ }tjj||d}| ¡  |j}|jdksJ ‚|jdks%J ‚d S rÌ   )	r?   r   r)   rF   rL   r~   r4   r†   rM   )rg   rE   r‡   rW   rÍ   r   r   r   Útest_open_file_optionsP  s   rÔ   c                 C   rÏ   rÐ   )r?   r   rq   rr   rÑ   r)   rF   rL   )rg   r‡   r   r   r   Ú!test_open_file_with_wrong_optionsa  rÓ   rÕ   c              	   C   sÒ   t  tj d¡t jdtjdddgddgdddœ¡}tjj	|dd}tj
j	|dd}|  | j|j¡}| |¡ | |¡ W d   ƒ n1 sGw   Y  tj t |  ¡ ¡¡ ¡ }t| ¡ t j||gdd	ƒ d S )
Nr   r#   r$   Trº   r½   F©Úpreserve_index©Úignore_index)r—   r¾   r¿   r   rÀ   rÁ   rÂ   r)   r0   rÄ   rÃ   r,   r   r(   r¡   rF   r«   rK   r   r~   r   r’   r˜   )rh   rÅ   r>   r=   rÆ   r   r   r   Útest_stream_write_dispatchi  s&   
þþ
þÿÿrÚ   c                 C   sú   t  dtj d¡i¡}tjj|d d… dd}tjj|dd}tj 	|||g¡}|  
| j|j¡}|j|dd W d   ƒ n1 sBw   Y  ttj |  ¡ ¡ƒ}ttt|ƒƒg d¢ks^J ‚tj 	|¡}t| ¡ t j|d d… ||d d… gd	d
ƒ d S )Nr   é   é
   FrÖ   é   )Úmax_chunksize)rÜ   rÝ   r   rÜ   TrØ   )r—   r¾   r¿   r   rÀ   r)   rÃ   rÄ   r0   r1   r,   r   r(   r2   ÚlistrF   r«   r   ÚmaprN   r   r’   r˜   )rh   rÅ   Úb1Úb2r>   rÆ   r;   Úresult_tabler   r   r   Útest_stream_write_table_batches~  s"   ÿÿÿÿrä   r^   Tc                 C   s¶   || _ |  ¡ }t |  ¡ ¡}tj |¡}|j |d j¡sJ ‚d}t	|ƒD ]\}}| || ¡s2J ‚|d7 }q%|t
|ƒks?J ‚t t¡ | ¡  W d   ƒ d S 1 sTw   Y  d S ©Nr   r¯   )r^   r?   r)   rK   r   rF   r«   r(   rQ   rO   rN   rq   rr   ÚStopIterationÚread_next_batch)rh   r^   r;   rV   rW   Útotalr<   Ú
next_batchr   r   r   Útest_stream_simple_roundtrip–  s   

"ÿrê   c                  C   s0  t  ¡ } dd„ tdƒD ƒ}tjj|gdgd}tjjdd}tjj| |j	|d}| 
|¡ W d   ƒ n1 s8w   Y  t|  ¡ ƒ}t  ¡ }tjdd	d
}tjj|d}tjj||j	|d}| 
|¡ W d   ƒ n1 sow   Y  t| ¡ ƒ}||ks€J ‚tj | ¡ ¡ }	tj |¡ ¡ }
|	|
ks–J ‚d S )Nc                 S   s   g | ]}t  d d¡‘qS )r   r±   )r   Úrandintr   r   r   r   r!   ­  s    z.test_compression_roundtrip.<locals>.<listcomp>i'  Úvalues©ÚnamesÚzstd©ÚcompressionrD   r   )Úcompression_level)r
   r   r-   r)   r0   Úfrom_arraysrF   ÚIpcWriteOptionsÚRecordBatchFileWriterr(   r2   rN   r   ÚCodecrL   r~   )r   rì   r>   rE   r:   Úlen1Úsink2ÚcodecÚlen2r¨   r©   r   r   r   Útest_compression_roundtripª  s2   ÿþÿþrû   c               	   C   sò  t j ¡ } | jdu sJ ‚| jdu sJ ‚| jt jjjksJ ‚d| _| jdu s'J ‚d| _| jdu s1J ‚t jjj| _| jt jjjksAJ ‚dD ]}t	 
ttf¡ || _W d   ƒ n1 sZw   Y  qC| jd u sgJ ‚dD ]}t j |¡r‡|| _| j|ks{J ‚| ¡ | _| j|ks‡J ‚qid | _| jd u s’J ‚t	 
t¡ d| _W d   ƒ n1 s¥w   Y  | jdu s±J ‚d| _| jdu s»J ‚t j d¡rõt jjt jjjddddd} | jt jjjksÙJ ‚| jdu sàJ ‚| jdu sçJ ‚| jdksîJ ‚| jdu s÷J ‚d S d S )NFT)ÚV5é*   )Úlz4rï   r   rþ   )Úmetadata_versionÚallow_64bitr\   rñ   rË   )r)   rF   rô   r   r\   rÿ   ÚMetadataVersionrü   ÚV4rq   rr   rÑ   Ú
ValueErrorrñ   rö   Úis_availableÚupperrË   )rE   Úvaluer   r   r   Útest_write_optionsÉ  s\   
ÿ€
€ÿûõr  c                 C   sN   t jtdd d| _tj ¡ | _|  ¡  W d   ƒ d S 1 s w   Y  d S )Nz4provide at most one of options and use_legacy_format©ÚmatchT)	rq   rr   r  r^   r)   rF   rô   rE   r?   )rh   r   r   r   Ú#test_write_options_legacy_exclusiveú  s   þ
"ûr
  )r   )r\   )rÿ   )r\   rÿ   c           	      C   sÜ   d | _ || _|  ¡ }t |  ¡ ¡}tj |  ¡ ¡}|j|jks!J ‚tj 	|¡}|j
 |d j
¡s2J ‚d}t|ƒD ]\}}| || ¡sEJ ‚|d7 }q8|t|ƒksRJ ‚t t¡ | ¡  W d   ƒ d S 1 sgw   Y  d S rå   )r^   rE   r?   r)   rK   r   rF   Úread_messagerÿ   r«   r(   rQ   rO   rN   rq   rr   ræ   rç   )	rh   rE   r;   rV   ÚmessagerW   rè   r<   ré   r   r   r   Útest_stream_options_roundtrip  s    	

"ÿr  c                  C   sì   t j ¡ } | jdu sJ ‚| jdu sJ ‚| jg ksJ ‚d| _| jdu s$J ‚d| _| jdu s.J ‚ddg| _| jddgks<J ‚t t¡ d | _W d   ƒ n1 sOw   Y  t jjdddgd} | jdu seJ ‚| jdu slJ ‚| jdgkstJ ‚d S )NTFr   r¯   )rË   Úensure_native_endianÚincluded_fields)	r)   rF   ÚIpcReadOptionsrË   r  r  rq   rr   rÑ   rD   r   r   r   Útest_read_options#  s(   

ÿþr  c                 C   s  t j ¡ }t jjdgd}t jjt  g d¢¡t  g d¢¡gddgd}|  | j|j¡}| 	|¡ W d   ƒ n1 s;w   Y  |  
¡ }t jj||d}t jj||t  ¡ d	}| ¡ }| ¡ }	|jd
ksfJ ‚|	jdksmJ ‚t jjt  g d¢¡gdgd}
|	|
ks‚J ‚||ksˆJ ‚d S )Nr¯   )r  ©r#   r$   Úbazr&   ©r¯   r°   r±   é   ÚaÚbrí   rD   )rE   Úmemory_poolr°   )r)   rF   r  r0   ró   r¶   r,   r   r(   r2   r   r«   Úsystem_memory_poolr~   Únum_columns)rh   Úoptions1Úoptions2r>   rÆ   r‡   rˆ   r‰   r‹   rŒ   r   r   r   r   Ú!test_read_options_included_fields>  s,   
ÿþÿ
ÿr  c                    sŒ  t  t  ¡ t  ¡ ¡‰g d¢g d¢ddgg d¢ddgg}‡fdd„|D ƒ}|d d	… }|d
 j‰d‡ ‡fdd„	}ˆ jrot t j¡ ||ƒ W d   ƒ n1 sOw   Y  t t j¡ ||ƒ W d   ƒ n1 siw   Y  n ||ƒ}|j	dkszJ ‚|j
d	ksJ ‚|jdksˆJ ‚|jd
ksJ ‚d ˆ _t jjddˆ _ˆ jr¸t t j¡ ||ƒ W d   ƒ n1 s²w   Y  n ||ƒ}|j	dksÃJ ‚|j
d	ksÊJ ‚|jdksÑJ ‚|jdksØJ ‚||ƒ}|j	d	ksãJ ‚|j
dksêJ ‚|jd
ksñJ ‚|jdksøJ ‚t jjddˆ _||dd}|j	dksJ ‚ˆ jr,|j
dksJ ‚|jd
ks"J ‚|jd
ks*J ‚d S |j
d	ks4J ‚|jdks<J ‚|jd
ksDJ ‚d S )N)r#   r#   N)r#   r$   r#   r#   r$   )r#   Nr$   Úquuxr  c                    s*   g | ]}t jjt j|ˆ d gdgd‘qS )r²   Údictsrí   )r)   rÃ   ró   r¶   )r   Úv)Útyr   r   r!   _  s    ÿÿz)test_dictionary_delta.<locals>.<listcomp>r  r   Fc                    sj   ˆ   t ¡ ˆ¡#}|rtj | ¡}| |¡ n
| D ]}| |¡ q|jW  d   ƒ S 1 s.w   Y  d S r   )r,   r)   ÚMockOutputStreamr0   r1   r2   r3   r4   )r;   r8   r:   r>   r=   )rn   r(   r   r   r?   e  s   
ÿ$øz,test_dictionary_delta.<locals>.write_batchesr   r±   T)Úemit_dictionary_deltasr¯   r°   )Úunify_dictionariesrJ   rY   )r)   Ú
dictionaryÚint8r+   r(   rZ   rq   rr   rs   rM   Únum_dictionary_batchesÚnum_replaced_dictionariesÚnum_dictionary_deltasr^   rF   rô   rE   )rn   Údatar;   Úbatches_delta_onlyr?   rÍ   r   )rn   r(   r!  r   Útest_dictionary_deltaW  sr   ü
þ

ÿ
ÿ€ÿ
ÿ€ÿr,  c               	   C   s\  t  t  dt  ¡ ¡g¡} t j t  ¡ | ¡}|jrJ ‚|jt jj	j
ks$J ‚t j t  ¡ | ¡}|jr2J ‚|jt jj	j
ks<J ‚tddƒ8 t j t  ¡ | ¡}|jsPJ ‚|jt jj	j
ksZJ ‚t j t  ¡ | ¡}|jshJ ‚|jt jj	j
ksrJ ‚W d   ƒ n1 s|w   Y  tddƒ8 t j t  ¡ | ¡}|jr•J ‚|jt jj	jksŸJ ‚t j t  ¡ | ¡}|jr­J ‚|jt jj	jks·J ‚W d   ƒ n1 sÁw   Y  tddƒX tddƒ9 t j t  ¡ | ¡}|jsàJ ‚|jt jj	jksêJ ‚t j t  ¡ | ¡}|jsøJ ‚|jt jj	jksJ ‚W d   ƒ n1 sw   Y  W d   ƒ d S W d   ƒ d S 1 s'w   Y  d S )Nr#   ÚARROW_PRE_0_15_IPC_FORMATÚ1ÚARROW_PRE_1_0_METADATA_VERSION)r)   r(   rµ   Úint32rF   r]   r}   Ú_use_legacy_formatÚ_metadata_versionr  rü   rG   r   r  )r(   r:   r   r   r   Ú!test_envvar_set_legacy_ipc_format¢  sF   



ú

ú

 úÿ$ÿr3  c                 C   sH   |   ¡ }t |  ¡ ¡}tj |¡}| ¡ }tj |¡}| 	|¡s"J ‚d S r   )
r?   r)   rK   r   rF   r«   r~   r0   r1   rQ   )rh   r;   rV   rW   r€   r   r   r   r   Útest_stream_read_allÆ  s   r4  c                 C   sN   dd„ |   ¡ D ƒ}|  ¡ }tj |¡}| ¡ }t |¡jdd}t	||ƒ d S )Nc                 S   r   r   r‘   r“   r   r   r   r!   Ò  r"   z+test_stream_read_pandas.<locals>.<listcomp>Tr”   )
r?   r   r)   rF   r«   r–   r—   r˜   r™   r   )rh   rš   rV   rW   r€   r   r   r   r   Útest_stream_read_pandasÐ  s   r5  c                 C   s2   |   ¡ }|  ¡ }t |¡}tj |¡}|t|ƒfS r   )r?   r   r)   rK   ÚMessageReaderr«   rß   )rh   r;   rV   Ú
buf_readerrW   r   r   r   Úexample_messagesÛ  s
   
r8  c                   C   sr   t  t¡ tt ¡ ƒ W d   ƒ n1 sw   Y  t  t¡ tt ¡ ƒ W d   ƒ d S 1 s2w   Y  d S r   )rq   rr   rÑ   Úreprr)   ÚMessager6  r   r   r   r   Útest_message_ctors_no_segfaultä  s   ÿ"ÿr;  c                 C   sÆ   | \}}t |ƒdksJ ‚|d jdksJ ‚t|d jtjƒs J ‚t|d jtjƒs+J ‚|d jtjj	ks6J ‚|dd … D ]$}|jdksEJ ‚t|jtjƒsNJ ‚t|jtjƒsWJ ‚|jtjj	ks`J ‚q<d S )Nr„   r   r(   r¯   zrecord batch)
rN   r³   rR   Úmetadatar)   ÚBufferÚbodyrÿ   r  rü   )r8  r    ÚmessagesÚmsgr   r   r   Útest_message_readerì  s   ürA  c           
      C   s  | \}}|d }|  ¡ }t | ¡ d ¡}tj |¡}tj |¡}tj | ¡ ¡}tj |¡}	| |¡s6J ‚| |¡s=J ‚| |¡sDJ ‚| |	¡sKJ ‚tjtj	dd tj t d¡¡ W d   ƒ n1 sgw   Y  t t
¡ tj |¡ W d   ƒ d S 1 sƒw   Y  d S )Nr   r°   zCorrupted messager  s   ab)Ú	serializer)   rK   Ú
to_pybytesrF   r  rQ   rq   rr   rs   ÚEOFError)
r8  r    r?  r@  ru   rW   ÚrestoredÚ	restored2Ú	restored3Ú	restored4r   r   r   Ú#test_message_serialize_read_messageü  s$   ÿ"ÿrI  c              	   C   s‚   | \}}|D ]8}t  ¡ }t j|dd}| |¡ W d   ƒ n1 s#w   Y  | ¡ }t j t j|dd¡}| |¡s>J ‚qd S )NÚgziprð   )	r)   r}   Úoutput_streamÚserialize_tor   rF   r  Úinput_streamrQ   )r8  r    r?  r  Úraw_outÚcompressed_outÚcompressed_bufr€   r   r   r   Ú!test_message_read_from_compressed  s   ÿÿ÷rQ  c                 C   s0   | \}}t j |d ¡}| |d j¡sJ ‚d S rå   )r)   rF   Úread_schemarQ   r(   )r8  r;   r?  r(   r   r   r   Útest_message_read_schema$  s   rS  c                 C   sF   | \}}t ||dd … ƒD ]\}}tj ||j¡}| |¡s J ‚qd S )Nr¯   )Úzipr)   rF   Úread_record_batchr(   rQ   )r8  r;   r?  r=   r  Ú
read_batchr   r   r   Útest_message_read_record_batch*  s
   þrW  c                  C   s°   t jt jdgt  ¡ dgdgd} t  ¡ }t j || j¡}| | ¡ W d   ƒ n1 s-w   Y  | 	¡ }t
jtdd t j || j¡ W d   ƒ d S 1 sQw   Y  d S )Nó   foor²   Ústrsrí   z type record batch but got schemar  )r)   r.   r¶   r+   r}   rF   r]   r(   r3   r   rq   rr   ÚIOErrorrU  )r=   Ústreamr:   ru   r   r   r   Ú.test_read_record_batch_on_stream_error_message2  s   ÿÿÿ"þr\  c                   @   s$   e Zd Zdd„ Zdd„ Zdd„ ZdS )ÚStreamReaderServerc                 C   sT   t   t jt j¡| _| j d¡ | j d¡ | j ¡ \}}|| _d | _g | _	d | _
|S )N)ú	127.0.0.1r   r¯   )ÚsocketÚAF_INETÚSOCK_STREAMÚ_sockÚbindÚlistenÚgetsocknameÚ_do_read_allÚ_schemaÚ_batchesÚ_table)r   Údo_read_allÚhostÚportr   r   r   ÚinitE  s   zStreamReaderServer.initc                 C   s¤   | j  ¡ \}}z@|jdd}tj |¡}|j| _| jr!| 	¡ | _
nt|ƒD ]\}}| j |¡ q%W | ¡  | j  ¡  d S W | ¡  | j  ¡  d S | ¡  | j  ¡  w )NÚrb©Úmode)rb  ÚacceptÚmakefiler)   rF   r«   r(   rg  rf  r~   ri  rO   rh  r/   r6   )r   Ú
connectionÚclient_addressr‡   rW   r<   r=   r   r   r   ÚrunP  s"   ýüÿzStreamReaderServer.runc                 C   s   | j | jr	| jfS | jfS r   )rg  rf  ri  rh  r   r   r   r   Ú
get_result_  s   ÿzStreamReaderServer.get_resultN)r@   rA   rB   rm  ru  rv  r   r   r   r   r]  C  s    r]  c                   @   s4   e Zd Zdd„ Zdd„ Zdd„ Zdd„ Zd	d
„ ZdS )ÚSocketStreamFixturec                 C   s   d S r   r   r   r   r   r   r   f  s   zSocketStreamFixture.__init__c                 C   sN   t ƒ | _| j |¡}| j ¡  t tjtj¡| _| j d|f¡ |  	¡ | _
d S )Nr^  )r]  Ú_serverrm  Ústartr_  r`  ra  rb  Úconnectr   r   )r   rj  rl  r   r   r   Ústart_serverk  s   
z SocketStreamFixture.start_serverc                 C   sD   dd l }| j | dd¡¡ | j ¡  | j ¡  | j ¡  | j 	¡ S )Nr   ÚQ)
Ústructr   r¡   ÚpackÚflushrb  r6   rx  rž   rv  )r   r}  r   r   r   Ústop_and_get_results  s   



z'SocketStreamFixture.stop_and_get_resultc                 C   s   | j jddS )Nr   ro  )rb  rr  r   r   r   r   r   {  s   zSocketStreamFixture.get_sinkc                 C   r`   r   ra   rH   r   r   r   r,   ~  rc   zSocketStreamFixture._get_writerN)r@   rA   rB   r   r{  r€  r   r,   r   r   r   r   rw  d  s    rw  c                   C   rd   r   )rw  r   r   r   r   Úsocket_fixture‚  rf   r  c                 C   sp   | j dd |  ¡ }|  ¡ \}}| |d j¡sJ ‚t|ƒt|ƒks$J ‚t|ƒD ]\}}||  |¡s5J ‚q(d S )NF©rj  r   )r{  r?   r€  rQ   r(   rN   rO   )r  Úwriter_batchesÚreader_schemaÚreader_batchesr<   r=   r   r   r   Útest_socket_simple_roundtrip‡  s   ÿr†  c                 C   s>   | j dd |  ¡ }|  ¡ \}}tj |¡}| |¡sJ ‚d S )NTr‚  )r{  r?   r€  r)   r0   r1   rQ   )r  rƒ  r    r€   r   r   r   r   Útest_socket_read_all“  s
   r‡  c                  C   s`   t  ddgi¡} tj | ¡}t ¡ }t||ƒ | ¡ }tj 	|dd … ¡}| 
¡ }t| |ƒ d S )Nr#   ç      ø?é   )r—   r¾   r)   rÃ   rÄ   r}   Ú
write_filer   rF   r«   r–   r   )rÅ   r=   r   ÚbufferrW   Úrdfr   r   r   Útest_ipc_file_stream_has_eos   s   
r  c                  C   sl   t  ddgi¡} tj | ¡}t ¡ }t||ƒ | ¡ }t |¡}t	|ƒ}|d  
¡ }t  |¡}t| |ƒ d S )Nr#   rˆ  r   )r—   r¾   r)   rÃ   rÄ   r}   rŠ  r   rK   Ú	read_filer’   r   )rÅ   r=   r   r‹  rW   r;   r*  rŒ  r   r   r   Útest_ipc_zero_copy_numpy²  s   


r  Úipc_typer[  Úfilec           
         s¢  t  ddgi¡}tj |¡}t ¡ }d}tjjtjjdœ|  }|||j	ƒ}t
|ƒD ]}|j|dt|ƒid q*| |¡ W d   ƒ n1 sGw   Y  | ¡ }| dkrqtj |¡‰ tˆ  ¡ ƒ}	W d   ƒ n1 skw   Y  n"tj |¡‰ ‡ fdd	„t
ˆ jƒD ƒ}	W d   ƒ n1 sŽw   Y  t
|ƒD ]$}|	| jjd
ks£J ‚t|	| jtjƒs®J ‚|	| jdt|ƒiks»J ‚q—|	| jjd
ksÆJ ‚|	| jd u sÏJ ‚d S )Nr#   rˆ  r°   )r[  r‘  Úbatch_id)Úcustom_metadatar[  c                    ó   g | ]}ˆ   |¡‘qS r   )Úget_batch_with_custom_metadata©r   r<   ©rW   r   r   r!   Ü  s    ÿz?test_batches_with_custom_metadata_roundtrip.<locals>.<listcomp>r¯   )r—   r¾   r)   rÃ   rÄ   r}   rF   r]   rG   r(   r-   r3   Ústrr   r«   rß   Ú!iter_batches_with_custom_metadatarL   rM   r=   Únum_rowsrR   r“  ÚKeyValueMetadata)
r  rÅ   r=   r   Úbatch_countÚfile_factoryr:   r<   r‹  Úbatch_with_metasr   r—  r   Ú+test_batches_with_custom_metadata_roundtripÃ  sB   ÿÿüÿ€
ÿÿÿrŸ  c                  C   sÆ   t jjt  g d¢¡t  g d¢¡gddgd} t  ¡ }t j || j¡ W d   ƒ n1 s-w   Y  | ¡ }t j 	|¡}| 
¡ }W d   ƒ n1 sKw   Y  |j | j¡sYJ ‚t|ƒdksaJ ‚d S )Nr  r  r  r  rí   r   )r)   r0   ró   r¶   r}   rF   r]   r(   r   r«   r~   rQ   rN   )r>   r   r‡   rW   r€   r   r   r   Útest_ipc_stream_no_batchesê  s   ÿþÿ
ÿr   c                  C   sF   d} d}t  dtj | ¡i¡}tj |¡}tj 	|¡| | ks!J ‚d S )NrÜ   r‰  r#   )
r—   r¾   r¿   r   rÀ   r)   rÃ   rÄ   rF   Úget_record_batch_size)ÚNÚitemsizerÅ   r=   r   r   r   Útest_get_record_batch_sizeü  s
   r¤  c                 C   s2   t j| |rdndd}t j||d}t|| ƒ d S )Nr°   r¯   )ÚnthreadsrÊ   )r)   Úserialize_pandasÚdeserialize_pandasr   )rÅ   rË   ru   r€   r   r   r   Ú"_check_serialize_pandas_round_trip  s   r¨  c                  C   sD   t jg d¢dd} ddg}t jg d¢tdƒdœ| |d	}t|ƒ d S )
Nr®   Úmy_index©Únamer#   r$   ©rˆ  gš™™™™™ù?g333333û?Úabc©r#   r$   ©ÚindexÚcolumns©r—   ÚIndexr¾   rß   r¨  ©r°  r±  rÅ   r   r   r   Ú test_pandas_serialize_round_trip  s   þrµ  c                  C   sH   t jg d¢dd} ddg}t jg d¢tdƒdœ| |d	}t|d
d d S )Nr®   r©  rª  r#   r$   r¬  r­  r®  r¯  TrÊ   r²  r´  r   r   r   Ú)test_pandas_serialize_round_trip_nthreads  s   þr¶  c                  C   sf   t jg d¢dd} t jtdƒd d}t j | |g¡}ddg}t jg d¢tdƒd	œ||d
}t|ƒ d S )Nr®   Úlevel_1rª  Údefr#   r$   r¬  r­  r®  r¯  )r—   r³  rß   Ú
MultiIndexró   r¾   r¨  )Úindex1Úindex2r°  r±  rÅ   r   r   r   Ú,test_pandas_serialize_round_trip_multi_index#  s   ýr¼  c                  C   s   t  ¡ } t| ƒ d S r   )r—   r¾   r¨  )rÅ   r   r   r   Ú%test_serialize_pandas_empty_dataframe2  s   r½  c                  C   s:   t  ttg d¢dƒƒ¡} t | ¡}t |¡}t|| ƒ d S )Nr¬  r­  )r—   r¾   rß   rT  r)   r¦  r§  r   )rÅ   ru   r€   r   r   r   Ú3test_pandas_serialize_round_trip_not_string_columns8  s   

r¾  c                  C   st   t jdg d¢ig d¢d} t  dg d¢i¡}tj| dd}t |¡}t||ƒ tj| dd}t |¡}t|| ƒ d S )Nr  r®   )r°  FrÖ   T)r—   r¾   r)   r¦  r§  r   )rÅ   r   ru   r€   r   r   r   Ú'test_serialize_pandas_no_preserve_index@  s   


r¿  c                  C   sr   d} t  tj | ¡dtjdddgdœ¡}tj |¡}|j	 
¡ }| 
¡ }tj |¡}tj ||¡}| |¡s7J ‚d S )Nr   r#   r$   r%   r&   r½   )r—   r¾   r¿   r   rÀ   rÂ   r)   rÃ   rÄ   r(   rB  rF   rR  rU  rQ   )r9   rÅ   r=   Ús_schemaÚs_batchÚrecons_schemaÚrecons_batchr   r   r   Ú#test_schema_batch_serialize_methodsN  s   
þ
rÄ  c                  C   s¢   dddœ} dddœ}t  dt  ¡ ¡}t jdt  ¡ | d}t j||g|d}| ¡ }t j |¡}| |¡s6J ‚|j	|ks=J ‚|d j	d u sFJ ‚|d	 j	| ksOJ ‚d S )
Ns   bars   field)rX  s   kinds   schemar  r  )r<  r   r¯   )
r)   rµ   r&  Ústringr(   rB  rF   rR  rQ   r<  )Úfield_metadataÚschema_metadataÚf0Úf1r(   rÀ  rÂ  r   r   r   Ú'test_schema_serialization_with_metadata^  s   

rÊ  c                 C   s@   t j || j¡}| | ¡ W d   ƒ d S 1 sw   Y  d S r   )r)   rF   rG   r(   r3   )r=   r   r:   r   r   r   rŠ  p  s   "ÿrŠ  c                    sH   t j | ¡‰ ‡ fdd„tˆ jƒD ƒW  d   ƒ S 1 sw   Y  d S )Nc                    r”  r   )rP   r–  r—  r   r   r!   w  s    zread_file.<locals>.<listcomp>)r)   rF   rL   r-   rM   )r‡   r   r—  r   rŽ  u  s   $ÿrŽ  c                  C   s®   t  dt  ¡ fg¡} t  ¡ }t j || ¡ W d   ƒ n1 s w   Y  | ¡ }t  t  |¡¡}| 	¡ }W d   ƒ n1 s@w   Y  t
|ƒdksMJ ‚|j | ¡sUJ ‚d S )Nrµ   r   )r)   r(   Úint64r}   rF   rG   r   r…   rK   r~   rN   rQ   )r(   r   ru   rW   r>   r   r   r   Útest_write_empty_ipc_filez  s   ÿ
ÿrÌ  c                     sŠ  dd„ ‰ ‡ fdd„} t | ƒ ƒ}t |¡}tj ˆ ƒ |¡!}d }|ƒ d us&J ‚t|ƒ| ƒ ks/J ‚|ƒ d u s6J ‚W d   ƒ n1 s@w   Y  tt | ƒ ƒƒ}t |¡}tj ˆ ƒ |¡!}d }|ƒ d uscJ ‚t|ƒ| ƒ kslJ ‚|ƒ d u ssJ ‚W d   ƒ n1 s}w   Y  | ƒ }t 	t
¡ tj dt ¡ fg|¡}W d   ƒ n1 s¡w   Y  t 	t
¡ tj d |¡}W d   ƒ d S 1 s¾w   Y  d S )Nc                   S   s   t  dt  ¡ fg¡S )Nrµ   )r)   r(   rË  r   r   r   r   Úmake_schema‹  rI   z0test_py_record_batch_reader.<locals>.make_schemac                     s6   ˆ ƒ } t jg d¢g| d}t jddgg| d}||gS )Nr®   r'   r  r   )r)   r.   )r(   Úbatch1Úbatch2©rÍ  r   r   Úmake_batchesŽ  s   z1test_py_record_batch_reader.<locals>.make_batchesrµ   )r   ÚweakrefÚrefr)   ÚRecordBatchReaderr1   rß   Úiterrq   rr   rÑ   rË  )rÑ  r;   rÆ   rW   r   rÐ  r   Útest_py_record_batch_readerŠ  sF   


ÿû

ÿû	ÿý"þrÖ  c                  C   sä  G dd„ dƒ} t jt jg d¢t  ¡ dgdgdt jt jg d¢t  ¡ dgdgdg}| |ƒ}t j |¡}t j |¡}| ¡ |ksCJ ‚t j |¡}| ¡ |ksQJ ‚t jj||d j	d	}| ¡ |ksdJ ‚t  	t  
dt  ¡ ¡g¡}t jj||d	}| ¡ | |¡ksƒJ ‚tjt jjd
d t jj|t  	t  
dt  t  ¡ ¡¡g¡d	 W d   ƒ n1 s¬w   Y  t t¡ t j |d d ¡ W d   ƒ n1 sËw   Y  t t¡ t jj||d d	 W d   ƒ d S 1 sëw   Y  d S )Nc                   @   s   e Zd Zdd„ Zddd„ZdS )zAtest_record_batch_reader_from_arrow_stream.<locals>.StreamWrapperc                 S   s
   || _ d S r   )r;   )r   r;   r   r   r   r   º  r   zJtest_record_batch_reader_from_arrow_stream.<locals>.StreamWrapper.__init__Nc                 S   s"   t j | jd j| j¡}| |¡S )Nr   )r)   rÔ  r1   r;   r(   Ú__arrow_c_stream__)r   Úrequested_schemarW   r   r   r   r×  ½  s   ÿ
zTtest_record_batch_reader_from_arrow_stream.<locals>.StreamWrapper.__arrow_c_stream__r   )r@   rA   rB   r   r×  r   r   r   r   ÚStreamWrapper¹  s    rÙ  r®   r²   r  rí   ©r  r   r„   r   r'   úField 0 cannot be castr  )r)   r.   r¶   rË  r0   r1   rÔ  Úfrom_streamr~   r(   rµ   r0  Úcastrq   rr   ÚlibÚArrowTypeErrorÚlist_rÑ   )rÙ  r*  Úwrapperr   rW   Úgood_schemar   r   r   Ú*test_record_batch_reader_from_arrow_stream·  s4   "
"þÿÿÿ"ÿrã  c                  C   s  t  t  dt  ¡ ¡g¡} t jt jg d¢t  ¡ dgdgdt jt jg d¢t  ¡ dgdgdg}t j |¡}t j | |¡}| 	| ¡ 
¡ |ksHJ ‚t  t  dt  ¡ ¡g¡}t j | |¡}| 	|¡ 
¡ | 	|¡ksiJ ‚t j | |¡}tjtdd | 	t  g ¡¡ W d   ƒ n1 sŠw   Y  t j | |¡}tjt jjdd | 	t  t  dt  t  ¡ ¡¡g¡¡ W d   ƒ n1 s¼w   Y  t  t  dt  ¡ ¡g¡} t jt d	d
d¡gt  ¡ d}t j|gdgdt j|gdgdg}t j |¡}t j | |¡}| 	| ¡ 
¡ |ksJ ‚d S )Nr  r®   r²   rí   rÚ  zTarget schema's field namesr  rÛ  iè  r„   é   )r)   r(   rµ   rË  r.   r¶   r0   r1   rÔ  rÝ  r~   r0  rq   rr   r  rÞ  rß  rà  Údate32ÚdatetimeÚdate)Ú
schema_srcr*  Ú	table_srcrW   Ú
schema_dstÚarrr   r   r   Útest_record_batch_reader_castè  s0   ""þÿ&ÿ$rì  c                  C   s\  t  t  dt  ¡ ¡g¡} t jt jg d¢t  ¡ dgdgdg}t jt jg d¢t  ¡ dgdgdg}t j |¡}t j |¡}t j | |¡}t  t  dt  	¡ ¡g¡}| 
|¡ ¡ | 
|¡ks_J ‚t j | |¡}t  t jdt  	¡ ddg¡}| 
|¡ ¡ | 
|¡ks‚J ‚t j | |¡}| 
|¡}tjt jjdd	 | ¡  W d   ƒ d S 1 s§w   Y  d S )
Nr  )r¯   r°   Nr²   rí   r®   F)ÚnullablezCan't cast arrayr  )r)   r(   rµ   rË  r.   r¶   r0   r1   rÔ  r0  rÝ  r~   rq   rr   rÞ  rs   )rè  Údata_with_nullsÚdata_without_nullsÚtable_with_nullsÚtable_without_nullsrW   rê  Úcasted_readerr   r   r   Ú#test_record_batch_reader_cast_nulls  s$   "ÿ"ÿ

"ÿró  rY   )jÚcollectionsr   ræ  r
   r¢   rq   r   r_  Ú	threadingrÒ  Únumpyr¿   ÚImportErrorÚpyarrowr)   Úpyarrow.tests.utilr   r   Úpandas.testingr   Úpandasr—   r   rC   r[   r_   r   re   rg   rh   rl   rn   rv   rz   r|   ÚmarkÚparametrizer‚   r   r›   rª   r¬   Ú	processesr¹   rÇ   rÉ   rF   r  rÎ   rÒ   rÔ   rÕ   rÚ   rä   rê   rï   rû   r  r
  rô   r  r  r  r  r  r,  r3  r4  r5  r8  r;  rA  rI  rJ  rQ  rS  rW  r\  ÚThreadr]  rw  r  Úsocketsr†  r‡  r  r  rŸ  r   r¤  r¨  rµ  r¶  r¼  r½  r¾  r¿  rÄ  rÊ  rŠ  rŽ  rÌ  rÖ  rã  rì  ró  r   r   r   r   Ú<module>   s2  ÿÿ'


þþû

þ


þ
þ




1	ÿû
K$




!




%
	








-1%