o
    h                     @   s  d dl m Z mZmZ d dlZd dlZd dlZd dlmZ d dlZd dl	Z	d dl
Z
d dlZd dlZd dlZd dlmZ d dlmZmZmZ d dlmZmZmZmZmZmZmZmZmZmZm Z  d dl!m"Z" ej#$ej#%e&Z'G dd	 d	eZ(ej)d
d Z*ej)dd Z+ej)dd Z,ej)dd Z-ej)dd Z.ej)dd Z/ej)dd Z0ej)dd Z1ej)dd Z2dZ3ej)dd Z4ej)dd  Z5ej)d!d" Z6ej)d#d$ Z7ej)d%d& Z8ej)ej9dd'd(ej9dd)d(ej9dd*d(ej9dd+ej:j;d,ej9dd-ej:j<d,ej9dd.ej:j=d,ej9d d/ej:j5d,ej9dd0d(ej9dd1d(ej9dd2d(ej9d"d3d(ej9d$d4d(ej9d&d5ej:j;d,gd6d7d8 Z>ej)d9d: Z?ej)d;d< Z@ej)d=d> ZAej)d?d@ ZBdAdB ZCdCdD ZDdEdF ZEdGdH ZFdIdJ ZGej:j;dKdL ZHdMdN ZIdOdP ZJdQdR ZKdSdT ZLdUdV ZMdWdX ZNdYdZ ZOd[d\ ZPd]d^ ZQd_d` ZRdadb ZSdcdd ZTdedf ZUdgdh ZVdidj ZWdkdl ZXdmdn ZYdodp ZZdqdr Z[dsdt Z\dudv Z]dwdx Z^dydz Z_ej:jej:`d{dde_fdd|e_fd}dejafd}d~ejafgdd Zbdd Zcdd Zdej:jej:`ddde_fdd|e_fd}dejefd}d~ejefgdd Zfej:jej:`ddde_e_fdd|e_e_fd}dejaejefd}d~ejaejefgdd Zgdd Zhdd Zidd Zjdd Zkdd Zlej:j<dd Zmej:j;dd Znej:j;dd Zoej:j;dd Zpej:j=dd Zqej:j5dd Zrej:`ddedfdedfdedfdedfdedfdedfdedfdedfdedfdedfdedfdedfdedfgdd Zsej:`ddddde&gdd Ztej:j;dd Zuej:j<dd Zvdd Zwdd Zxdd ZyddÄ Zzddń Z{ddǄ Z|ddɄ Z}dd˄ Z~dd̈́ Zddτ Zddф Zej:j;ddӄ Zej:j;ddՄ Zej:j;ddׄ Zej:j;ddل Zddۄ Zej:j;dd݄ Zej:j;dd߄ Zej:jej:jej:j;dd Zej:j;dd ZdS )    )datetimetimezone	timedeltaN)urlopen)assert_file_not_found)_filesystem_uriProxyHandler_configure_s3_limited_user)FileTypeFileInfoFileSelector
FileSystemLocalFileSystemSubTreeFileSystem_MockFileSystemFileSystemHandlerPyFileSystemFSSpecHandler
copy_files)find_free_portc                   @   s   e Zd Zd'ddZdd Zdd Zdd	 Zd
d Zdd Zdd Z	dd Z
dd Zdd Zdd Zdd Zdd Zdd Zdd Zd d! Zd"d# Zd$d% Zd&S )(DummyHandler*   c                 C   s
   || _ d S N)_value)selfvalue r   I/var/www/vscode/kcb/lib/python3.10/site-packages/pyarrow/tests/test_fs.py__init__.   s   
zDummyHandler.__init__c                 C   s   t |tr| j|jkS tS r   
isinstancer   r   NotImplementedr   otherr   r   r   __eq__1      
zDummyHandler.__eq__c                 C   s   t |tr| j|jkS tS r   r   r"   r   r   r   __ne__6   r%   zDummyHandler.__ne__c                 C      dS )Ndummyr   r   r   r   r   get_type_name;      zDummyHandler.get_type_namec                 C   s   |S r   r   r   pathr   r   r   normalize_path>   r+   zDummyHandler.normalize_pathc                 C   s~   g }|D ]8}d|v r| t|tj qd|v r"| t|tj qd|v r0| t|tj qd|v r;| t  qt|S )Nfiledirnotfoundbadtype)appendr   r
   File	DirectoryNotFoundobjectIOError)r   pathsinfor-   r   r   r   get_file_infoA   s   zDummyHandler.get_file_infoc                 C   sZ   |j dkr|jr
g S t|j tdtjddtdtjg}|jr+|tdtjddg7 }|S )Nsomedirsomedir/file1{   )sizesomedir/subdir1somedir/subdir1/file2i  )base_dirallow_not_foundFileNotFoundErrorr   r
   r4   r5   	recursive)r   selectorinfosr   r   r   get_file_info_selectorQ   s   


z#DummyHandler.get_file_info_selectorc                 C   s4   |dkr|du s
J d S |dkr|du sJ d S t )NrE   Tnon-recursiveF)r8   )r   r-   rE   r   r   r   
create_dira   s
   zDummyHandler.create_dirc                 C      |dksJ d S )N
delete_dirr   r,   r   r   r   rL   i      zDummyHandler.delete_dirc                 C   s   | dst|dksJ d S )N/delete_dir_contents)strip
ValueError)r   r-   missing_dir_okr   r   r   rO   l   s   
z DummyHandler.delete_dir_contentsc                 C   s   d S r   r   r)   r   r   r   delete_root_dir_contentsq   r+   z%DummyHandler.delete_root_dir_contentsc                 C   rK   )Ndelete_filer   r,   r   r   r   rT   t   rM   zDummyHandler.delete_filec                 C      |dksJ |dksJ d S )N	move_frommove_tor   r   srcdestr   r   r   movew      zDummyHandler.movec                 C   rU   )Ncopy_file_fromcopy_file_tor   rX   r   r   r   	copy_file{   r\   zDummyHandler.copy_filec                 C   *   d|v rt |d|d}t|S )Nr1   z{0}:input_streamutf8rD   formatencodepaBufferReaderr   r-   datar   r   r   open_input_stream      
zDummyHandler.open_input_streamc                 C   r`   )Nr1   z{0}:input_filera   rb   rg   r   r   r   open_input_file   rj   zDummyHandler.open_input_filec                 C      d|v rt |t S Nr1   rD   re   BufferOutputStreamr   r-   metadatar   r   r   open_output_stream      zDummyHandler.open_output_streamc                 C   rl   rm   rn   rp   r   r   r   open_append_stream   rs   zDummyHandler.open_append_streamN)r   )__name__
__module____qualname__r   r$   r&   r*   r.   r;   rH   rJ   rL   rO   rS   rT   r[   r_   ri   rk   rr   rt   r   r   r   r   r   -   s&    
r   c                    s   t t  fdddddS )Nc                        |    S r   as_posixptempdirr   r   <lambda>       zlocalfs.<locals>.<lambda>Tfspathfnallow_move_dirallow_append_to_filedictr   requestr~   r   r}   r   localfs   s   
r   c                    s"   t ttt  fdddddS )Nc                    rx   r   ry   r{   r}   r   r   r      r   zpy_localfs.<locals>.<lambda>Tr   )r   r   r   r   r   r   r}   r   
py_localfs   s   
r   c                 C   s   t t dd dddS )Nc                 S      | S r   r   r{   r   r   r   r          zmockfs.<locals>.<lambda>Tr   )r   r   r   r   r   r   mockfs   s   r   c                 C   s   t ttt dd dddS )Nc                 S   r   r   r   r{   r   r   r   r      r   zpy_mockfs.<locals>.<lambda>Tr   )r   r   r   r   r   r   r   r   	py_mockfs   s   r   c                    s   t tdd fdddddS )NTuse_mmapc                    rx   r   ry   r{   r}   r   r   r      r   z#localfs_with_mmap.<locals>.<lambda>r   r   r   r   r}   r   localfs_with_mmap   s   
r   c                 C   s"   t tt||d dd dddS )Nr   c                 S   r   r   r   r{   r   r   r   r      r   z!subtree_localfs.<locals>.<lambda>Tr   )r   r   str)r   r~   r   r   r   r   subtree_localfs   s   r   c              
   c   s    | j jd ddlm} |d \}}d}|| d| ddtd	d
dd}z|| W n tyK } zt	d| d|  W Y d }~nd }~ww t
||jdddV  || d S )Ngcsr   GcsFileSystem
connectionpyarrow-filesystem/:httpT-   )secondstest-project-id)endpoint_overridescheme	anonymousretry_time_limit
project_idzCould not create directory in : Fr   )configpyarrowrequires
pyarrow.fsr   r   rJ   OSErrorpytestskipr   __add__rL   )r   
gcs_serverr   hostportbucketr   er   r   r   gcsfs   s2   "r   c           	      c   sx    | j jd ddlm} |d \}}}}d}|||d||dddd	}|| t||jd
d
dV  |	| d S )Ns3r   S3FileSystemr   r   {}:{}r   T)
access_key
secret_keyr   r   allow_bucket_creationallow_bucket_deletionFr   )
r   r   r   r   r   rc   rJ   r   r   rL   )	r   	s3_serverr   r   r   r   r   r   r   r   r   r   s3fs   s*   

r   c                 C   s    d}t t||d |jdddS )Nzpyarrow-filesystem/prefix/r   Fr   )r   r   r   )r   r   prefixr   r   r   subtree_s3fs  s   r   a  {
    "Version": "2012-10-17",
    "Statement": [
        {
            "Effect": "Allow",
            "Action": [
                "s3:ListAllMyBuckets",
                "s3:PutObject",
                "s3:GetObject",
                "s3:ListBucket",
                "s3:PutObjectTagging",
                "s3:DeleteObject",
                "s3:GetObjectVersion"
            ],
            "Resource": [
                "arn:aws:s3:::*"
            ]
        }
    ]
}c                 c   s    | j jd ddlm} |d \}}}}| d| }d}d}	|||||||d}
|
|	 t|
|	jd	d	d
V  |
|	 d S )Nazurer   AzureFileSystemr   r   r   r   account_nameaccount_keyblob_storage_authoritydfs_storage_authorityblob_storage_schemedfs_storage_schemeTr   )	r   r   r   r   r   rJ   r   r   rL   )r   azure_serverr   r   r   r   r   azurite_authorityazurite_scheme	containerr   r   r   r   azurefs,  s,   
r   c                 C   sX   | j jd t std ddlm} |\}}}||||d}t	|dd ddd	S )
NhdfsCannot locate libhdfsr   HadoopFileSystem)r   userc                 S   r   r   r   r{   r   r   r   r   V  r   zhdfs.<locals>.<lambda>Tr   )
r   r   r   re   have_libhdfsr   r   r   r   r   )r   hdfs_connectionr   r   r   r   r   r   r   r   r   I  s   

r   c                    s4   t d}|d}ttt| fdddddS )Nfsspecr/   c                    rx   r   ry   r{   r}   r   r   r   b  r   z#py_fsspec_localfs.<locals>.<lambda>Tr   )r   importorskip
filesystemr   r   r   r   r~   r   r   r   r}   r   py_fsspec_localfs\  s   



r   c                 C   sH   t jddd}|jdkrt d |d}ttt|dd d	d	d
S )Nr   z0.7.5)
minversionz0.8.5z,Bug in fsspec 0.8.5 for in-memory filesystemmemoryc                 S   r   r   r   r{   r   r   r   r   q  r   z$py_fsspec_memoryfs.<locals>.<lambda>Tr   )r   r   __version__r   r   r   r   r   r   r   r   r   py_fsspec_memoryfsh  s   



r   c           	      c   sv    t d}|d \}}}}d}|j||td||dd}tt|}|| t||jddd	V  |	| d S )
Nr   r   r   zhttp://{}:{})endpoint_url)keysecretclient_kwargsFTr   )
r   r   r   r   rc   r   r   rJ   r   rL   )	r   r   r   r   r   r   r   r   r   r   r   r   py_fsspec_s3fsw  s$   

r   zLocalFileSystem())idzLocalFileSystem(use_mmap=True)z$SubTreeFileSystem(LocalFileSystem())r   )r   marksr   r   r   z_MockFileSystem()z-PyFileSystem(ProxyHandler(LocalFileSystem()))z-PyFileSystem(ProxyHandler(_MockFileSystem()))z5PyFileSystem(FSSpecHandler(fsspec.LocalFileSystem()))z8PyFileSystem(FSSpecHandler(fsspec.filesystem("memory")))z0PyFileSystem(FSSpecHandler(s3fs.S3FileSystem())))paramsc                 C   s   |  | jS r   )getfixturevalueparamr   r   r   r   filesystem_config  s   <r   c                 C      | d S Nr   r   r   r   r   r   r        r   c                 C   r   )Nr   r   r   r   r   r   r     r   r   c                 C   r   )Nr   r   r   r   r   r   r     r   r   c                 C   r   )Nr   r   r   r   r   r   r     r   r   c                 C   sz   t | jtsJ t | jtsJ | jdksJ | jt| j d ks&J | jj}|d us0J |	d t
dks;J d S )Nr   g    eA)r    mtimer   mtime_nsintr   approx	timestamptzinfo	utcoffsetr   )	file_infor   r   r   r   check_mtime  s   
r   c                 C   s    | j d u sJ | jd u sJ d S r   )r   r   r   r   r   r   check_mtime_absent  s   r   c                 C   s"   | j d u rt|  d S t|  d S r   )r   r   r   r   r   r   r   check_mtime_or_absent  s   
r   c                 C   s   | j dkrtjdd d S d S )Npy::fsspec+('s3', 's3a')zNot working with fsspec's s3fsreason)	type_namer   xfailr   r   r   r   skip_fsspec_s3fs     
r   c                 C   s   | j dkrtj|d d S d S )Nabfsr   )r   r   r   )r   r   r   r   r   
skip_azure  r  r  c                 C   s   ddl m} t| tdd | d \}}}}|ddd||dd}|d	 tjtj	d
d |d W d    n1 s>w   Y  tjtj	dd |
d W d    d S 1 s\w   Y  d S )Nr   r   test_fs_limited_user
limited123r   r   r   )r   r   r   r   zexisting-bucket/testzBucket 'new-bucket' not foundmatchz
new-bucketzWould delete bucketzexisting-bucket)r   r   r	   _minio_limited_policyrc   rJ   r   raisesre   ArrowIOErrorrL   )r   r   r   r   _r   r   r   r   +test_s3fs_limited_permissions_create_bucket  s$   

"r  c                  C   s,  t dtj} td}|jdksJ |jdksJ |jtj	ks!J |j
d u s(J t| tdtjddd}|jdks<J |jdksCJ |jtjksKJ |j
dksRJ |jd	ksYJ t| td
tj| d}|jd
kslJ |jd
kssJ |jtjks{J |j
d u sJ |j| ksJ |jdksJ t| d S )NR]foo/barbarzfoo/baz.txtr>   g  `A)typer?   r   zbaz.txtl    t<|+. foo)r  r   l    4 |+. )r   fromtimestampr   utcr   r-   	base_namer  r
   Unknownr?   r   r4   r   r   r5   r   )dtr:   r   r   r   test_file_info_constructor  s0   
r  c                   C   s6   t t t  W d    d S 1 sw   Y  d S r   )r   r	  	TypeErrorr   r   r   r   r   'test_cannot_instantiate_base_filesystem6  s   "r  c                  C   s   t  } t  }t }| | sJ | |sJ tt | d W d    n1 s,w   Y  | |   kr<|ks?J  J | dksEJ ||ksKJ |t ksRJ td| td| ks^J td| td|ksjJ td| td| ksvJ d S )Nstring   /basez/other)r   r   equalsr   r	  r  r   )fs0fs1fs2r   r   r   test_filesystem_equals;  s   r!  c                 C   sH   t jtdd | d  W d    n1 sw   Y  | d us"J d S )Nzgot NoneTyper  )r   r	  r  r  r   r   r   r   test_filesystem_equals_noneO  s   r"  c                  C   s|   t  } td| }|jdksJ |j| ksJ t|dsJ tdt  }|jdks,J |j| ks3J t|ds<J d S )Nr  z/base/zHSubTreeFileSystem(base_path=/base/, base_fs=<pyarrow._fs.LocalFileSystemz/another/base/zPSubTreeFileSystem(base_path=/another/base/, base_fs=<pyarrow._fs.LocalFileSystem)r   r   	base_pathbase_fsrepr
startswith)r   subfsr   r   r   test_subtree_filesystemV  s   
r(  c                 C   sT   | j dd dkrtjdd || }||}t|ts!J || s(J d S )N::mock"MockFileSystem is not serializabler   )	r   splitr   r   dumpsloadsr    r   r  )r   pickle_module
serializedrestoredr   r   r   test_filesystem_picklingf  s   

r3  c                 C   s  | j dd dkrtjdd t|  |d}|d}|d}| | | | W d    n1 s5w   Y  | |}|d	 W d    n1 sOw   Y  ||	| }|
|||g\}}	}
|jtjksoJ |	jtjkswJ |
jtjksJ d S )
Nr)  r*  r+  r,  r   	a/aa/aaa/a/bbc.txt   test)r   r-  r   r   r   rJ   rr   writer/  r.  r;   r  r
   r5   r4   )r   r   r0  aaabbcfpr2  aaa_infobb_infoc_infor   r   r   ,test_filesystem_is_functional_after_picklingp  s$   
r@  c                  C   s,   t  } | jdks
J t } | jdksJ d S )Nlocalr+  )r   r   r   r   r   r   r   test_type_name  s   rB  c                 C   s   |  ddks	J d S )Nr  )r.   r   r   r   r   test_normalize_path  s   rC  c              	   C   sp   G dd d}dd| t  i g dd t g}|D ]}tt | | W d    n1 s0w   Y  qd S )Nc                   @   s   e Zd ZdS )z-test_non_path_like_input_raises.<locals>.PathN)ru   rv   rw   r   r   r   r   Path  s    rD     g?c                   S   r'   )NrE  r   r   r   r   r   r     r   z1test_non_path_like_input_raises.<locals>.<lambda>)tuplepathlibrD  r   r	  r  rJ   )r   rD  invalid_pathsr-   r   r   r   test_non_path_like_input_raises  s   rI  c                 C   s  |d}|d}|d}|d}|  | | | W d    n1 s%w   Y  | |}|d W d    n1 s?w   Y  | ||||g\}}}	}
|j|ksXJ dt|v s`J |jdksgJ | jdkru|jt	j
kstJ n|jt	jks}J d	t|v sJ |jd u sJ t| |jt|ksJ |jd
ksJ |jdksJ |jt	jksJ dt|v sJ |jdksJ | jdvrt| |	jt|ksJ |	jdksJ |	jdksJ |	jt	jksJ dt|	v sJ |	jdksJ | jdvrt|	 |
jt|ksJ |
jdksJ |
jdksJ |
jt	j
ks!J |
jd u s)J |
jd u s1J dt|
v s:J t|
 | |}|j|jksLJ |j|jksUJ d S )Nr4  r5  r6  zzzr7  r9   r   zFileType.Directoryr:  zFileType.Filer   )zpy::fsspec+memoryr   txtr  zFileType.NotFound)rJ   rr   r8  r;   r-   r%  	extensionr   r  r
   r6   r5   r?   r   r   r  r4   r   r   r   )r   r   r9  r:  r;  rJ  r<  r=  r>  r?  zzz_info	aaa_info2r   r   r   test_get_file_info  s`   




rP  c              	   C   s  |d}|d}|d}|d}|d}|d}z|  | | | W d    n1 s.w   Y  | | W d    n1 sCw   Y  |  | | | W d    n1 s]w   Y  |  | t|ddd	}|j|ksuJ | |}	| jd
krt|	dk nt|	dksJ |	D ]D}
|
j|s|
j|s|
j|r|
j	t
jksJ n#|
jd|s|
jd|r|
j	t
jksJ ntd|
jt|
 qt|dd}| |}	| jd
krt|	dksJ nt|	dksJ W | | d S W | | d S | | w )Nzselector-dir/zselector-dir/test_file_azselector-dir/test_file_bzselector-dir/test_dir_az#selector-dir/test_dir_a/test_file_czselector-dir/test_dir_bFT)rC   rE   r   r     rN   zunexpected path {}rE      )rJ   rr   r   rB   r;   r   lenr-   endswithr  r
   r4   rstripr5   rQ   rc   r   rL   )r   r   rB   file_afile_bdir_afile_cdir_brF   rG   r:   r   r   r    test_get_file_info_with_selector  s\   








r\  c                 C   s~   t |  |d}ttj | | W d    n1 sw   Y  | | | | |d}| j|dd | | d S )Nztest-directory/zdeeply/nested/test-directory/TrR  )r   r   r	  re   r
  rL   rJ   )r   r   dr   r   r   test_create_dir  s   

r^  c                 C   s   t |  |d}|d}| | | | ttj | | W d    n1 s,w   Y  ttj | | W d    d S 1 sHw   Y  d S )N
directory/directory/nested/)r   rJ   rL   r   r	  re   r
  r   r   r]  ndr   r   r   test_delete_dir"  s   

"rc  c                 C   s   t |  |d}|d}| | | | | | | |}|jtjks(J |d}|d}|d}| | | | | |}|d W d    n1 sSw   Y  | | | |}|jtjksjJ d S )Nr_  r`  
directory2zdirectory2/nestedzdirectory2/nested/target-file   data)	r   rJ   rL   r;   r  r
   r6   rr   r8  )r   r   r]  rb  dir_infofsr   r   r   $test_delete_dir_with_explicit_subdir0  s&   







ri  c                 C   s   t |  |d}|d}| | | | ttj | | W d    n1 s,w   Y  | j|dd ttj | | W d    n1 sNw   Y  | | ttj | | W d    d S 1 sow   Y  d S )Nr_  r`  T)rR   )r   rJ   rO   r   r	  re   r
  rL   ra  r   r   r   test_delete_dir_contentsP  s    


"rj  c                 C   s2  | d }| d }|d}|d}| | ttj |d W d    n1 s+w   Y  ttj |d W d    n1 sFw   Y  ttj |d W d    n1 saw   Y  |jddd	 |jddd	 |jddd	 ttj || W d    d S 1 sw   Y  d S )
Nr   r   r_  r`  rK  rN   //Taccept_root_dir)rJ   r   r	  re   ArrowInvalidrO   r
  rL   )r   r   r   r]  rb  r   r   r   _check_root_dir_contentsb  s(   
"ro  c                 C   s   t |  t | d S r   )ro  )r   r   r   r   r   test_delete_root_dir_contentsx  s   rp  c                 C   s^   |d}|d}|  | W d    n1 sw   Y  | || | | | | d S )Nztest-copy-source-fileztest-copy-target-file)rr   r_   rT   r   r   rh  tr   r   r   test_copy_file}  s   
rs  c                 C   s   t | d |d}|d}| | |r<| || ttj | | W d    n1 s0w   Y  | | d S ttj | || W d    d S 1 sTw   Y  d S )N.Not implemented yet in for Azure. See GH-40025zsource-dir/ztarget-dir/)r  rJ   r[   r   r	  re   r
  rL   )r   r   r   rh  rr  r   r   r   test_move_directory  s   

"ru  c                 C   s   t |  t| d |d}|d}| | W d    n1 s!w   Y  | || ttj | | W d    n1 sBw   Y  | | d S )Nrt  ztest-move-source-fileztest-move-target-file)	r   r  rr   r[   r   r	  re   r
  rT   rq  r   r   r   test_move_file  s   
rv  c                 C   s   |d}|  | W d    n1 sw   Y  | | ttj | | W d    n1 s4w   Y  |d}| | |d}|  |}|d W d    n1 s[w   Y  | | d S )Nztest-delete-target-fileztest-delete-nestedztest-delete-nested/target-filere  )	rr   rT   r   r	  re   r
  rJ   r8  rL   )r   r   r|   r]  rg  rh  r   r   r   test_delete_file  s   

rw  c                 C   r   r   r   )vr   r   r   identity  r+   ry  )compressionbuffer_size
compressor@   gzip   c           	      C   s   |d}dd }|  |}||| W d    n1 sw   Y  | |||}| }W d    n1 s:w   Y  ||ksEJ d S )Nzopen-input-streams   some data for reading
   )rr   r8  ri   read)	r   r   rz  r{  r|  r|   rh   rh  resultr   r   r   test_open_input_stream  s   
r  c                 C   s   |d}dd }|  |}|| W d    n1 sw   Y  tdd }| |}| }W d    n1 s<w   Y  ||ksGJ | |}|| | }W d    n1 s`w   Y  |||d  ksoJ d S )Nzopen-input-file	   some data   r  )rr   r8  rT  rk   r  seek)r   r   r|   rh   rh  	read_fromrg  r  r   r   r   test_open_input_file  s   


r  c                 C   sB   |d}t t | | W d    d S 1 sw   Y  d S )Nzopen-input-stream-not-found)r   r	  rD   ri   )r   r   r|   r   r   r    test_open_input_stream_not_found  s   "r  )rz  r{  decompressorc                 C   s   |d}dd }|  |||}|| W d    n1 sw   Y  | |||}|t||ks7J W d    d S 1 sBw   Y  d S )Nzopen-output-streams   some data for writingr  )rr   r8  ri   r  rT  )r   r   rz  r{  r  r|   rh   rg  r   r   r   test_open_output_stream  s   "r  )rz  r{  r|  r  c                 C   s  |d}|d}|  |}	|	| W d    n1 sw   Y  |rf| j|||d}
|
d W d    n1 s<w   Y  | |}
|
 }W d    n1 sUw   Y  ||}|dksdJ d S ttj | j|||d W d    d S 1 sw   Y  d S )Nzopen-append-streams   already existing)rz  r{  s   
newly addeds   already existing
newly added)	rr   r8  rt   ri   r  r   r	  re   ArrowNotImplementedError)r   r   rz  r{  r|  r  r   r|   initialrh  rg  r  r   r   r   test_open_append_stream  s,   
"r  c           	      C   s   |d}ddi}d}| j ||d}|| W d    n1 s!w   Y  | |}| |ks4J | }W d    n1 sBw   Y  | jdv sQd| jv rkt| d | D ]\}}|| | kshJ qZd S |i ksqJ d S )	Nzopen-output-stream-metadataContent-Typezx-pyarrow/testr  )rq   )r   r   r  r+  zWAzure filesystem currently only returns system metadata not user metadata. See GH-40026)	rr   r8  ri   r  rq   r   r  itemsrd   )	r   r   r|   rq   rh   rg  got_metadatakrx  r   r   r    test_open_output_stream_metadata0  s$   
r  c                   C   sD   t dd tt t dd W d    d S 1 sw   Y  d S )NFr   )xxx)r   r   r	  r  r   r   r   r   test_localfs_optionsG  s   
"r  c                 C   sf  | d }t   |d W d    n1 sw   Y  t   |d W d    n1 s/w   Y  t   |jddd W d    n1 sIw   Y  t   |d W d    n1 saw   Y  t   |d W d    n1 syw   Y  t   |dd W d    n1 sw   Y  t   |dd W d    d S 1 sw   Y  d S )Nr   z/non/existent/filez/non/existent/dirFrR  z/non/existentz/xxx)r   ri   rr   rJ   rL   rT   r[   r_   )r   r   r   r   r   test_localfs_errorsO  s,   "r  c                 C   s   | d }t t}|j}|| | g\}}|j| jks#J |j	| j
ks-J t| |j	| j
ks;J t| d S r   )rG  rD  __file__parentr;   rz   r?   statst_sizer   st_mtime_nsr   )r   r   	file_pathdir_pathr   rf  r   r   r   test_localfs_file_infoc  s   


r  c                 C   s`   t dtj}t|}|d W d    n1 sw   Y  |dg\}|j|ks.J d S )Nr  r  )r   r  r   r  r   rr   r;   r   )r   r  r   r:   r   r   r   test_mockfs_mtime_roundtripq  s   r  c              	   C   s  ddl m} t }|dd|ddddd	}t||sJ |jdks#J |jdks*J | | ||ks6J | }t||s@J | | ||ksLJ |d
d}t||sXJ | | ||ksdJ |dddd}t||ssJ | | ||ksJ t	
t |dd W d    n1 sw   Y  t	
t |d
dd W d    n1 sw   Y  t	
t |d
dd W d    n1 sw   Y  t	
t |t d W d    d S 1 sw   Y  d S )Nr   r   abczservice_account@apachezus-west2httpslocalhost:8999r   )access_tokentarget_service_accountcredential_token_expirationdefault_bucket_locationr   r   r   Tr   authenticated-read
text/plainACLr  default_metadataaccess)r  r   )r   r  acct)r   r  )r  )r   r   r   nowr    r  r   r/  r.  r   r	  rQ   )r0  r   r  r   r   r   r   test_gcs_options{  sH   

"r  c                 C   s  ddl m}m}m}m} |ddddddd	}t||sJ |jdks$J | | ||ks0J |d
dddd}t||s?J | | ||ksKJ ||ddd}t||sZJ ||ddd}t||siJ |d
d}t||suJ | | ||ksJ ||ksJ |dd}t||sJ | | ||ksJ |dd}t||sJ | | ||ksJ |ddddd}t||sJ | | ||ksJ ||ksJ |ddd}t||sJ | | ||ksJ |dddd}t||sJ | | ||ksJ |ddd}t||sJ | | ||ks)J |ddd}t||s7J | | ||ksDJ ||ksKJ |ddd}t||sYJ | | ||ksfJ t	
t |dd  W d    n	1 s|w   Y  t	
t |dd! W d    n	1 sw   Y  t	
t |ddd" W d    n	1 sw   Y  t	
t |ddd# W d    n	1 sw   Y  t	
t |ddd$d% W d    n	1 sw   Y  t	
t |dddd& W d    n	1 s	w   Y  t	
t |d$dd' W d    n	1 s%w   Y  t	
t |d(d)gd* W d    n	1 sBw   Y  t	
t || d W d    d S 1 s_w   Y  d S )+Nr   )AwsDefaultS3RetryStrategyAwsStandardS3RetryStrategyr   S3RetryStrategyr  r   token	us-east-2r  r  )r   r   session_tokenregionr   r   rolesessionr   d   )role_arnsession_nameexternal_idload_frequencyrQ  )max_attempts)retry_strategy)r  Tr  )background_writesr  r  r  )r  r  )r   r   )r   r   )check_directory_existence_before_creationg      ?g      ?)request_timeoutconnect_timeout)r   force_virtual_addressing)r   )r   )r   r  )r   r  arn)r   r   r  )r   r   r   )r  r   r  r  r  )r   r  r  r   r  r    r  r/  r.  r   r	  rQ   )r0  r  r  r   r  r   r   r   r   r   test_s3_options  s   


$r  c           
      C   s  ddl m} dddd}d}ddd	d}d
}||d}t||s"J ||||ks.J ||d}t||s:J ||||ksFJ ||d}t||sRJ ||||ks^J ||d}t||sjJ ||||ksvJ ||d}||d}	||	ksJ ||||	ksJ |||	|ksJ ||d}||d}	||	ksJ ||||	ksJ |||	|ksJ ||d}||d}	||	ksJ ||||	ksJ |||	|ksJ ||d}||d}	||	ksJ ||||	ksJ |||	|ksJ ||d}||d}	||	ks)J ||||	ks6J |||	|ksCJ ||d}||d}	||	ksTJ ||||	ksaJ |||	|ksnJ ||d}||d}	||	ksJ ||||	ksJ |||	|ksJ ||d}||d}	||	ksJ ||||	ksJ |||	|ksJ ||d}||d}	||	ksJ ||||	ksJ |||	|ksJ ||d}||d}	||	ks J ||||	ksJ |||	|ksJ ||d}| }	||	ks)J ||||	ks6J |||	|ksCJ ||d}| }	||	ksRJ ||||	ks_J |||	|kslJ ||d}| }	||	ks{J ||||	ksJ |||	|ksJ ||d}| }	||	ksJ ||||	ksJ |||	|ksJ tt |dd W d    n	1 sw   Y  tt |dddd W d    n	1 sw   Y  tt |dddd W d    n	1 sw   Y  tt |dddd W d    n	1 s.w   Y  tt	j
 |dd W d    n	1 sJw   Y  tt	j
 |ddddd W d    d S 1 skw   Y  d S )Nr   r   r   	localhosti'#  )r   r   r   zhttp://localhost:8999r  i  zhttps://localhost:8080)proxy_options)r   r  #  r  )r   r   )r   r   )r   r   zhttpsB://localhost:9000httpA)r   r   r    r/  r.  r   r	  r  KeyErrorre   rn  )
monkeypatchr0  r   proxy_opts_1_dictproxy_opts_1_strproxy_opts_2_dictproxy_opts_2_strr   r  r   r   r   r   test_s3_proxy_options  s   



























$r  c                  C   s   ddl m}  | ddd}d}tjt|d}|d W d    n1 s%w   Y  d	t|jv r3d S | d
dd}|d d S )Nr   r   z
eu-north-1T)r  r   a  When getting information for bucket 'voltrondata-labs-datasets': AWS Error UNKNOWN \(HTTP status 301\) during HeadBucket operation: No response body. Looks like the configured region is 'eu-north-1' while the bucket is located in 'us-east-2'.|NETWORK_CONNECTIONr  voltrondata-labs-datasetsNETWORK_CONNECTIONr  )r   r   r   r	  r   r;   r   r   )r   r   msgexcr   r   r   test_s3fs_wrong_region  s   r  c                 C   s   ddl m} |dd}t||sJ | | ||ksJ |ddd}t||s+J | | ||ks7J ||ks=J |ddd	d
ddd}t||sNJ | | ||ksZJ ||ks`J tt |  W d    d S 1 stw   Y  d S )Nr   r   zfake-account-name)r   fakeaccountkey)r   r   zfake-accountfakeaccountzfake-blob-authorityzfake-dfs-authorityr  r   )r   r   r    r/  r.  r   r	  r  )r0  r   r  r   fs3r   r   r   test_azurefs_options  s,   
"r  c                 C   s  ddl m} t std | \}}}d}d}d}d}	|||d|||d	}
||	||d|||}||	||d
|||}||	||d
|d ||}|||}|d||}|||dd}|||ddd}|||dt	dd}|||ddd}|||ddddid}|
|ksJ ||ksJ ||ksJ ||ksJ ||ksJ ||ksJ ||ksJ ||ksJ ||ksJ ||ksJ ||ksJ ||ksJ t
t |  W d    n1 sw   Y  t
t |d W d    n1 sw   Y  |
||||||||||fD ]}||||ks J q| \}}}||||d}|tds8J |d|||}|tdsLJ d S )Nr   r   r      i   i   zIhdfs://{}:{}/?user={}&replication={}&buffer_size={}&default_block_size={}libhdfs)r   replicationr{  default_block_sizemerE  zhdfs://{}:{}	localuser)r   
cache_path)r   kerb_ticketcache_path2
hdfs_tokenabcd)r   r  
extra_confrS  rN   zhdfs://{}:{}/?user={})r   r   re   r   r   r   from_urirc   rG  rD  r	  r  r/  r.  r;   r   )r   r0  r   r   r   r   r  r{  r  urihdfs1hdfs2hdfs3hdfs4hdfs5hdfs6hdfs7hdfs8hdfs9hdfs10hdfs11r   r   r   r   r   test_hdfs_options  s   



r  )r  expected_klassexpected_pathzmock:rK  zmock:foo/barr  zmock:/foo/barzmock:///foo/barzmock:///some%20path/%C3%A9u   some path/ézfile:/rN   zfile:///zfile:/foo/barz/foo/barzfile:///foo/barzfile:///some%20path/%C3%A9u   /some path/éu   /some path/%20éc                 C   s,   t | \}}t||sJ ||ksJ d S r   )r   r  r    )r  r  r  r   r-   r   r   r   test_filesystem_from_uri  s   r  r-   c                 C   sB   t | }t|\}} t|tsJ | |   ksJ d S r   )	rG  rD  r   r  r    r   resolveabsoluterz   )r-   r|   r   r   r   r    test_filesystem_from_path_object  s   
r  c           
      C   s   ddl m} | d \}}}}d||||}t|\}}t||s$J |dks*J || ||g\}	|	j|ks=J |	j	t
jksEJ d S )Nr   r   r   zZs3://{}:{}@mybucket/foo/bar?scheme=http&endpoint_override={}:{}&allow_bucket_creation=Truemybucket/foo/bar)r   r   rc   r   r  r    rJ   r;   r-   r  r
   r5   )
r   r   r   r   r   r   r  r   r-   r:   r   r   r   test_filesystem_from_uri_s3)  s   
r  c                 C   s   ddl m} | d \}}dd| d| d d }t|\}}t||s'J |d	ks-J || ||g\}|j|ks@J |jt	j
ksHJ d S )
Nr   r   r   zgs://anonymous@z/mybucket/foo/bar?scheme=http&endpoint_override=r   &z0retry_limit_seconds=5&project_id=test-project-idr  )r   r   r   r  r    rJ   r;   r-   r  r
   r5   )r   r   r   r   r  r   r-   r:   r   r   r   test_filesystem_from_uri_gcs=  s   
r   c                  C   sp   t  } t| }t|tsJ |jdksJ |j| u sJ tt td  W d    d S 1 s1w   Y  d S Nz	py::dummy)r   r   r    r   handlerr   r	  r  )r  r   r   r   r   test_py_filesystemQ  s   
"r  c                  C   s   t d} t d}t d}t| }t| }t|}t|}||us"J ||us(J ||us.J ||ks4J ||ks:J ||ks@J |t ksGJ |t ksNJ d S )NrE  r  )r   r   r   r7   )handler1handler2handler3r  r   r  fs4r   r   r   test_py_filesystem_equality\  s   r  c                 C   s\   t  }t|}| |}| |}t|tsJ ||ksJ |j|ks%J |jdks,J d S r  )r   r   r.  r/  r    r   r  r   )r0  r  r   r1  r2  r   r   r   test_py_filesystem_picklingp  s   

r	  c                  C   s   t  } t| }t|tsJ t| }d } | d usJ d }| d u s%J t  } t| }t| }d } | |ju s;J | d usBJ d }| d u sKJ d S r   )r   r   r    weakrefrefr  )r  r   wrr   r   r   test_py_filesystem_lifetime|  s    

r  c                  C   s  t  } t| }|dg\}|jdksJ |jtjksJ |dg\}|jdks+J |jtjks3J |dg\}|jdksAJ |jtjksIJ t	
t |dg W d    n1 s_w   Y  t	
t |dg W d    d S 1 s{w   Y  d S )Nzsome/dirz	some/filer1   r2   r  )r   r   r;   r-   r  r
   r5   r4   r6   r   r	  r  r8   )r  r   r:   r   r   r    test_py_filesystem_get_file_info  s"   "r  c                  C   s^  t  } t| }tdd}||}t|dksJ |d jdks"J |d jtjks,J |d j	dks5J |d jdks>J |d jtj
ksHJ |d j	d u sQJ tdd	d
}||}t|dksdJ |d jdksmJ |d jdksvJ |d jdksJ tdd}tt || W d    n1 sw   Y  tdd	d}||g ksJ d S )Nr<   )rB   r  r   r=   r>   rE  r@   T)rB   rE   rS  rA   r1   )rB   rC   )r   r   r   r;   rT  r-   r  r
   r4   r?   r5   r   r	  rD   )r  r   rF   rG   r   r   r   )test_py_filesystem_get_file_info_selector  s.   



r  c               	   C   s   t  } t| }|jddd |jddd tt |d W d    n1 s*w   Y  |d |d d	D ]#}tt || W d    n1 sRw   Y  |j|dd
 q;|	d |
dd |dd d S )NrE   TrR  rI   FfoobarrL   rO   )rK  rN   rk  rl  rT   rV   rW   r]   r^   )r   r   rJ   r   r	  r8   rL   rO   rQ   rT   r[   r_   )r  r   r-   r   r   r   test_py_filesystem_ops  s"   


r  c                  C   ~   t t } | d}| dksJ W d    n1 sw   Y  tt | d W d    d S 1 s8w   Y  d S )Nsomefiles   somefile:input_streamr1   )r   r   ri   r  r   r	  rD   r   rg  r   r   r   test_py_open_input_stream     
"r  c                  C   r  )Nr  s   somefile:input_filer1   )r   r   rk   r  r   r	  rD   r  r   r   r   test_py_open_input_file  r  r  c                  C   D   t t } | d}|d W d    d S 1 sw   Y  d S Nr  re  )r   r   rr   r8  r  r   r   r   test_py_open_output_stream     
"r  c                  C   r  r  )r   r   rt   r8  r  r   r   r   test_py_open_append_stream  r  r  c                  C   s   ddl m}  tjdpd}| dd}|j|ksJ | ddd}|td	}t|dks/J d
}|	|#}|
 }d|v sAJ |d dksIJ |d dksQJ W d    d S 1 s\w   Y  d S )Nr   r   PYARROW_TEST_S3_REGIONz	us-east-1Tr  r  )r   r  z"voltrondata-labs-datasets/nyc-taxizCvoltrondata-labs-datasets/nyc-taxi/year=2019/month=6/part-0.parquetr  zLast-Modifieds   2022-07-12T23:32:00ZETags%   "4c6a76826a695c6ac61592bc30cda3df-16")r   r   osenvirongetr  r;   r   rT  ri   rq   )r   default_regionr   entriesr   rg  mdr   r   r   test_s3_real_aws  s$   
"r%  c                  C   s   t d\} }| jdksJ | |}d|dv sJ W d    n1 s'w   Y  t d\} }| jdks:J tjtdd t d	 W d    n1 sQw   Y  t d
\} }| jdksdJ d S )Nzs3://mf-nwp-models/README.txt	eu-west-1s&   Meteo-France Atmospheric models on AWS2   z.s3://mf-nwp-models/README.txt?region=us-east-2r  zBucket '.*' not foundr  zs3://x-arrow-nonexistent-bucketz0s3://x-arrow-nonexistent-bucket?region=us-east-3z	us-east-3)r   r  r  ri   r  r   r	  r8   )r   r-   rg  r   r   r   !test_s3_real_aws_region_selection  s    r(  c                  C   s   ddl m}  | ddksJ | ddksJ tjtdd | d	 W d    n1 s,w   Y  tjtdd | d
 W d    d S 1 sHw   Y  d S )Nr   resolve_s3_regionr  r  zmf-nwp-modelsr&  zNot a valid bucket namer  r  z	s3:bucket)r   r*  r   r	  rQ   r)  r   r   r   test_resolve_s3_region(  s   

"r+  c              
   C   s0  |d }|d }|d}| |}|d W d    n1 s!w   Y  | \}}}	}
d|	 d|
 d| d| d| 
}t|d	 }t|| t }||}| dksYJ W d    n1 scw   Y  t|d
 }t|||d ||}| dksJ W d    n1 sw   Y  t|d }t|}t|| ||}| dksJ W d    n1 sw   Y  t|d }t|||d ||}| dksJ W d    n1 sw   Y  t|d }t||ddd ||}| dksJ W d    d S 1 sw   Y  d S )Nr   r   r6  r7  zs3://r   @z?scheme=http&endpoint_override=zc_copied1.txtzc_copied2.txt)source_filesystemzc_copied3.txtzc_copied4.txt)destination_filesystemzc_copied5.txtrE  F
chunk_sizeuse_threads)rr   r8  r   r   r   ri   r  r   )s3_connectionr   r~   r   r   r-   rg  r   r   r   r   
source_urilocal_path1r   local_path2local_path3destination_urilocal_path4local_path5r   r   r   test_copy_files4  sP   

$r:  c                    s|  t   | d }|   t|d }|d W d    n1 s$w   Y   t|d }|d W d    n1 sBw   Y   fdd}| d }|  tt|t| || | d	 }|  tt|t|  d
 || | d }|  tt|}tt|}t|| || | d }	|	  t||	 ||	 | d }
|
  t||
ddd ||
 d S )Nsourcefile1   test1file2   test2c                    s     t| d }| dksJ W d    n1 sw   Y    t| d }| dks3J W d    d S 1 s>w   Y  d S )Nr<  r=  r>  r?  )ri   r   r  )destination_dirrg  r   r   r   check_copied_filest  s   "z5test_copy_files_directory.<locals>.check_copied_filesdestination1destination2)r-  r.  destination3destination4destination5rE  Fr/  )r   mkdirrr   r   r8  r   r   )r~   
source_dirrg  rB  destination_dir1destination_dir2destination_dir3r3  r7  destination_dir4destination_dir5r   rA  r   test_copy_files_directoryi  sD   

rO  c                  C      d} t tjd| g d S )Na  if 1:
        import pytest
        from pyarrow.fs import (FileSystem, S3FileSystem,
                                ensure_s3_initialized, finalize_s3)

        fs, path = FileSystem.from_uri('s3://mf-nwp-models/README.txt')
        assert fs.region == 'eu-west-1'
        f = fs.open_input_stream(path)
        f.read(50)

        finalize_s3()

        with pytest.raises(ValueError, match="S3 .* finalized"):
            f.read(50)
        with pytest.raises(ValueError, match="S3 .* finalized"):
            fs.open_input_stream(path)
        with pytest.raises(ValueError, match="S3 .* finalized"):
            S3FileSystem(anonymous=True)
        with pytest.raises(ValueError, match="S3 .* finalized"):
            FileSystem.from_uri('s3://mf-nwp-models/README.txt')
        -c
subprocess
check_callsys
executablecoder   r   r   test_s3_finalize  s   rY  c                  C   rP  )Na  if 1:
        import pytest
        from pyarrow.fs import resolve_s3_region, ensure_s3_initialized, finalize_s3

        resolve_s3_region('mf-nwp-models')

        finalize_s3()

        # Testing both cached and uncached accesses
        with pytest.raises(ValueError, match="S3 .* finalized"):
            resolve_s3_region('mf-nwp-models')
        with pytest.raises(ValueError, match="S3 .* finalized"):
            resolve_s3_region('voltrondata-labs-datasets')
        rQ  rR  rW  r   r   r    test_s3_finalize_region_resolver  s   rZ  c                  C   rP  )Na  if 1:
        import threading
        import pytest
        from pyarrow.fs import (FileSystem, S3FileSystem,
                                ensure_s3_initialized, finalize_s3)
        threads = []
        fn = lambda: FileSystem.from_uri('s3://mf-nwp-models/README.txt')
        for i in range(4):
            thread = threading.Thread(target = fn)
            threads.append(thread)
            thread.start()

        for thread in threads:
            thread.join()

        finalize_s3()
        rQ  rR  rW  r   r   r   test_concurrent_s3fs_init  s   r[  c               
   C   sN  z	t ddg W n ty   td Y nw t } dddd|  dtjt	dg}t j
|t jd	}zdd
|  d}t }d }t |d k rzt|}|jdksUJ W d    n1 s_w   Y  W n, ty~ } z|}td W Y d }~nd }~ww t |d k sHtd|d|  W |  n|  w | dksJ d S )Nuwsgiz	--versionz"uwsgi not installed on this Pythonz-iz--httpz
127.0.0.1:z--wsgi-filezwsgi_examples.py)stdinzhttp://127.0.0.1:z/s3/rQ     g?zCould not fetch r      )rS  rT  rD   r   r   r   r  r-   joinherePopenDEVNULLtimer   statusr   sleepfail	terminatewait)r   argsprocurl
start_timeerrorrespr   r   r   r   test_uwsgi_integration  s<   
	rp  )r   r   r   r~  r  rG  urllib.requestr   rS  rU  rd  r   r
  r   re   pyarrow.tests.test_ior   pyarrow.tests.utilr   r   r	   r   r
   r   r   r   r   r   r   r   r   r   r   pyarrow.utilr   r-   dirnameabspathr  ra  r   fixturer   r   r   r   r   r   r   r   r   r  r   r   r   r   r   r   markr   r   r   r   r   r   r   r   r   r   r   r   r  r  r  r  r!  r"  r(  r3  r@  rB  rC  rI  rP  r\  r^  rc  ri  rj  ro  rp  rs  ru  rv  rw  ry  parametrizecompressr  r  r  
decompressr  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r  r  r	  r  r  r  r  r  r  r  r  r%  r(  r+  r:  rO  rY  rZ  	processes	threadingr[  rp  r   r   r   r   <module>   s  4i
	
	
	
	
	
	









;





;8 

	

	

	

%
\
 


G



		



44

