o h@sddlZddlmZddlmZddlZddlmZddlZddl m Z ddl Z ddl Z ddl Z ddlZzddlZWn eyEdZYnwddlZddlmZmZmZiaiaiaddZd d Zd d Zd dZddZddZ dQddZ!ddZ"ddZ#ddZ$ddZ%ddZ&dd Z'd!d"Z(d#d$Z)d%d&Z*d'd(Z+dQd)d*Z,  ,dRd-d.Z-d/d0Z.dSd1d2Z/d3d4Z0 dTd6d7Z1hd8Z2d9d:Z3d;d<Z4d=d>Z5dQd?d@Z6 dQdAdBZ7dCdDZ8dEdFZ9dGdHZ:dIdJZ;dKdLZdS)UN)Sequence)futures)deepcopy) zip_longest) _pandas_api frombytesis_threading_enabledc Cstsjtitjjdtjjdtjjdtjjdtjjdtjj dtjj dtjj dtjj d tjj d tjjd tjjd tjjd tjjdtjjdtjjdtjjdtjjdtjjdtjjditS)Nemptyboolint8int16int32int64uint8uint16uint32uint64float16float32float64datetimebytesunicode)_logical_type_mapupdatepalibType_NA Type_BOOL Type_INT8 Type_INT16 Type_INT32 Type_INT64 Type_UINT8 Type_UINT16 Type_UINT32 Type_UINT64Type_HALF_FLOAT Type_FLOAT Type_DOUBLE Type_DATE32 Type_DATE64 Type_TIME32 Type_TIME64 Type_BINARYType_FIXED_SIZE_BINARY Type_STRINGr2r2I/var/www/vscode/kcb/lib/python3.10/site-packages/pyarrow/pandas_compat.pyget_logical_type_map.sR     r4cCst}z||jWStyLt|tjjrYdSt|tjjr+dt |j YSt|tjj r?|j dur;dYSdYSt|tjj rIYdSYdSw)N categoricalzlist[{}] datetimetzdatetimedecimalobject)r4idKeyError isinstancerrDictionaryTypeListTypeformatget_logical_type value_type TimestampTypetzDecimal128Type) arrow_typelogical_type_mapr2r2r3r@Ks  r@cCsdts0ttjdtjdtjdtjdtjdtjdtj dtj dtj d tj d tj d d d tjdtjditS)Nr r r r rrrrrrr datetime64[D]rstringr)_numpy_logical_type_maprnpbool_r r r rrrrrrrstr_bytes_r2r2r2r3get_numpy_logical_type_map\s$rNcCstt}z||jjWSty9t|jdrYdSt|jdr)t|jYSt|}|dkr5YdS|YSw)NrCr6 datetime64rHr) rNdtypetyper;hasattrstr startswithr infer_dtype)pandas_collectionnumpy_logical_type_mapresultr2r2r3get_logical_type_from_numpyrs   rYcCs|j}t|dkr(t|d|}|dusJt|j|jd}t|jj}||fSt|drz&construct_metadata..cSsg|] }t|ts|qSr2)r<dict)rzdescrr2r2r3r{ )rmrErnFrz&The DataFrame has non-str index name `z@` which will be converted to string and not roundtrip correctly. stacklevellevelsnamesspandaspyarrow)libraryversion) index_columnscolumn_indexescolumnscreatorpandas_versionutf8)raziprxappendr<r}rmrS_column_name_to_stringswarningswarn UserWarningr`r_get_simple_index_descriptorjsondumpsr __version__rrencode)columns_to_convertdf column_names index_levelsindex_descriptorspreserve_indextypescolumn_field_namesnum_serialized_index_levelsntypesdf_types index_typescolumn_metadatacolrmrnrErgindex_column_metadatanon_str_index_nameslevel descriptorrrrr2r2r3construct_metadatasp          rcCsTt|\}}t|}d|vrtjdtdd|dkr"|rJddi}|||||dS) NmixedzlThe DataFrame has column names of mixed type. They will be converted to strings and not roundtrip correctly.rrrencodingUTF-8rl)rirYrrr)rrmrvrwror2r2r3r%s rcCsxt|tr|St|tr|dSt|trtttt|St|tr(td|dus6t|t r8t |r8|St|S)a!Convert a column name (or level) to either a string or a recursive collection of strings. Parameters ---------- name : str or tuple Returns ------- value : str or tuple Examples -------- >>> name = 'foo' >>> _column_name_to_strings(name) 'foo' >>> name = ('foo', 'bar') >>> _column_name_to_strings(name) "('foo', 'bar')" >>> import pandas as pd >>> name = (1, pd.Timestamp('2017-02-01 00:00:00')) >>> _column_name_to_strings(name) "('1', '2017-02-01 00:00:00')" rz%Unsupported type for MultiIndex levelN) r<rSrdecodetuplemaprrrsrqrJrrrmr2r2r3r9s     rcCs(|jdur|j|vrt|jSd|S)zReturn the name of an index level or a default name if `index.name` is None or is already a column name. Parameters ---------- index : pandas.Index i : int Returns ------- name : str Nz__index_level_{:d}__)rmrr?)indexirr2r2r3_index_level_name`s  rcCsNt|||}|jjstdt|j|durt|||Sg}g}|dur+t|jng}g}g}|D],} || } t | } t | rIt d| | | | d| | | t| q3g} g} t|D]0\} }t|| |} t|t jjr|durt|}n| || d| }| | | |qh|| }|||| | |||fS)Nz Duplicate column names found: {}F-Sparse pandas data (column {}) not supported.)_resolve_columns_of_interestr is_unique ValueErrorr?list$_get_columns_to_convert_given_schema_get_index_level_valuesrrr is_sparsersrrS enumeraterr<pd RangeIndex_get_range_index_descriptor)rschemarrrrrrconvert_fieldsrmrrindex_column_namesr index_levelr~ all_namesr2r2r3_get_columns_to_convertssR             rc Cs4g}g}g}g}g}g}|jD]|} z|| } d} Wn@tyYzt|| } Wnttfy7td| w|durCtd| |durUt| tjj rUtd| d} Ynwt | rft d| | | } | | | | | | | r| | | | | | q||} | |||||||fS)z Specialized version of _get_columns_to_convert in case a Schema is specified. In that case, the Schema is used as the single point of truth for the table structure (types, which columns are included, order of columns, ...). FzNname '{}' present in the specified schema is not found in the columns or indexzlname '{}' present in the specified schema corresponds to the index, but 'preserve_index=False' was specifiedNaname '{}' is present in the schema, but it is a RangeIndex which will not be converted as a column in the Table, but saved as metadata-only not in columns. Specify 'preserve_index=True' to force it being added as a column, or remove it from the specified schemaTr)rr;_get_index_level IndexErrorr?rr<rrrrrsfieldr)rrrrrrrrrrmris_indexrrr2r2r3rsd           rcCs8|}||jjvrt|rt|tdd}|j|S)z_ Get the index level of a DataFrame given 'name' (column name in an arrow Schema). __index_level_)rr_is_generated_index_nameintraget_level_values)rrmkeyr2r2r3rs rcCs,zt||WStyt|YSwN)rrrsrSrr2r2r3 _level_names    rcCs.dt|jt|dt|dt|ddS)Nrangestartstopstep)kindrmrrr)rrmrget_rangeindex_attribute)rr2r2r3r s    rcs(ttdg}fddt|DS)Nrcsg|]}|qSr2)rrzrrr2r3r{z+_get_index_level_values..)rar`r)rnr2rr3rsrcsR|dur |dur td|dur|j}|S|dur$fdd|D}|Sj}|S)NzJSchema and columns arguments are mutually exclusive, pass only one of themcsg|] }|jvr|qSr2)rrzcrr2r3r{ sz0_resolve_columns_of_interest..)rrr)rrrr2rr3rsrc Cst|d||\}}}}}}} }g} | D]T} | j} t| r&tj| ddj} n       rTc" sDt||||\}}}} } } } } |dur.t|t|j}}||dkr,|dkr,t}nd}ts3d}fdddd}|dkrNfddt| | D}nNg}t|*}t| | D]\}}||j rm| ||q[| | ||q[Wdn1swYt |D]\}}t |tjr|||<qd d|D}|durg}t||D] \}}| t||qt|}t| ||| | |||d }|jrt|jnt}||||}d}t|d krz'| d d }|d kr| d d}| d d} | d d}!tt|| |!}Wn tyYnw|||fS)Ndrc s|dur d}d}n|j}|j}z tj||dd}Wn"tjtjtjfy<}z|jd|j |j f7_|d}~ww|sO|j dkrOt dt ||j |S)NT)rQrsafez0Conversion failed for column {!s} with type {!s}rz>Field {} was non-nullable but pandas column had {} null values)nullablerQrr ArrowInvalidArrowNotImplementedErrorArrowTypeErrorargsr?rmrP null_countrrS)rrfield_nullablerrXe)rr2r3convert_columncs0  z+dataframe_to_arrays..convert_columncSs$t|tjo|jjot|jjtjSr) r<rJndarrayflags contiguous issubclassrPrQinteger)arrr2r2r3_can_definitely_zero_copyys z6dataframe_to_arrays.._can_definitely_zero_copycsg|] \}}||qSr2r2)rzrf)rr2r3r{sz'dataframe_to_arrays..cSsg|]}|jqSr2rQ)rzxr2r2r3r{srrrrrrr)rrarr cpu_countrrrThreadPoolExecutorrrsubmitrr<FuturerXrrrrgrr}r with_metadatarr)"rrrnthreadsrrrrrrrrrrnrowsncolsrarraysexecutorrrr maybe_futrfieldsrmrpandas_metadatargn_rowsrrrrr2)rrr3dataframe_to_arraysJs~                  rcCsd|jjtjkr ||fSt|r$|dur$|j}|j}t ||}||fS|dur.t |j}||fSr) rPrQrJrOr is_datetimetzrCunitr timestampfrom_numpy_dtype)rrPrrCrr2r2r3rs  rcCs$ddlmm}|dd}|d}d|vr%tjj||d|dd}n`d|vrZt|j \}} t ||d} t rItj j |d | d d }n<|}|rY|j|||j| d } | Sn+d |vr|d }t|dksjJ||d} || } t| ds}td| |}n|}|r|j||dS||fS)a Construct a pandas Block from the `item` dictionary coming from pyarrow's serialization or returned by arrow::python::ConvertTableToPandas. This function takes care of converting dictionary types to pandas categorical, Timestamp-with-timezones to the proper pandas Block, and conversion to pandas ExtensionBlock Parameters ---------- item : dict For basic types, this is a dictionary in the form of {'block': np.ndarray of values, 'placement': pandas block placement}. Additional keys are present for other types (dictionary, timezone, object). columns : Column names of the table being constructed, used for extension types extension_columns : dict Dictionary of {column_name: pandas_dtype} that includes all columns and corresponding dtypes that will be converted to a pandas ExtensionBlock. Returns ------- pandas Block rNblock placement dictionaryr])rbr]r^rF)rPcopy)rklassrPpy_arrayr__from_arrow__zGThis column does not support to be converted to a pandas ExtensionArray)r)pandas.core.internalscore internalsgetrcategorical_type from_codesrJ datetime_datarPmake_datetimetz is_ge_v21rrview make_blockDatetimeTZBlockrarRrr)itemrextension_columns return_block_int block_arrrrrrrPrrm pandas_dtyper2r2r3_reconstruct_blocksF     r0cCs&trd}tj|}tj||dS)NnsrC)ris_v1rrstring_to_tzinfodatetimetz_type)rrCr2r2r3r%s r%FcsVg}g}|jj}|s3|dur3|d}|dg}|d}t||}t||||\}} t|||||ntj|j } t|g|||t |t |||} |j t j|||t} tryddlm} fdd| D} | | | | d}|Sdd lm}dd lm}fd d| D} | | g}|| |}tr|||j}|S||}|S) Nrrrr)create_dataframe_from_blockscsg|] }t|ddqS)F)r,r0rzr*rext_columns_dtypesr2r3r{-s z&table_to_dataframe..)rr) BlockManager) DataFramecsg|]}t|qSr2r7r8r9r2r3r{8s )rrr!_add_any_metadata_reconstruct_index_get_extension_dtypesrrrnum_rows'_check_data_column_metadata_consistency_deserialize_column_indexrrrtable_to_blocksrkeysis_ge_v3pandas.api.internalsr6rr;pandasr<r& _from_mgraxes)optionstablerbignore_metadata types_mapper all_columnsrrrrrrXr6blocksrr;r<rImgrr2r9r3table_to_dataframesV              rQ> r r r r rrr9rrrrrrc Cs|d}|pg}i}tjdur|S|r)|jD]}|j}||} | dur(| ||j<q|jD]%}|j}|j|vrQt|tjrQz|} Wn t yKYq,w| ||j<q,|D]]} z| d} Wn t yi| d} Ynw| d} | |vr| t vrt | } t| tjrt| tj jr|s| |vrqTztj|j| jrWqTWn t yYnwt| dr| || <qTtr|s|jD],}|j|vrtj|jstj|jstj|jr|j|vrtj jtjd||j<q|S)a Based on the stored column pandas metadata and the extension types in the arrow schema, infer which columns should be converted to a pandas extension dtype. The 'numpy_type' field in the column metadata stores the string representation of the original pandas dtype (and, despite its name, not the 'pandas_type' field). Based on this string representation, a pandas/numpy dtype is constructed and then we can check if this dtype supports conversion from arrow. strings_to_categoricalNrnrmrpr)na_value)rextension_dtyperrQrmr<rBaseExtensionTypeto_pandas_dtypeNotImplementedErrorr;_pandas_supported_numpy_typesr/r StringDtyper is_dictionaryrrRuses_string_dtype is_stringis_large_stringis_string_viewrJnan) rKcolumns_metadatarMrJrbrR ext_columnsrtypr/col_metarmrPr2r2r3r?Ost                       r?cCstdd|Ds JdS)Ncss0|]}|ddurd|vp|dduVqdS)rmNrnr2rr2r2r3 s  z:_check_data_column_metadata_consistency..)all)rNr2r2r3rAsrAcs|rdd|Dfdd|jD}n|j}t|dkr2tjjjtttj |dd|Dd}ntjj ||r>|dd ndd }t|dkrMt ||}|S) NcSs&i|]}|dt|d|dqSrnrm)r!rrr2r2r3 sz-_deserialize_column_index..csg|]}||qSr2r!rycolumns_name_dictr2r3r{s z-_deserialize_column_index..rcSsg|]}|dqSrr2)rz col_indexr2r2r3r{r|rrrmr) rrarr MultiIndex from_tuplesrrast literal_evalIndex"_reconstruct_columns_from_metadata) block_tablerNrcolumns_valuesrr2rir3rBs&     rBc Cs6dd|D}g}g}|}|D]O}t|tr't|||||\}} } | dur&qn-|ddkrK|d} tjj|d|d|d| d } t| t|krJqn td |d| | | | qtj} t|d krt| j j ||d } || fSt|d kr|d } t| | j s| j | |d d} || fS| |j } || fS)NcSsi|] }|d|d|qSrfrhrr2r2r3rgsz&_reconstruct_index..rrrmrrr)rrmzUnrecognized index kind: {}rrlrr)r<rS_extract_index_levelrrrrarr?rrm from_arraysrqr@) rKrrNrMfield_name_to_metadata index_arrays index_names result_tabler~r index_namerrr2r2r3r>sP            r>c Csl||d}t||}|j|}|dkr|ddfS||}|j|d} d| _||j|}|| |fS)Nrm)rM) _backwards_compatible_index_namerget_field_indexre to_pandasrm remove_column) rKrzrnrwrM logical_namer{rrrr2r2r3rus        rucCs||kr t|r dS|S)a1Compute the name of an index column that is compatible with older versions of :mod:`pyarrow`. Parameters ---------- raw_name : str logical_name : str Returns ------- result : str Notes ----- * Part of :func:`~pyarrow.pandas_compat.table_to_blockmanager` N)r)raw_namerr2r2r3r}sr}cCsd}t||duS)Nz^__index_level_\d+__$)rematch)rmpatternr2r2r3r/src Cs2tstddddtjdtjtjtjtjd tS)NrGr_rS) rr7r6rrrHrfloatingr8r )_pandas_logical_type_maprrJrMrrobject_r2r2r2r3get_pandas_logical_type_map4s rcCs@t}z||WStyd|vrtjYSt|YSw)a Get the numpy dtype that corresponds to a pandas type. Parameters ---------- pandas_type : str The result of a call to pandas.lib.infer_dtype. Returns ------- dtype : np.dtype The dtype that corresponds to `pandas_type`. r)rr;rJrrP)ropandas_logical_type_mapr2r2r3_pandas_type_to_numpy_typeGs    rc Cstj}t|ddp |g}t|ddpdg}ddt||idD}g}tdd}|D]\}} } t| } | tjkr>| |}| d krgt j |d d d } |j |d d| }trf|t| d }n1| dkrwtjdd|D}n!|jdkr| dkrd| vs| dvr||q+|j| kr|| }|j| kr| d kr|| }||q+t|dkr|j|||jdS|j|d |d j|jdS)a_Construct a pandas MultiIndex from `columns` and column index metadata in `column_indexes`. Parameters ---------- columns : List[pd.Index] The columns coming from a pyarrow.Table column_indexes : List[Dict[str, str]] The column index metadata deserialized from the JSON schema metadata in a :class:`~pyarrow.Table`. Returns ------- result : MultiIndex The index reconstructed using `column_indexes` metadata with levels of the correct type. Notes ----- * Part of :func:`~pyarrow.pandas_compat.table_to_blockmanager` rNrccSs0g|]\}}||dt|j|ddfqS)rorpN)r!rSrP)rzrrkr2r2r3r{{s  z6_reconstruct_columns_from_metadata..) fillvaluerrr6rrgr^T)utcr8cSsg|]}t|qSr2)r8Decimalrr2r2r3r{rrSr9r)rrHrrl)rPrm)rrr`roperator methodcallerrrJrMrrrr4 to_datetime tz_convertrEas_unitr$rqrPrastyperarmrrm) rrrrlabels levels_dtypes new_levelsencoderrr/ numpy_dtyperPrCr2r2r3rr^sH         rrcCsi}i}|j}|d}dd|D}t|}t|d|}t|dD]q\}} | d} | sC| d} ||kr=|||} | durCd} || } | dkr| d d kr|| } t| jtjj s_q$| d } | sfq$| d }|r|| jj kr| }tj d |d}tj j||d}t|| j||| <||| <q$t|dkrg}g}tt|jD]$}||vr||||||q|||||j|qtjj|t|dS|S)NrcSsg|] }t|tr|qSr2)r<rS)rzidx_colr2r2r3r{rz%_add_any_metadata..rrnrmNoner|ror6rgr^r1r2rr)r)rrarr!r~r<rQrrrBrCrrArrayrrrmrrTablerv)rKrmodified_columnsmodified_fieldsrrn_index_levels n_columnsrrcridxrrg metadata_tz converted tz_aware_typerrrr2r2r3r=s^       r=cCs$tj|}|jdj|}|S)zB Make a datetime64 Series timezone-aware for the given tz r)rrr4dt tz_localizer)seriesrCr2r2r3 make_tz_awares  rr)rNT)NNT)NFN)?rocollections.abcr concurrentrconcurrent.futures.threadrrr8 itertoolsrrrrrnumpyrJ ImportErrorrr pyarrow.librrrrrIrr4r@rNrYrirxrrrrrrrrrrrrrrr0r%rQrXr?rArBr>rur}rrrrrr=rr2r2r2r3s|      2 _'B>    # e E 9S !7 W A