o 0 hR}&@s" UddlZddlZddlZddlTddlmZddlZddlmmZ ddl m Z ddl m Z ddlmZgZeeed<iZeeefed <d,d d Z d-ddZd.ddZdeddfddZdededededdf ddZddZddZd d!ZeeeZ dee!fd"d#Z"d$d%Z#d&d'Z$d(d)Z%d*d+Z&e ej'j(j)j*ej'j(j+j*ej'j(j,j*ej'j(j-j*ej'j(j.j*ej'j(j/j*ej'j(j0j*gd,d-d.Z1e ej'j2j3j*d,d/d0Z4e ej'j(j5j*gd,d1d2Z6e ej'j(j7j*d,d3d4Z8e ej'j(j7j9d5e8e ej'j(j:j*d6d7d8Z;e ej'j(jj*d<d=d>Z?e ej'j(j@jAd?d@dAZBe ej'j(jCj*d,dBdCZDe ej'j(jEj*dDdEdFZFe ej'j(jGj*d,e$e ej'j(jHj*ej'j(jIj*ej'j(jJj*ej'j(jKj*ej'j(jLj*gd,dGdHZMe ej'j(jNj*dIeMe ej'j(jOj*dJeMe ej'j(jOjPdKeMe ej'j(jQj*d,dLdMZRe ej'j(jSj*dNdOdPZTe ej'j(jUj*dQdRdSZVe ej'j(jWj*dTdUdVZXe ej'j(jYj*dWdXdYZZe ej'j(j[j\dZd[d\Z]e ej'j(j[j*d]d^d_Z^e ej'j(j_j`d`dadbZae ej'j(jbj*dcdddeZce ej'j(jdj*dfdgdhZde ej'j(jej*didjdkZfe ej'j(jgjhdldmdnZie ej'j(jjj,dodpdqZke ej'j(jlj*drdsdtZme ej'j(jnj*dudvdwZoe ej'j(jpj*dxdydzZqe ej'j(jrj*d{d|d}Zse ej'j(jtj*d~ddZue ej'j(jvj*dddZwe ej'j(jxj*dddZxe ej'j(jyj*dddZye ej'j(jzj{dddZ|e ej'j(j}j*dddZ~e ej'j(jj*dddZe ej'j(jj*dddZddZe ej'j(jj*d]ddZe ej'j(jjdddZe ej'j(jjhdddZe ej'j(jj*dddZe ej'j(jj*ej'j(jj*gdddZe ej'j(jj*dddZe ej'j(jj*dddZe ej'j(jjhdddZe ej'j(jj`dddZe ej'j(jj*de ej'j(jj*dddZe ej'j(jj*dddZe ej'j(jj,dddZe ej'j(jj*d]ddZe ej'j(jjdddZe ej'j(jj,dddZe ej'j(jjdddZe ej'j(jj,dddÄZe ej'j(jj*ej'j(jj*ej'j(jj*ej'j(jj*gd,ddńZe ej'j(jj,dddDŽZe ej'j(jj,dddɄZe ej'j(jj*ddd˄Ze ej'j(jj*ddd̈́Ze ej'j(jj*dddτZe ej'j(jj*dddфZe ej'j(jj*d҃ddԄZe ej'j(jj*duddքZe ej'j(jj*d׃ddلZe ej'j(jj*dڃdd܄Ze ej'j(jj*ej'j(jj*gd,ddބZe ej'j(jj*d,ddZe ej'j(jj*dddZe ej'j(jj*dddZe ej'j(jj*dddZe ej'j(jj*d,ddZe ej'j(jj*d,ddZe ej'j(jj*d,ddZe ej'j(jj*d,ddZe ej'j(jj*d,ddZe ej'j(jj*dddZe ej'j(jj*dddZe ej'j(jj*dddZe ej'j(jj*ej'j(jj*ej'j(jj*gdddZe ej'j(jj*ej'j(jj*ej'j(jj*ej'j(jj*ej'j(jj*ej'j(jj*gdddZe ej'j(jjېdddZe ej'j(jjېdddZe ej'j(jj`d,dd Ze ej'j(jj*d d d Zdd lmZmZddlmZee  d/dej`dej`dej`de!dededeeefdededeej`ej`ffddZee  d/dej`dej`dej`dej`dej`dej`dej`dee!efd edededeeefdededeej`ej`ej`eeej`d!fffd"d#Ze ej'j(jj*d$d%d&Zejd'd($Z(e(d&ed)e(d&ed*e(d&ed+WddS1swYdS(0N)*)Optional)normalize_function)#jagged_scaled_dot_product_attention NestedTensor__all__JAGGED_OPS_TABLEFcsddlm}t|ttfr$t|fdd|D}t|t|S|r+||}|dkr3|ks5J|dkr=dS|dS)Nrcanonicalize_dimsc3|] }t|VqdSN_outer_to_inner_dim.0dndim ragged_dimN/var/www/vscode/kcb/lib/python3.10/site-packages/torch/nested/_internal/ops.py z&_outer_to_inner_dim..r)torch._prims_commonr isinstancetuplelisttypedictfromkeys)rdimr canonicalizer outputrrrrs  rTc Csvddlm}|||}||kr|st|d|dkr%|s%t|d|r-t|||n|} |r9|dk} | | fS| S)Nrr z0(): not supported for NestedTensor on ragged dimz+(): not supported for NestedTensor on dim=0)rr RuntimeErrorr) rr"rop_nameconvert_to_inner_dimallow_ragged_dimallow_batch_dimr wrappedretoperating_on_batchrrr_wrap_jagged_dim's   r-c sddlmt|ttfsJdt|fdd|D}d|v}|v}tfdd|D}ttfdd|D}||||fS) z For NestedTensor operators, wraps dimensions to non-negative values, and returns metadata related to reduction dimension(s). rr z<_wrap_jagged_dims(): cannot iterate over dimensions of type csg|]}|qSrrr)r rrr Ps z%_wrap_jagged_dims..c3s |] }|dko |kVqdS)rNrr) ragged_idxrrrVz$_wrap_jagged_dims..c3r rrr)rr/rrrZr) rr rrrranyr r!) rdimsr&r/ wrapped_dimsoperate_on_batchoperate_on_raggedoperate_on_non_batchouter_to_inner_dimr)r rr/r_wrap_jagged_dimsDs    r8 schema_strreturnc s|d}dd|Dd}t||}|ddkr"|dd}n't||kr0t|t|ksItd|jd|d |d t|d t|d d dddddddd}t|D]k\}} | d\} } | d} | rt| ddn| } | |vrtd| |t|kr| std|jd|d| q\|| | ffdd }|||sdddddd}td|jd|d| d || q\dS)!N, cSsg|]}|dqS)?)endswithrxrrrr.bz check_schema..T... NestedTensor (z): expected at least z arguments and at most z arguments, but got: z argumentscSst|tjo t|t Sr)rtorchTensorrr?rrrqzcheck_schema..cSst|to|jduo|jdkSNr)rr_lengths _ragged_idxrGrrrrHrs  cSs t|tSrrrrGrrrrHuscSdS)NTrrGrrrrHx)tjtjt_allr1z: r<zUnknown arg type: z) missing required argument: cs|r |dup |S|Srr)r? is_optional _check_fnrrcheck_fnszcheck_schema..check_fntensorzoptional tensorz%contiguous jagged layout NestedTensorzjagged layout NestedTensorz )rPzt?rQrRr1z ): expected z to be a ) splitcountlen ValueError__name__ enumerater=keysAssertionError)r9funcargskwargsnamed_arg_typesnum_optional_argsmin_argsarg_type_check_fnsinamed_arg_typenamearg_typerSnormalized_arg_typerV type_to_descrrTr check_schema`s`        rmaa_namebb_namecCs:|j|j|j|jkrtd|jd|d|ddS)NrCz : expected  and z' to have the same exact offsets tensor.)_sizerLr%r\)r`rnrorprqrrrcheck_ragged_dim_sames rtcCsL|jd}|jd|}|d|}t|t|ko%tddt||DS)Nrcss$|] \}}||kp |dkVqdS)rANr)rnssrrrr"z%raggedness_matches..)rLrsrZallzip)ntsizeend nt_ragged size_raggedrrrraggedness_matchess  rcCsB|dkr|jddkr|d}|dkr|jddks |SNrr)r"shapesqueeze)rPrrrsqueeze_leading_oness rcs4ttsgttsgfdd}|S)Ncs2D]}fdd}D]}||||<q qS)Ncsfdd}|S)Ncs.tg|Ri|g|Ri|Sr)rm)rarb)aten_opr`r9rrinnersz@register_func..wrapper..get_inner..innerr)rr)r`r9)rr get_innersz1register_func..wrapper..get_innerr)r`rrtableaten_opsr9tablesr`rwrappers zregister_func..wrapper)rr)rrr9rrrr register_funcs  rc Os$t|d}|dur |Stjj|jvrddlm}|D]}||r,td|j d|qt dd|D}|dkrzg}|j j D]}t |jtjrT||jdn ||jd q@|d d |}t||g|Ri|tt|S|d krtd |g|Ri|tt|SdS)Nr is_nested_intrCz: invalid argument css|] }t|tjVqdSr)rrErFr>rrrrrz lookup_jagged..rz: jt_allz: anyrBr;zlhs: any, rhs: any, ...)r getrETag pointwisetags%torch.fx.experimental.symbolic_shapesrr%r\sum_schema argumentsrr TensorTypeappendrijoinrm functoolspartialjagged_unary_pointwisejagged_binary_pointwise) r`rarb dispatch_funcrargnum_tensor_args schema_partscheck_schema_strrrr lookup_jaggeds6       rcCs|||j|jd}|S)Noffsetslengths_metadata_cacherLr)rrbrrrextract_kwargss rcs>tdd|Dt|fdd|Di|fitS)Ncs|] }t|tr|VqdSrrMrrrrrrz)jagged_unary_pointwise..c3s"|] }|ur |jn|VqdSr_valuesrnjtrrrs nextrrr`rarbrrrrs rcsN|d|d}}t|tst|tsJd}t|trMt|trMt||jrAt||j|jg|ddRi|fit|St||j|j|jt|t}|rXt|nt|}|rb||fn||f\}| krtt dt |} | dkr|rj| fn| jf\} } t|| | g|ddRi|fi|S| | kr|j d|j dkrt||j|j |j ddl m} j} j}|jj jd}dur||fddt|j D}| |jj|| |d }|r|fn|f\} } || | g|ddRi|St||j|j |j ) NrrzHcannot call binary pointwise function {} with inputs of shapes {} and {}rz+NYI: broadcasting NT with T with larger dimnested_from_paddedcs"g|] \}}|jkr |nqSrrL)rrgr?rz padded_max_Srrr.^"z+jagged_binary_pointwise..rr/sum_S min_seqlen max_seqlen)rrrrsrrr%formatr\r"NotImplementedErrorrr nested_tensorr_maybe_min_seqlen_maybe_max_seqlenrLexpandr]_offsets)r`rarbrnrpmismatch_error_msga_is_ntextracted_kwargsrP t_squeezedlhsrhsrrrtotal_Lt_as_ntrrrr$s^ " ,  rcOs|tjjjurt|i|S|jdkr)||djg|ddRi||dS|jdkrddd}t|||dd \}}|d }t | |d |j dd d }t | |d|j dd d }||krd|St tj|j||d} g|jd|| |j|ddR} |j| S|jdkrddd} t| ||dd \}}|d }|d} | |j d} t| | krtdtj||i|WdS1swYt|)Napply_rrflattenrAcSdSrr)input start_dimend_dimrrr _flatten_sigz+jagged_torch_function.._flatten_sigTrarbnormalize_to_only_use_kwargsrrF)r'rrms_normcSrrr)rnormalized_shapeweightepsrrr _rms_norm_sigrz,jagged_torch_function.._rms_norm_sigrzNrms_norm(): Normalization over the ragged dim not supported for nested tensors)rrANN)rE_C_nnscaled_dot_product_attentionrr\rrpopr-r"rLrreduceoperatormulrreshaperZr[DisableTorchFunctionSubclassr)r`rarbr_ new_kwargsinprrproduct new_shaperrmax_normalizablerrrjagged_torch_functionrs\ $   *         rz self: jt_allcOs|tjjjjkr dS|tjjjjkr|djS|tjjjjkr&t|djS|tjjj jtjjj jfvrW|dj durPt t |dj t|djddS|dj S|tjjjjkrd|djS|tjjjjkrs|djSdS)NFrr)rEopsatenis_non_overlapping_and_densedefaultsym_sizersr"rZ sym_numelnumelrKintrmathprodr sym_stride_stridessym_storage_offsetstorage_offsetrrrrtensor_attr_supported_getters  * rcOstjSr)rEjaggedrrrrprim_layout_defaultsrcOs|tjjjjkr tddS)NznNestedTensor does not support directly calling torch.ops.aten.size; please use `nested_tensor.size()` instead.)rErrr{rr%rrrrtensor_attr_unsupported_getters rcOspddlm}t|||dd\}}|d}|durdS|dtj|d<|dtjkr/dS||j fi|S)Nr)is_contiguous_for_memory_formatTrrF memory_format) rrrrrrrEcontiguous_formatpreserve_formatr)r`rarbrrrrrrris_contiguous_generals   rz!self: jt_all, memory_format: any?z"input: jt_all, memory_format: any?c Ost|||dd\}}|d}t|}|jdur:|dtjkr:ddlm}|jdks.Jd|| dd\}}|St ||j fi|fi|S) NTrrrr)jagged_from_listz;NJT with ragged_idx != 1 not supported for contiguous clone)r) rrrrKrErrrrLunbindrr) r`rarbrrrnew_metarcontigrrr clone_defaults    r zinput: jt, weight: t, bias: t?cO@t|||dd\}}|d}t||jfi|fit|SNTrrrrrrrr`rarbrrrrrrlinear_default#  "rz6self: jt, grad_output: jt, weight: t, output_mask: anycOst|||dd\}}|d}|d}|d}|d}d\} } } t||d|d|d r>tt|j|fit|} |d r^|jd | d } |jd | d } t| | } |d rt t |jj d }|d krv|j} n tj|j|dd} | | | fS)NTrr grad_outputr output_mask)NNNselfrrrArrF)keepdim)rrrtrrEmatmulrrrr{rPrrangercloner)r`rarbrrrrrrdsdwdbgrad_2dinput_2d reduce_dimsrrrlinear_backward_default.s2        rzinput: jt_all, dtype: anycOr r r r rrrto_dtypeWrrcOsddlm}t|||dd\}}|d}|d||jfi|}|jj|jd}d} |jdur:|jj|jd} dd l m } dd l m } m } |jdurP|jn|j} | durY|n| }t|| | fro| |}| | }|j|_n|| ||<t|}||d <| |d <t|fi|}|S) Nr_tensor_symint_registryTrrlayoutdevicer FakeTensorFunctionalTensormb_unwrap_functional_tensorrr)rr rrrrtor#rKtorch._subclasses.fake_tensorr%#torch._subclasses.functional_tensorr'r(rnested_int_memorr)r`rarbr rrr new_values new_offsets new_lengthsr%r'r( ragged_source new_thingtgtsrc inp_kwargsr$rrrto_copy_defaultbs2       r5z-self: jt_all, src: jt_all, non_blocking: any?c Ost|||dd\}}|d}|d}|j|jkrO|}dd|D}|} dd| D} || kr@td|jd |jt|| D] \} } | | qE|j|j|S) NTrrr3cSg|]}|jqSrrrcrrrr.z copy_default..cSr6rr7r8rrrr.r:z|jdkr>td |rI|jdurItd |dd |d<|rt j j j t j jj|j|jjd d |jg|jgtd d|jd} t j jj| |jg|jjd djd g|jjddR} t| fit|St||jfi|fit|S)NTrr"zHsoftmax(): not supported for dimensions of type 'tuple' for NestedTensorrsoftmaxzRsoftmax(): not supported when reducing across the batch dimension for NestedTensorrzesoftmax(): not supported when reducing along the ragged dimension for ragged_idx > 1 for NestedTensorzksoftmax(): not supported where lengths is not None if reducing across the ragged dimension for NestedTensorrrAz-inf) max_lengths padding_valuer"r)rrrr%rr8r"rLrKrEnn functionalrHrr_jagged_to_padded_dense_forwardrrrr _max_seqlenfloat_padded_dense_to_jagged_forwardrr) r`rarbrrrreduce_on_batchreduce_on_ragged_reduce_on_non_batchpadded_softmax_valuessoftmax_valuesrrr_softmax_defaultsx     "rXz7grad_output: jt, output: jt, dim: any, input_dtype: anycONt|||dd\}}|d}|d}t||j|jfi|fit|S)NTrrr$r )r`rarbrrgrad_outr$rrr_softmax_backwardLs  r[z!self: jt, float: any, train: any?cOs\t|||dd\}}|d}||jfi|\}}t|fit|t|fit|fSr rrrrr)r`rarbrrrout1out2rrrnative_dropout_default[s r_z%grad_output: jt, mask: jt, scale: anycOrY)NTrrmaskr )r`rarbrrrr`rrrnative_dropout_backward_defaultks  raz2self: jt_all, dim: any, keepdim: any?, dtype: any?cOt|ddg|Ri|S)Nrr_apply_reductionrrrr prod_dim_int{rezself: jt_all, dtype: any?cO0t|||dd\}}|d}||jfi|Sr rFr rrr prod_default  rhz$self: jt, split_size: any, dim: any?cs^t|||dd\}}|dt|djd|d<tfdd|jfi|DS)NTrrr"rXc3s&|]}tdd|itVqdS)valuesNrrrr>rrrrs  zsplit_tensor..)rrr-r"rLrrr`rarbrrrrlr split_tensors  rnz%self: jt, split_sizes: any, dim: any?csZt|||dd\}}|dt|djd|d<fdd|jfi|DS)NTrrr"split_with_sizesc"g|] }tdd|itqSrjrrkr>rlrrr.z,split_with_sizes_default..)rrr-r"rLrrmrrlrsplit_with_sizes_defaults  rsz+self: jt, dim: any, start: any, length: anycOsdt|||dd\}}|d}t||d|jd}||j||d|dd}t|fit|S) NTrrr"narrowstartlengthr"rurv)rrr-r"rLrrr)r`rarbrrrr"rjrrrrts rtz self: jt, chunks: any, dim: any?c st|||dd\}}|dt|djddd\|d<}|rd|d}j}||}dd |D} d d | D} fd d | Dd d |D} j | fd d t dt DSfdd |jfi|DS)NTrrr"chunkr)chunkscSsg|] }tj|ddqS)rrK)rEcumsumr>rrrr.sz!chunk_default..cSsg|] }tj|dddqS)rrr)value)Fpadr>rrrr.scsg|]}|jdqS))rrLr)r per_offsetsrlrrr.s cSsg|]}|qSr)ritemr>rrrr.rIcs&g|]}tdd|i|qSrqrrrg) chunk_values nested_kwargsrrr.srcrprqrkr>rlrrr.rr) rrr-r"rLrdiffrxrrXrrZ) r`rarbrrr,rzrchunked_lengthschunked_offsets split_sizesr)rrrr chunk_defaults0        rzself: jt_all, dim: any?c st|||dd\}}|d}|dkrtd|d}||}|}|jddttdt ttffd d } |durU| | t j d d Sdkr]td | | | fddt|jdDS)NTrr"rz2unbind(): only supported for NestedTensor on dim=0rrKrcsd}jd}tt|D],}t||t|||k|||7}|dur;t|||||kddqt||k|dur`tt|D]}t||t|||kqMdSdS)NrrcSrN)NzMunbind(): nested tensor offsets and lengths do not match ragged_idx dimensionrrrrrrHrOz2unbind_int.._torch_check..)rrrZrE_check_is_size_check)rKr lengths_sumragged_dim_sizerg)r/rjrr _torch_checks&  z unbind_int.._torch_checkrrKzAunbind(): nested tensor ragged_idx out of bounds (should be >= 1)cs*g|]}tjd||dqS)rrw)rErtrlengths_scalarsoffsets_scalarsr/rjrrr.1szunbind_int..r)rr%rrjrrrLrrrrtolistrErXrr) r`rarbrrr"rrrrrrr unbind_ints4  &   rzself: jt, dim: anycOsbt|||dd\}}|d}|j}tt|j|d|jd|d<t||fi|fit|S)NTrrr"r) rrrr-rZrsrLrr)r`rarbrrrrjrrr squeeze_dim<s  rzself: jt_all, dim: anyc Ost|||dd\}}|d}|j}|d}tt|jd||jddd|d<t|}|d|jdkr<|dd7<t||fi|fi|S) NTrrr"r unsqueeze)r(rL) rrrr-rZrsrLrr) r`rarbrrrrjr"rDrrrunsqueeze_defaultKs  rztensors: any, dim: anycst|||dd\}}|d}dd|D}t|dksJ|dfdd|D}|d}ttj|jd |d<t|d d|Dfi|fit|dS) NTrtensorscSsg|]}|jr|qSr) is_nestedrrPrrrr.kr@zcat_default..rcs g|] }|jr |n|qSr)r expand_asrfirstrrr.ns r"catcSr6rrrrrrr.wr:)rrrZr-rrLrr)r`rarbrrrnestedr"rrr cat_defaultbs"  rzself: any, other: anyc s(t||dd\}}|d}|d}fdd}fdd}|jrl|jsl|d krB||krB|j|d krB|||S|d krk||krk|j|d krkt|j|fi|fit|Sn|js|jr|d kr||kr|jd kr|||S|d kr||kr|jd krt||jfi|fit|SnX|jr|jr|d kr|d krt||j rt|j|jfit|S|d kr|d kr|jd kr|jd kr| |j| |jkrt |||St d |j d |j)NTrrothercs fddt||DS)Ncsg|] \}}||qSrr)ra_compb_comprrrr.sz8matmul_default.._unbind_impl..)ryr)rnrprrr _unbind_impls z$matmul_default.._unbind_implc s|jr|}n|}ddlm}|j}|j}|}|jj|jd}|dur%|}g|jd|j||j|jddR}|jd|d} |jrL| |} n|| } || |j |j|||dS)Nrr output_sizer) rrrrrrrrLto_padded_tensorr) rnrprzrrrrr padded_shape padded_ntpadded_trrr _padded_impls:   z$matmul_default.._padded_implrrz1matmul(): not supported between inputs of shapes rr)rrrr"rLrrrrrsr{rEstackr%r) r`rarbrrrrrrrrrmatmul_default{sV     %    & &$  rzself: jt_all, mat2: anycOsdt|||dd\}}|d}|d}|dkrtd|dkr(tdttjjjj ||S)NTrrmat2rzbmm(): input must be 3Dzbmm(): mat2 must be 3D) rrr"r[rrErrrrr`rarbrrrrrrr bmm_defaults    rz'self: jt_all, size: any, implicit: any?cst|||dd\}}|d|dd|vs|drJts.tdjdfdd td D}t|j|fit S) NTrrr{implicitzexpand(): cannot expand shape z -> cs"g|] }|jkr dn|qS)rArrrr{rrr. rz"expand_default..r) rrrr%rsrr"rrr)r`rarbrr expand_argrrrexpand_defaults  rzself: t, other: jtcOsDt|||dd\}}|d}|d}t|||jfit|S)NTrrrr rrrrexpand_as_default s   rzself: jt_all, size: anycOslt|||dd\}}|d}|d}t||kr2|gddt|t|D|Std)NTrrr{css|]}dVqdS)rNr)rrrrrr#szbroadcast_to..zwbroadcast_to(): broadcasting to a higher-dim shape is currently not supported for nested tensors with the jagged layout)rrrZr"rrr[)r`rarbrrrr{rrr broadcast_tos  ,rz tensors: anyc Ost|||dd\}}|d}t|dkrtdt|dkr#|dSg}tjdd|D}td d|D}|D],} | jrH|| |q:| t|krc|t | |j j fit|q:td t|S) NTrrrz7broadcast_tensors(): expected at least one tensor inputrcss|]}|jVqdSrr7rrrrr8sz$broadcast_tensors..csrrrMrrrrr:rzubroadcast_tensors(): broadcasting nested tensors with dense tensors of equal or higher dim is not currently supported)rrrZr[rEbroadcast_shapesrrrrr"rrrrr) r`rarbrrroutsbroadcast_shaperrPrrrbroadcast_tensors+s,   rz(condition: jt_all, self: any, other: anycOspt|||dd\}}|d}|d}|d}t|||\}}}t||j|j|jfi|fit|S)NTr conditionrr)rrrErrrr)r`rarbrrrrrrrr where_selfKs   rzself: jt, device: any?cOr r r r rrr_pin_memory_default`rrcOrgr rFr rrris_pinned_defaultkrirzself: jt_all, other: jt_allcOs|dj|djkSr)rsrrrris_same_size_defaultvsrcsDt|||dd\}}|d|ddup%t|dttfo%t|ddk}|rU|jfi|}|ddrSt|ttfrMt|fdd |D}|S| j }|S|d} t|dttf} | sg| g} t | |j \} } } }| s|| d} | |d<| rj durt|d dd lm}| r| r|jfi|}|ddr|d d |}|S|rt|d|d| rĈj gnj }||fd|i|S| rt|d|jfi|}t|ddst|dttfr|dn|dg}|D]}|j dkrdd8<q|fdd |S)NTrrr"rrFc3s|] }|jVqdSr)rrL)rorlrrrrz#_apply_reduction..zf(): reducing across the ragged dimension is not supported for non-contiguous nested tensors with holes)tree_mapcSs |dS)Nr)rrrrrrHs z"_apply_reduction..zW(): reducing along a ragged and non-batch dimension is not supported for nested tensorszg(): reducing along the batch dimension but not the ragged dimension is not supported for nested tensorsrrLcst|fiSrrr) out_kwargsrrrHs)rrrrrrZrrrrrLr8r"rKr%torch.utils._pytreerrr)r` func_nameidentity_elementrarbrrfull_reductionoutdim_to_convert is_dimlist converted_dimrSrTreduce_on_non_batchr dim_to_passdimlistrr)rrrrd}s         rdcOrgr rFr rrr sum_defaultrirz3self: jt_all, dim: any?, keepdim: any?, dtype: any?cOrb)Nrrrcrrrrsum_dim_IntListrfrz"self: jt_all, dim0: any, dim1: anyc Os.t|||dd\}}ddlm}|d}|||d|df\}}||jks.||jkrj|dks6|dkr:td||jkrB|} n|} t|} | | d <t| t t |j ||jt t |j ||jfi| St||d|jd |d<t||d|jd |d<t||jfi|fit|S) NTrrr rdim0dim1z?Transpose is not supported on the batch dimension for jagged NTrL transpose)rrr rr"rLr[rrrjrrrZrsr-r) r`rarbrrr rrrto_dimr4rrr transpose_ints>   "rzself: jt_all, dims: anyc st|||dd\}}|d|d}t}tjt|kr3tdddt|dd d lm}||}t|tt|krLtd j durUtd |d d kr_td | j |d<fdd|ddD} | |d<t |j fi|fi|S)NTrrr2z5permute(): number of dimensions in the tensor input (z) zAdoes not match the length of the desired ordering of dimensions ().rr z*permute(): duplicate dims are not allowed.zBpermute(): not supported on jagged layout nested tensor with holesz=Permute is not supported on the batch dimension for jagged NTrLcsg|] }t|jqSr)rrL)rr"rinp_dimrrr.Hs z#permute_default..r)rrrrZrsr[rr setrKindexrLrr) r`rarbrrr2r4r canonicalized_dims inner_dimsrrrpermute_default's>           rcst|||dd\}}|d|djdkr.tjtkr.tdjddtd ks9tsDtd jd fd d fddttdD}t  t |j |fitWdS1szwYdS)NTrrr{rzVview(): does not support ragged_idx != 1 except when inp._size == size. inp._size is (z) and size is (rrzview(): cannot view shape z as cs&|jdkr j|S|dSrJ)rLrr{) inner_idxrrrget_inner_sizets  z$view_default..get_inner_sizecsg|]}|qSrrr)rrrr.{sz view_default..)rrrLrrsr%rZrrrEinference_mode is_inferencerrr)r`rarbrr inner_sizer)rrr{r view_defaultPs(  $rzHinput: jt_all, normalized_shape: any, weight: any?, bias: any?, eps: anycOst|||dd\}}|d}|dkrtd|d}|j|j}|t|}|dkr3td||vr@|jdur@td ||vrtj j j |j j |jd |jg|jgd } tj j j tj|j jdd f|j|jd |jg|jgd | j} |jd d | jd} tj| ddd| } | | | } tjt| ddd| }t||d}| |}tj j j||jg|j jddd|j|jd d}t|fit|| |fS||j fi|\}} }t|fit|| |fS)NTrrrzOlayer_norm(): not supported for NestedTensor objects with 2 or fewer dimensionsrrzVlayer_norm(): not supported when normalizing over the batch dimension for NestedTensorzklayer_norm(): not supported where lengths is not None if operating on the ragged dimension for NestedTensorr)rIr)r#dtype)rr)r"rrrLrA)rrr"r%rrLrZrKrErrrOrrrrPonesr#rrrrrsquaresqrtrR unflattenrr)r`rarbrrrr ragged_sizenum_dims_not_normalized padded_input padded_maskragged_lengthsmeanpadded_normalizedvariancestdpadded_layer_normjagged_layer_norm_valuesr$rrrnative_layer_norm_defaults       rzpgrad_out: jt, input: jt, normalized_shape: any, mean: any, rstd: any, weight: any?, bias: any?, output_mask: anyc Ospt|||dd\}}|d}|d}||j|jfi|\}}} |dur+d|| fSt|fit||| fS)NTrrZrr\) r`rarbrrrZrd_inputd_gammad_betarrr"native_layer_norm_backward_defaults   rz"self: jt_all, dim: any, index: anycOst|||dd\}}|d}t||d|jddd\|d<}|r+||dS|jdur4tdt|}|d|jd krI|d d 8<t ||j fi|fi|S) NTrrr"selectryrzSselect(): not yet supported on dim != 0 for non-contiguous nested tensor with holesrrL) rrr-r"rLrrKr[rrr)r`rarbrrrr,rrrr select_ints"  rz7self: jt, dim: any?, start: any?, end: any?, step: any?cOs\t|||dd\}}|d}t||d|jd|d<t||jfi|fit|S)NTrrr"slice)rrr-r"rLrrrr rrr slice_tensors "rz8input: jt_all, indices: any, values: t, accumulate: any?cOst|||dd\}}|d}|d}t||ksJt||jdkr|s/tdddlm}|j }|j } | } |j j |jd} | durL| } g|j d|j| |j |jddR} |j d| d } ||| |fi||j|j| || d }|tjjjjkr|j ||S|S|dur|}n|}tt||j|kd ||d ||j}|d|j|g||jdd}|tjjjjkr||j |fi||_ |St||j |fi|fit|S) NTrrindicesrz[index_put(): If ragged dimension is not part of indices, this only works on contiguous NJTsrrrrz7Some indices in the ragged dimension are out of bounds!r)rrrZr"rL is_contiguousr%rrrrrrrrrErr index_put_rr;rjrrr _assert_asyncrxrr)r`rarbrrrrrrrrrr padded_inpnew_njtrragged_indices func_indicesrrrr'sx        rz{input: jt, weight: t, bias: t?, stride: any, padding: any, dilation: any, transposed: any, output_padding: any, groups: anycOr r r r rrrconvolution_defaults  "rcOst|||dd\}}|d}t||dd|j\}}}}|rg|sg|r&J|d} d|d<ttjjjj ddfi|} |j durD|j n|j } t | dD]}| d } qQ| | } | se| |j} | Sd } t|d| fi|S) NTrrr"rrrrrAgzG?)rr8r"rLrdrErrr dim_IntListrKrrrrr)r`rarbrrrrSrTrrintermediate_sumrrintermediate_valuerrrmean_dims8    r cOrgr rFr rrr mean_defaultrir z&self: jt_all, dim: any?, keepdim: any?cOrb)Nr1Frcrrrrany_dimsr z%self: jt_all, dim: any, keepdim: any?cO:t|||dd\}}|dg|d<ttjjjjfi|SNTrr")rr rErrr1r2rmrrrany_dim  rcOrb)NrxTrcrrrrall_dimsrrcOrr)rrrErrrxr2rmrrrall_dimrrcOrgr rFr rrrall_any_max_min_defaults   rcO@t|||dd\}}t|djj}t|d|g|Ri|S)NTrrminrrEfinformaxrdr`rarbrr dtype_maxrrrmin_dim  rcOr)NTrrrrrErrrrdr`rarbrr dtype_minrrrmax_dimrr"cOr)NTrraminrrrrr amin_default   r$cOr)NTrramaxrr rrr amax_defaultr%r'cOr)NTrrargminrrrrrargmin_default%r%r)cOr)NTrrargmaxrr rrrargmax_default1r%r+zAgrad: jt_all, dim: any, indices: jt_all, sizes: any, keepdim: anyc sddlmt|||dd\}}|d}|j|d<|d}|j|d<tfddt|d D}tt|d |d |d |d <|d }|j |j d ||<|d d}||d <t |} || d <t |di|fi| S)NrrTrgradrc3s |] \}}|r|VqdSrr)rrgrvrrrrMr0z=value_selecting_reduction_backward_default..sizesr""value_selecting_reduction_backwardrrLr) rrrrrrr]r-rZr{rLrr) r`rarbrrr,rr/r-rDrrr*value_selecting_reduction_backward_default=s,        r/cOst|||dd\}}|d}|D]%}t|tstd||dkr*tdt||djs6tdqt|dd|d |dj d |d <t|d d |Dfi|fit |dS) NTrrz+stack(): expected all nested tensors inputsrz9stack(): expected all nested tensors to have the same dimzFstack(): expected all nested tensors to have the same nested structurerr"rcSr6rrrrrrr.|r:z!stack_default..) rrrrr%r"rrr-rLr)r`rarbrrrrPrrr stack_defaultas0   r0zQweight: t, indices: jt, padding_idx: any?, scale_grad_by_freq: any?, sparse: any?cOsLt|||dd\}}|d}|d}t|||jfi|fit|S)NTrrrr )r`rarbrrrrrrrembedding_defaults  r1zYgrad_output: jt, indices: jt, num_weights: any, padding_idx: any, scale_grad_by_freq: anycOs>t|||dd\}}|d}|d}||j|jfi|S)NTrrrrF)r`rarbrrrrrrr embedding_dense_backward_defaults   r2cOs(t|||dd\}}|d}|jSr )rrrdetachr rrrvalues_defaults   r4cOs(t|||dd\}}|d}||jSr rFr rrr all_defaults   r5z-self: jt_all, padding: any, output_size: any?c Oslt|||dd\}}|d}|jdurtd|d}|dur&||j}n|jdur.|jn|jd}| }|jdkrF| |jdd}|j } | dkrV|j dd }n | dkra|d }|jtju} | rr|jrr|tj}tjj||jg|g|d } | r| jr| tj} t| dkr| d | dd} n t| dkr| d } |jdkr| |jd} | S) NTrrz?to_padded_tensor(): not supported for nested tensors with holesrrrrrrApadding)rrrKr%rL_max_seqlen_tensorrPrr{rjrrr"rrrrEboolis_cudar)halfrrrOrrZrr) r`rarbrrrr max_seq_lenrj values_shapeis_bool padded_outrrrto_padded_tensor_defaultsR                  r?zcpadded: t, offsets: t, dummy: jt, ragged_idx: any?, min_seqlen: any?, max_seqlen: any?, sum_S: any?cOs`t|||dd\}}|d|d}}|dd}|dkr#||d}|j}|dkr3|jdd }n |dkr>|d }|jtj u} | rO|j rO| tj }tj j||g|d } | rf| j rf| tj } t|dkrw| d |dd} n t|dkr| d } |dkr| |dd } |d } |d} i} | dur| | d <| dur| | d<t| ||| dS)NTrpaddedrr/rrrrrArrrr)rLr)rrrrr"rrrrEr8r9r)r:rrrRrZrrr)r`rarbrrr@rr/padded_ragged_dim1_shaper=rjrrmetadata_cacherrr"_nested_from_padded_tensor_defaultsN               rCzcvalues: t, offsets: t, dummy: jt_all, lengths: t?, ragged_idx: any?, min_seqlen: t?, max_seqlen: t?c Os~t|||dd\}}|d|d|d}}}|d}|d} |d} i} | dur.| | d<| dur6| | d<t||||| d S) NTrrrrr/rr)rrLr)rr) r`rarbrrrjrrr/rrrBrrr _nested_view_from_jagged_default6 s,  rDcO$t|||dd\}}|d}|jSr )rrrr rrr_nested_get_offsetsV  rFcOrEr )rrrKr rrr_nested_get_lengths` rGrHcOrEr )rrrLr rrr_nested_get_ragged_idxj rGrIcO,t|||dd\}}|d}|jddS)NTrrrrrrrr rrr_nested_get_min_seqlent  rLcOrJ)NTrrrrKr rrr_nested_get_max_seqlen~ rMrNzself: jt, mask: anyc Ost|||dd\}}|d}|d}|jdkrtd|j|jkr/td|jd|j|j|}t |j d d d }t |}||j |d <t dd |i|S)NTrrr`rz3masked_select only support 2-D selections currentlyzMask with shape z& is not compatible with input's shape rrKr|rrjr)rrrr%rr masked_selectrjr~rr{rrr) r`rarbrrrr` res_values mask_cumsumrrrmasked_select_default s(    rRz2grad_output: t, self: jt_all, dim: any, index: anycOsVt|||dd\}}|d}|d}tj||jd}||d|d||S)NTrrr)rr"r)rrrE zeros_likerrr;)r`rarbrrrr grad_inputrrr_nested_select_backward_default s  rUzself: jt_all, s: anycOsF|d}|d}||j|||j||jdur!||j|dSdSr)rrrK)r`rarbrstreamrrrrecord_stream_default s   rWzRself: jt_all, size: any, dtype: any?, layout: any?, device: any?, pin_memory: any?cOsHt|||dd\}}|d}t|ddkr ||jfi|Std)NTrrr{rz2new_empty() not supported for NJT with shape != ())rrrZrr%r rrrnew_empty_default s  rXzself: jt_all, ...cOs:tdd|D}t|dd|Di|fit|S)NcsrrrMrrrrr rz&activation_backward..css$|] }t|tr |jn|VqdSr)rrrrrrrr rwr)r`rarbrrrractivation_backward s  rYzself: jt_all, value: anycOr r r r rrr fill_Scalar rrZcOs4t|||dd\}}|d}||jfi||Sr rFr rrr fill__Scalar s  r[c OsTt|||dd\}}|d}t|}||j\}}t|fi|t|fi|fSr )rrrrr) r`rarbrrrrDmantissaexponentrrr frexp_Tensor s r^z+grad: any, self: any, other: any, mask: anyc Ost|||dd\}}|d}|d}|d}|d}|dur$dSd} |dr4t||d d } d} |d rDt|d d |} | | fS) NTrr,rrr`rrrAr)rrrErr) r`rarbrrr,rrgrad_input_mask grad_self grad_otherrrrmatmul_backward_default s     rc)flex_attentionflex_attention_backward) GraphModulerquerykeyr} score_mod block_maskscalekernel_optionsscore_mod_other_buffersmask_mod_other_buffersc Cs|dkr|dkr|dksJtdd||Dr#tdt|d|d|d||||||d } tjj| d dd d|j |j |j |jd  dd} tjj| d dd d|j |j |j |jd  dd} | | fS) Ncss"|] }t|tjo |jVqdSr)rrErFr)rbufrrrrC s  zflex_njt..zflex_attention(): Nested tensor score_mod / mask_mod buffers are not currently supported. Please file an issue if this is important to you.r)rirjrkrlrmrnrrrr)r"r1r%flex_attention_hoprjrrErnested_tensor_from_jaggedrrrrKrr) rgrhr}rirjrkrlrmrnr$ output_njt logsumexp_njtrrrflex_njt4 sJ(     rvr logsumexprZgrad_logsumexpfw_graph joint_graph.cCst|d|d|d|d|d|d|d||| | | | | d}|\}}}}tjj|ddd|j|j |j |j ddd}tjj|ddd|j|j |j |j ddd}tjj|ddd|j|j |j |j ddd}||||fS)Nr) rrwrZrxryrzrjrkrlrmrnrrrq) flex_attention_backward_hoprjrrErrsrrrrKrr)rgrhr}rrwrZrxryrzrjrkrlrmrnr$ dense_q_grad dense_k_grad dense_v_gradscore_mod_other_buffer_grads njt_q_grad njt_k_grad njt_v_gradrrrflex_njt_backwardm sZ         rz self: anycOsddlm}|S)Nr)_nt_view_dummy)$torch.nested._internal.nested_tensorr)r`rarbrrrr_nested_get_jagged_dummy s rrIMPLCPUCUDAMeta)F)TFF)r)rr)rrrtypingrrEtorch.nn.functionalrMrNr~torch.fx.operator_schemasrtorch.nested._internal.sdparrrr rAny__annotations__r Dictrr-r8strrmrtrrrrregister_jagged_funcCallablerrrrrrrrrrr"rrrrrprimr!rr{rrrrrr linearrlinear_backwardrr)rr_to_copyr5r;rBr3 empty_like ones_likerS rand_like randn_likerE full_like randint_like low_dtypezero_rG_softmaxrX_softmax_backward_datar[native_dropoutr_native_dropout_backwardrardim_intrerhrXrFrnrorsrtrxrrrrrrrrrrrrbmmrrrrrrrwhererr _pin_memoryr is_pinnedr is_same_sizerrdrrrrrrpermuterview _unsafe_viewrnative_layer_normrnative_layer_norm_backwardrrrrr index_putr convolutionrrr r r1r2r rrxrrrrrrr"r#r$r&r'r(r)r*r+r.r/rr0 embeddingr1embedding_dense_backwardr2rj_nested_get_valuesr4r5rr?_nested_from_padded_tensorrC_nested_view_from_jaggedrDrFrHrIrLrNrOrR_nested_select_backwardrU record_streamrW new_empty new_zerosnew_onesrX elu_backwardhardshrink_backwardhardsigmoid_backwardhardtanh_backwardsoftplus_backwardsoftshrink_backwardrYfillScalarrZfill_r[frexpr^matmul_backwardrc&torch._higher_order_ops.flex_attentionrdrrrer{torch.fx.graph_modulerfpy_implTuplerQrvUnionrrlibrary_scoped_libraryimplrrrrsf       @    )  NH                   %  (        /    H                ) H    m            j     * ( /  a        T    &                                 ;  5                              8       ?$