o I h*.@sddlZddlmmZddlmZddlZddZddZ ddZ d d Z d d Z ej d dZddZddZddZddZddZdS)N) namedtuplecstfdd}|S)Nc sdrdj}||i|SdsdrDdr#dnd}|dkr+dnd}|j}td|dd|d|d |i|S) Nautogradsave_for_backwardbackwardz We found a 'z' registration for  at z but were unable to find a 'z' registration. To use the CustomOp API to register a backward formula, please provide us both a backward function and a 'save for backward' function via `impl_backward` and `impl_save_for_backward` respectively.) _has_impl _get_implfunclocation RuntimeError)argskwargskernelmissingfoundlocautograd_fallback custom_opM/var/www/vscode/kcb/lib/python3.10/site-packages/torch/_custom_op/autograd.pyinners"   z*autograd_kernel_indirection..inner)autograd_not_implemented)rrrrrautograd_kernel_indirectionsrcsfdd}|S)Ncs`trtdd||frtdtj|i|WdS1s)wYdS)NcSst|tjo|jSN) isinstancetorchTensor requires_gradxrrr4z:autograd_not_implemented..kernel..z.Autograd has not been implemented for operator)ris_grad_enabledpytreetree_anyr _C_AutoDispatchBelowAutograd)r r rrrr2s    $z(autograd_not_implemented..kernelr)rrrr(rr1s rc Cs|dur_t|ts |f}n|}t|t|ksJg}tt||D]3\}\}}t|tjr6|s5||q"t|trC|sB| |q"|rUt d|d|dt |dq"|ra|j |dSdSdS)NzWith output_differentiability=z . At idx z , we received an object of type za that is not a Tensor, so it cannot have be marked as differentiable in output_differentiability.) rtuplelen enumerateziprrappendlistextendr typemark_non_differentiable)ctxoutputoutput_differentiability tuple_outputnon_differentiable_tensorsidxdifferentiableoutrrrr1<s8     r1csfdd}|S)Ncspt|\}dfdd}fdd}tjd||}|j|}dus0Jtt|S)Ncs|dtt|}tj |}Wdn1s!wYttt |}t|}||}t |||ft ||t |\}t |S)NT)set_materialize_gradsr$tree_unflattenr.rr&r'namedtuple_argstree_mapr0save_pytree_for_backwardr1 tree_flattenr))r2 flat_argsr r3 args_infosave_for_backward_fn_inputsto_save flat_output) op_overloadout_specr4save_for_backward_fnschemaspecrrforwardhs       z9construct_autograd_kernel..apply..forwardcsfdusJtt|}t|\}}t}t|ts|f}||g|R}t||t||Sr) r$r;r. unpack_savedobjectrr)validate_grad_inputs_dictgrad_inputs_dict_to_flat_tuple)r2flat_grad_outputgradssavedrA inner_ctxgrad_inputs_dict) backward_fnrrFrrr|s     z:construct_autograd_kernel..apply..backward _customop)r$r?gen_autograd_function_opnameapplyr;r.)r r@rJr generated_clsrDrTrrEr4rGrH)rFrIrrXds   z(construct_autograd_kernel..applyr)rHr4rrErGrTrXrrZrconstruct_autograd_kernel\s.r[cCs$t|tjjft|t|d}|S)N)rJr)r0rrFunction staticmethod)namerJrrYrrrrVsrVcCs.dd|jjD}t|jd}t||}|S)NcSsg|]}|jqSr)r^.0argrrr r"z'namedtuple_args_cls.._args) argumentsflat_allstrr^r)rHattribsr^ tuple_clsrrrnamedtuple_args_clss ricCst|tsJt|}||Sr)rr)ri)rHr rhrrrr<sr<c sfdd}t|ts|dt|ddjjjD}|}||kr1|d|d|d|D]\}}t||}t|t rt|t t fsW|d |d t|d t |t |ksp|d |d t |d t |t t ||D]6\} \} } | durqwt| tjs|d |dt| d| t| tjs|d |d| d| d|qwq5|durq5t|tjs|dt|d|dt|tjs|d|d|d|dq5dS)Ncs&d}tdd|jd|)Nrz%In the backward function defined for rz using the CustomOp API, )rr r )whatr forward_oprrerrors z(validate_grad_inputs_dict..errorzBexpected the output of the backward function to be a dict but got cSsh|] }|jr|jqSr)r0is_tensor_liker^r_rrr s z,validate_grad_inputs_dict..z3expected the returned grad_input dict to have keys z but got z. The backward function must return a gradient (can be None) for each arg to the CustomOp that may be a Tensor or Sequence[Tensor]. Args declared to be non-Tensor-like types should not appear in the grad_input dictz for input 'zR' expected the grad_input dict to hold a list of gradients but got object of type .z1' expected the grad_input dict to hold a list of z gradients but got z\' expected the grad_input dict to hold a list of None or Tensor gradients but got object of z at index z(', got a Tensor as the gradient for the z(-th value but expected None because the z(-th value was not a Tensor (it was type zgot object of type z as the gradient for input 'z:', but expected the gradient to be either None or a Tensorz(got a Tensor as the gradient for input 'z3' but expected None as the gradient because input 'z ' was not a Tensor (it was type z).)rdictr0_schemardrekeysitemsgetattrr.r)r*r+r,rr issubclass) rSrlrArm expected_keys actual_keysr^gradarg_infor7ginforrkrrMsv                rMcCsVg}|D]\}}||vr|tdd|q|||qtt|S)NcSsdSrrrrrrr!sz0grad_inputs_dict_to_flat_tuple..)_asdictrtr-r$r=r) tree_leaves)rSrAresultr^rzrrrrNsrNc Cst|\}}t|}ddt|D}ddt|D}dd|D}dd|D}||_||_|j|||_||_||_ dS)NcSs g|] \}}t|tjr|qSrrrrr`r7thingrrrrb  z,save_pytree_for_backward..cSs g|] \}}t|tjs|qSrrrrrrrbrcSsg|] }t|tjr|qSrrr`rrrrrbcSsg|] }t|tjs|qSrrrrrrrbr) r$r?r*r+rInum_eltsr tensor_idxssaved_non_tensorsnon_tensor_idxs) r2stuff flat_stuffrIrrrtensors non_tensorsrrrr>s  r>cCs^dg|j}t|j|jD]\}}|||<q t|j|jD]\}}|||<qt||j}|Sr) rr, saved_tensorsrrrr$r;rI)r2rtensorr7 non_tensorrrrrrK s   rK)rtorch.utils._pytreeutils_pytreer$ collectionsr functoolsrrr1r[rV lru_cacherir<rMrNr>rKrrrrs     9 <