o
    Ih                     @   s  d Z ddlZddlZddlZddlZddlmZmZmZm	Z	 ddl
ZddlmZ ddlmZmZ ddlmZmZmZ ddlmZ dd	lmZ dd
lmZmZ ddlmZmZ ddlm Z  ddl!m"Z"m#Z#m$Z$m%Z%m&Z& ddl'm(Z( erxddl)m*Z* ej+G dd dZ,G dd de Z-G dd de(Z.G dd de-Z/G dd de-Z0G dd de-Z1G dd de-Z2G dd  d e-Z3G d!d" d"e-Z4G d#d$ d$e-Z5G d%d& d&e-Z6G d'd( d(e-Z7G d)d* d*e-Z8G d+d, d,e-Z9G d-d. d.e-Z:G d/d0 d0e-Z;G d1d2 d2e-Z<G d3d4 d4e-Z=G d5d6 d6e-Z>G d7d8 d8e-Z?G d9d: d:e-Z@G d;d< d<e-ZAG d=d> d>e-ZBG d?d@ d@e-ZCG dAdB dBe ZDG dCdD dDe ZEG dEdF dFe ZFdS )Ga)  
This file contains a collection of context manager classes used by Dynamo for tracking
and managing various PyTorch runtime states during graph compilation. These context
managers handle different aspects of PyTorch's execution environment, including:

- Autograd states (grad mode, inference mode)
- CUDA streams and events
- Profiling contexts
- Deterministic algorithms
- Forward/backward AD modes
- SDPA (Scaled Dot Product Attention) kernels
- FSDP (Fully Sharded Data Parallel) states
- AMP (Automatic Mixed Precision) autocast states

The context managers ensure proper state transitions during graph compilation by
tracking enter/exit points and managing cleanup operations. They help maintain
consistency between eager execution and compiled graph behavior by capturing and
restoring state changes.
    N)CallableOptionalTYPE_CHECKINGUnion)Guard   )graph_break_hints	variables)create_call_functioncreate_instructioncreate_setup_with)get_interface_for_device)unimplemented_v2)GuardBuilderinstall_guard)
AttrSourceGlobalStateSource   )VariableTracker)NestedUserFunctionVariableUserFunctionVariableUserMethodVariableWrappedUserFunctionVariableWrappedUserMethodVariable)UserDefinedObjectVariable)InstructionTranslatorc                   @   sF   e Zd ZU dZdZee ed< dZee	j
j ed< dd Zdd ZdS )	ContextManagerStatez
    Mutating `self` in VariableTracker is not allowed because we copy
    them.  This is a mutable container pointed to by context managers
    that won't get copied, so it is safe to mutate.
    N
cleanup_fnproxyc                 C   s    | j d ur|    d | _ d S d S N)r   self r"   W/var/www/vscode/kcb/lib/python3.10/site-packages/torch/_dynamo/variables/ctx_manager.pycleanupD   s   

zContextManagerState.cleanupc                 C   s   | j sJ d|   d S )Nzmultiple exits?)r   r$   r    r"   r"   r#   cleanup_assertI      z"ContextManagerState.cleanup_assert)__name__
__module____qualname____doc__r   r   r   __annotations__r   torchfxProxyr$   r%   r"   r"   r"   r#   r   9   s   
 r   c                       s   e Zd ZddddhejZ	d%dd	d& fdd	Zd
d Zd%d'ddZd'ddZdd Z	dd Z
dd Zdd Z								d(dd Zd!d" Zd#d$ Z  ZS ))ContextWrappingVariablecm_objtarget_valuesinitial_valuesstateN)r3   returnc                   s<   t  jdi | || _|| _|d u rt | _d S || _d S Nr"   )super__init__r1   r2   r   r3   )r!   r1   r2   r3   kwargs	__class__r"   r#   r7   W   s   z ContextWrappingVariable.__init__c                 C   s$   |  || j | | tjd S r   )
_call_funcr1   set_cleanup_hookr	   ConstantVariablecreater!   txr"   r"   r#   enter_   s   
zContextWrappingVariable.enterr@   r   c                    s2   |d u r fdd}| j _j j j d S )Nc                      s      j d S r   )r;   r2   r"   r?   r"   r#   fng   s   z4ContextWrappingVariable.set_cleanup_hook.<locals>.fn)r3   r   outputadd_cleanup_hookr$   )r!   r@   rB   r"   r?   r#   r<   d   s   z(ContextWrappingVariable.set_cleanup_hookc                 G   s   | j   tjd S r   )r3   r%   r	   r=   r>   r!   r@   argsr"   r"   r#   exitm   s   
zContextWrappingVariable.exitc                 C   s"   |t |j|  |   d S r   )r   r@   import_sourcemodule_namefn_namer!   codegenr"   r"   r#   reconstruct_typeq   s   z(ContextWrappingVariable.reconstruct_typec                    sR      fdd j}|sd}  fdd|D   tt|d d S )Nc                      s
     S r   )rM   r"   rL   r!   r"   r#   <lambda>w   s   
 z5ContextWrappingVariable.reconstruct.<locals>.<lambda>r"   c                       g | ]}  |qS r"   create_load_const.0valrL   r"   r#   
<listcomp>{       z7ContextWrappingVariable.reconstruct.<locals>.<listcomp>F)add_push_nullr1   extend_outputr
   len)r!   rL   r1   r"   rN   r#   reconstructv   s   z#ContextWrappingVariable.reconstructc                 C      t d)Nzmodule_name called on baseNotImplementedErrorr    r"   r"   r#   rI   ~      z#ContextWrappingVariable.module_namec                 C   r]   )Nzfn_name called on baser^   r    r"   r"   r#   rJ      r`   zContextWrappingVariable.fn_namerF   list[VariableTracker]r8   dict[str, VariableTracker]r   c                 C   s   t |dksJ t|d trt|d  |d< t|d ttfs$J t|d tr2t|d | S t|d tr@t|d | S d S Nr   r   )r[   
isinstancer   r   get_functionr   r   r   r!   r@   rF   r8   r"   r"   r#   call_function   s   z%ContextWrappingVariable.call_functionc                 C      dS NTr"   r    r"   r"   r#   supports_graph_breaks      z-ContextWrappingVariable.supports_graph_breaksc                 C   rh   ri   r"   r    r"   r"   r#   exit_on_graph_break   rk   z+ContextWrappingVariable.exit_on_graph_breakr   r4   Nr@   r   r@   r   rF   ra   r8   rb   r4   r   )r'   r(   r)   r   _nonvar_fieldsr7   rA   r<   rG   rM   r\   rI   rJ   rg   rj   rl   __classcell__r"   r"   r9   r#   r/   N   s@    	
	
r/   c                       sP   e Zd Zd fddZdd Zdd Zd	d
 ZdddZdd Zdd Z	  Z
S )GenericContextWrappingVariabler4   Nc                    s0   |d usJ t  jd||jd| || _d S )N)value
value_typer"   )r6   r7   r:   r0   )r!   r0   r8   r9   r"   r#   r7      s   
z'GenericContextWrappingVariable.__init__c                 C   s   | j jS r   )r0   r(   r    r"   r"   r#   rI      r`   z*GenericContextWrappingVariable.module_namec                 C   s   t | jjS r   )typer0   r'   r    r"   r"   r#   rJ         z&GenericContextWrappingVariable.fn_namec                 C   s:   | j d u rd nt| j d}tj| jjj| |d|g i S )N	__enter__source)ry   r   r	   r   r0   rw   __func__rg   )r!   r@   ry   r"   r"   r#   rA      s   
z$GenericContextWrappingVariable.enterr@   r   c                 G   sH   | j d u rd nt| j d}tj| jjj| |d||i }|j	  |S )N__exit__rx   )
ry   r   r	   r   r0   r{   rz   rg   active_generic_context_managerspop)r!   r@   rF   ry   xr"   r"   r#   rG      s   

z#GenericContextWrappingVariable.exitc                 C   rh   )NFr"   r    r"   r"   r#   rj      rk   z4GenericContextWrappingVariable.supports_graph_breaksc                 C   rh   ri   r"   r    r"   r"   r#   rl      rk   z2GenericContextWrappingVariable.exit_on_graph_breakrm   rn   )r'   r(   r)   r7   rI   rJ   rA   rG   rj   rl   rq   r"   r"   r9   r#   rr      s    	

rr   c                   @   0   e Zd ZdZedddZdd Zddd	Zd
S ))GradInplaceRequiresGradCtxManagerVariablez#represents torch grad requries gradr@   r   c                 K      t d|d d|S Nr1   r2   r"   )r   r@   r1   r8   r"   r"   r#   r>         z0GradInplaceRequiresGradCtxManagerVariable.createc                    sb    j \}tjj  _tjj|  | fdd |j	dtjjj|fi  j
_tjd S )Nc                         t jj jS r   )r,   _C
_functorch!set_inplace_requires_grad_allowed
prev_stater"   r    r"   r#   rO      s    zAGradInplaceRequiresGradCtxManagerVariable.enter.<locals>.<lambda>rg   )r1   r,   r   r   !get_inplace_requires_grad_allowedr   r   r<   rC   create_noder3   r   r	   r=   r>   )r!   r@   enabledr"   r    r#   rA      s   
z/GradInplaceRequiresGradCtxManagerVariable.enterc                 G   s2   | j   |jdtjjj| jfi  t	j
d S Nrg   )r3   r$   rC   r   r,   r   r   r   r   r	   r=   r>   rE   r"   r"   r#   rG      s   
z.GradInplaceRequiresGradCtxManagerVariable.exitNrn   r'   r(   r)   r*   staticmethodr>   rA   rG   r"   r"   r"   r#   r      s    r   c                   @   r   )0TemporarilyPopInterpreterStackCtxManagerVariablezJrepresents torch._functorch.pyfunction.temporarily_pop_interpreter_stack()r@   r   c                 K   r   r   )r   r   r"   r"   r#   r>      r   z7TemporarilyPopInterpreterStackCtxManagerVariable.createc                    sJ   t jj  _ | fdd |jdt jjjdi  j_	t
jd S )Nc                      r   r   )r,   r   r   push_dynamic_layer_stacksavedr"   r    r"   r#   rO          zHTemporarilyPopInterpreterStackCtxManagerVariable.enter.<locals>.<lambda>rg   r"   )r,   r   r   pop_dynamic_layer_stackr   r<   rC   r   r3   r   r	   r=   r>   r?   r"   r    r#   rA      s   
z6TemporarilyPopInterpreterStackCtxManagerVariable.enterc                 G   s4   | j   |jdtjjj| j jfi  t	j
d S r   )r3   r$   rC   r   r,   r   r   r   r   r	   r=   r>   rE   r"   r"   r#   rG     s   
z5TemporarilyPopInterpreterStackCtxManagerVariable.exitNrn   r   r"   r"   r"   r#   r      s    r   c                   @   >   e Zd ZdZee ejZe	dddZ
dd Zddd	Zd
S )%JvpIncrementNestingCtxManagerVariablez5represents torch.func.jvp increment/decrement nestingr@   r   c                 K      t dd d d|}|S r   )r   r@   r8   varr"   r"   r#   r>        z,JvpIncrementNestingCtxManagerVariable.createc                 C   sN   t | j tjj }| |dd  |jdtj	jj
di | j_tj|S )Nc                   S      t jj S r   )r,   r   eager_transformsexit_jvp_nestingr"   r"   r"   r#   rO   *      z=JvpIncrementNestingCtxManagerVariable.enter.<locals>.<lambda>rg   r"   )r   _guards_singletonr,   r   r   enter_jvp_nestingr<   rC   r   r   _jvp_increment_nestingr3   r   r	   r=   r>   )r!   r@   	jvp_levelr"   r"   r#   rA   &  s   
z+JvpIncrementNestingCtxManagerVariable.enterc                 G   .   | j   |jdtjjjdi  tj	
d S Nrg   r"   )r3   r$   rC   r   r,   r   r   _jvp_decrement_nestingr	   r=   r>   rE   r"   r"   r#   rG   4  
   
z*JvpIncrementNestingCtxManagerVariable.exitNrn   r'   r(   r)   r*   r   r   r   FUNCTORCH_STACK_MATCHr   r   r>   rA   rG   r"   r"   r"   r#   r     s    r   c                   @   r   )SetFwdGradEnabledContextManagerzWrepresents torch.autograd.forward_ad._set_fwd_grad_enabled() to enable/disable fwd gradr@   r   c                 K   r   r   )r   r   r"   r"   r#   r>   ?  r   z&SetFwdGradEnabledContextManager.createc                    s\    j \}tj  _tj|  | fdd |jdtjj|fi  j	_
tjd S )Nc                      s   t j jS r   )r,   r   _set_fwd_grad_enabledr   r"   r    r"   r#   rO   M      z7SetFwdGradEnabledContextManager.enter.<locals>.<lambda>rg   )r1   r,   r   _is_fwd_grad_enabledr   r   r<   rC   r   r3   r   r	   r=   r>   )r!   r@   moder"   r    r#   rA   G  s   
z%SetFwdGradEnabledContextManager.enterc                 G   0   | j   |jdtjj| jfi  tj	
d S r   )r3   r$   rC   r   r,   r   r   r   r	   r=   r>   rE   r"   r"   r#   rG   W     
z$SetFwdGradEnabledContextManager.exitNrn   r   r"   r"   r"   r#   r   <  s    r   c                   @   r   )DualLevelContextManagerz;Represents torch.autograd.forward_ad.dual_level ctx managerr@   r   c                 K   s   t dd d d|S r   )r   )r@   r8   r"   r"   r#   r>   g  r   zDualLevelContextManager.createc                    sT   t  j tjj  _ | fdd |j	dtj
jdi  j_tj jS )Nc                      s   t jjj jdS )N)level)r,   autograd
forward_adexit_dual_level	new_levelr"   r    r"   r#   rO   s      z/DualLevelContextManager.enter.<locals>.<lambda>rg   r"   )r   r   r,   r   r   enter_dual_levelr   r<   rC   r   r   _enter_dual_levelr3   r   r	   r=   r>   r?   r"   r    r#   rA   o  s   
zDualLevelContextManager.enterc                 G   r   r   )r3   r$   rC   r   r,   r   _exit_dual_levelr   r	   r=   r>   rE   r"   r"   r#   rG   }  r   zDualLevelContextManager.exitNrn   )r'   r(   r)   r*   r   r   r   
DUAL_LEVELr   r   r>   rA   rG   r"   r"   r"   r#   r   b  s    r   c                   @   r   )&GradIncrementNestingCtxManagerVariablez6represents torch.func.grad increment/decrement nestingr@   r   c                 K   r   r   )r   r   r"   r"   r#   r>     r   z-GradIncrementNestingCtxManagerVariable.createc                 C   sN   t | j tjj }| |dd  |jdtjjjdi | j	_
tj|S )Nc                   S   r   r   )r,   r   r   _grad_decrement_nestingr"   r"   r"   r#   rO     r   z>GradIncrementNestingCtxManagerVariable.enter.<locals>.<lambda>rg   r"   )r   r   r,   r   r   _grad_increment_nestingr<   rC   r   r3   r   r	   r=   r>   )r!   r@   
grad_levelr"   r"   r#   rA     s   
z,GradIncrementNestingCtxManagerVariable.enterc                 G   r   r   )r3   r$   rC   r   r,   r   r   r   r	   r=   r>   rE   r"   r"   r#   rG     r   z+GradIncrementNestingCtxManagerVariable.exitNrn   r   r"   r"   r"   r#   r     s    r   c                       s@   e Zd ZdZedddZd fdd	Zd
d Zdd Z  Z	S )CatchWarningsCtxManagerVariablez'Delay a call to warnings.catch_warningsr@   r   c                 C   s   t |d d dS )N)catch_warnings_argsr1   r2   )r   )r@   r   r"   r"   r#   r>     s
   z&CatchWarningsCtxManagerVariable.creater4   Nc                    s.   t |ts	J |t jdi | || _d S r5   )rd   dictr6   r7   r   )r!   r   r8   r9   r"   r#   r7     s   
z(CatchWarningsCtxManagerVariable.__init__c                    sH   dd | j  D }tjdi | | | fdd tj  S )Nc                 S   s   i | ]	\}}||  qS r"   )as_python_constant)rT   kvr"   r"   r#   
<dictcomp>  s    z9CatchWarningsCtxManagerVariable.enter.<locals>.<dictcomp>c                      s     d d d S r   )r{   r"   ctx_valr"   r#   rO     r   z7CatchWarningsCtxManagerVariable.enter.<locals>.<lambda>r"   )	r   itemswarningscatch_warningsr<   r	   r=   r>   rw   )r!   r@   r8   r"   r   r#   rA     s   z%CatchWarningsCtxManagerVariable.enterc                    sL      fdd  | j  t| j }  t||d d S )Nc                      s     ddS )Nr   r   )load_import_fromr"   cgr"   r#   rO     r   z=CatchWarningsCtxManagerVariable.reconstruct.<locals>.<lambda>F)	rY   foreachr   valuestuplekeysrZ   create_call_function_kwr[   )r!   r   r   r"   r   r#   r\     s   z+CatchWarningsCtxManagerVariable.reconstructrn   rm   )
r'   r(   r)   r*   r   r>   r7   rA   r\   rq   r"   r"   r9   r#   r     s    r   c                   @   r   )&VmapIncrementNestingCtxManagerVariablez1represents torch VMap increment/decrement nestingr@   r   c                 K   s   t d|d d|}|S r   )r   )r@   r1   r8   r   r"   r"   r#   r>     r   z-VmapIncrementNestingCtxManagerVariable.createc                 C   s   t | j | j\}}t|tjr|j}| j}n|	 }|	 }|	 }t
jj||}| |dd  |jdt
jjj||fi | j_tj|S )Nc                   S   r   r   )r,   r   r   _vmap_decrement_nestingr"   r"   r"   r#   rO     r   z>VmapIncrementNestingCtxManagerVariable.enter.<locals>.<lambda>rg   )r   r   r1   rd   r	   SymNodeVariablesym_numas_proxynoder   r,   r   r   _vmap_increment_nestingr<   rC   r   r3   r   r=   r>   )r!   r@   
batch_size
randomnessbatch_size_valuebatch_size_node
vmap_levelr"   r"   r#   rA     s&   

z,VmapIncrementNestingCtxManagerVariable.enterc                 G   r   r   )r3   r$   rC   r   r,   r   r   r   r	   r=   r>   rE   r"   r"   r#   rG     r   z+VmapIncrementNestingCtxManagerVariable.exitNrn   r   r"   r"   r"   r#   r     s    r   c                       s   e Zd ZdZee ejZe	ddddZ
	d	d fd
dZdd ZdddZ						d  fddZdddZdd Zdd Z  ZS )!GradModeVariablez6represents torch.{no_grad,enable_grad,set_grad_mode}()Fr@   r   c                 K   s2   t d|gt gd|}|r|| |j |S r   )r   r,   is_grad_enabledr;   r1   )r@   target_valueinitializedr8   r   r"   r"   r#   r>     s   zGradModeVariable.createNTr4   c                    s&   t  jd||d| t| j d S r   r6   r7   r   r   )r!   r1   r2   r   r8   r9   r"   r#   r7     s   zGradModeVariable.__init__c                 C      |  || j tjd S r   r;   r1   r	   r=   r>   r?   r"   r"   r#   rA     r&   zGradModeVariable.enterc                 G   r   r   r;   r2   r	   r=   r>   rE   r"   r"   r#   rG     r&   zGradModeVariable.exitrF   ra   r8   rb   c                       |  || j t |||S r   r;   r2   r6   rg   rf   r9   r"   r#   rg        zGradModeVariable.call_functionc                 C   sP   t |dksJ |d }t |kr&|jdtjj|fi  tj| d S d S Nr   r   rg   )r[   r,   r   rC   r   r   _set_grad_enabledr!   r@   r   rs   r"   r"   r#   r;   (  s   zGradModeVariable._call_funcc                 C   rh   Nr,   r"   r    r"   r"   r#   rI   2  rk   zGradModeVariable.module_namec                 C   rh   )Nset_grad_enabledr"   r    r"   r"   r#   rJ   5  rk   zGradModeVariable.fn_name)Frn   ri   rm   r@   r   rF   ra   r8   rb   )r'   r(   r)   r*   r   r   r   	GRAD_MODEr   r   r>   r7   rA   rG   rg   r;   rI   rJ   rq   r"   r"   r9   r#   r     s*    

	
r   c                       sT   e Zd ZedddZ	d	d fddZdd	d
Zdd Zdd Zdd Z	  Z
S )InferenceModeVariabler@   r   c                 K   s   t |gfdt i|}|S )Nr2   )r   r,   is_inference_mode_enabledr@   r   r8   r   r"   r"   r#   r>   :  s   zInferenceModeVariable.createNr4   c                    s2   |d u rt  }t jd||d| || _d S r   )r,   r   r6   r7   r1   r!   r1   r2   r8   r9   r"   r#   r7   A  s   
zInferenceModeVariable.__init__c                 G   s,   | j   |jdtjjj| j jfi  d S r   )	r3   r%   rC   r   r,   r   	grad_mode_exit_inference_moder   rE   r"   r"   r#   rG   O  s   
zInferenceModeVariable.exitc                    sL   t jjj| j  | | fdd |jdt jjjg | jR i | j_	d S )Nc                      s   t jj S r   )r,   r   r   r   r"   ctxr"   r#   rO   [  r   z-InferenceModeVariable.enter.<locals>.<lambda>rg   )
r,   r   r   _enter_inference_moder1   r<   rC   r   r3   r   r?   r"   r   r#   rA   X  s   
zInferenceModeVariable.enterc                 C   rh   r   r"   r    r"   r"   r#   rI   d  rk   z!InferenceModeVariable.module_namec                 C   rh   )Ninference_moder"   r    r"   r"   r#   rJ   g  rk   zInferenceModeVariable.fn_namern   r   rm   r'   r(   r)   r   r>   r7   rG   rA   rI   rJ   rq   r"   r"   r9   r#   r   9  s    	
	r   c                       sX   e Zd ZdZedddZ	d	d fdd	Zdd
dZdd Zdd Z	dd Z
  ZS )CUDADeviceVariablezrepresents torch.cuda.devicer@   r   c                 K   s&   t dtjj|ddgd d|}|S )NT)optionalr   r"   )r   r,   cuda_get_device_index)r@   devicer8   r   r"   r"   r#   r>   n  s   zCUDADeviceVariable.createNr4   c                    "   t  jd||d| || _d S r   r6   r7   r1   r   r9   r"   r#   r7   w  s   
zCUDADeviceVariable.__init__c                 G   s2   | j   |jdtjj| j jfi  tj	
dS )Nrg   F)r3   r%   rC   r   r,   r   _maybe_exchange_devicer   r	   r=   r>   rE   r"   r"   r#   rG     s   
zCUDADeviceVariable.exitc                    H   t jj| j  | | fdd |jdt jjg | jR i | j_d S )Nc                         t j S r   )r,   r   r  r"   prev_idxr"   r#   rO     r   z*CUDADeviceVariable.enter.<locals>.<lambda>rg   )	r,   r   _exchange_devicer1   r<   rC   r   r3   r   r?   r"   r  r#   rA     s   
zCUDADeviceVariable.enterc                 C   rh   )Nz
torch.cudar"   r    r"   r"   r#   rI     rk   zCUDADeviceVariable.module_namec                 C   rh   )Nr  r"   r    r"   r"   r#   rJ     rk   zCUDADeviceVariable.fn_namern   r   rm   )r'   r(   r)   r*   r   r>   r7   rG   rA   rI   rJ   rq   r"   r"   r9   r#   r   k  s    


r   c                       sR   e Zd ZdZee ejZe	dddZ
dd fdd	Zd
d ZdddZ  ZS )TorchFunctionDisableVariablez>represents whether torch function overrides are enabled or notr@   r   c                 K   s8   t ddg| jjgd|}|| dg ||  |S )NFr   r"   )r
  rC   torch_function_enabledr;   r<   r   r"   r"   r#   r>     s   
z#TorchFunctionDisableVariable.createNr4   c                    &   t  jd||d| t| j d S r   r   r   r9   r"   r#   r7        z%TorchFunctionDisableVariable.__init__c                 C      t jd S r   r	   r=   r>   r?   r"   r"   r#   rA     rv   z"TorchFunctionDisableVariable.enterc                 C   s<   t |dksJ |d |j_|d |j_|j|d  d S rc   )r[   symbolic_torch_function_statetorch_function_subclass_enabledtorch_function_mode_enabledrC   set_torch_function_state)r!   r@   r   r"   r"   r#   r;     s   z'TorchFunctionDisableVariable._call_funcrn   r   rm   )r'   r(   r)   r*   r   r   r   TORCH_FUNCTION_STATEr   r   r>   r7   rA   r;   rq   r"   r"   r9   r#   r
    s    r
  c                       sb   e Zd ZdZee ejZe	dddZ
dd fdd	Zd
d ZdddZdd Zdd Z  ZS )DeterministicAlgorithmsVariablezVrepresents torch.{are_deterministic_algorithms_enabled,use_deterministic_algorithms}()r@   r   c                 K   s8   t d|gt gd|}|| |g ||  |S r   )r  r,   $are_deterministic_algorithms_enabledr;   r<   r   r"   r"   r#   r>     s   
z&DeterministicAlgorithmsVariable.createNr4   c                    r  r   r   r   r9   r"   r#   r7     r  z(DeterministicAlgorithmsVariable.__init__c                 C   r  r   r  r?   r"   r"   r#   rA     rv   z%DeterministicAlgorithmsVariable.enterc                 C   sB   t |dksJ |d }|jdtjj|fi f tj| d S r   )r[   rC   r   r,   r   _set_deterministic_algorithmsr   r"   r"   r#   r;     s   z*DeterministicAlgorithmsVariable._call_funcc                 C   rh   r   r"   r    r"   r"   r#   rI     rk   z+DeterministicAlgorithmsVariable.module_namec                 C   rh   )Nuse_deterministic_algorithmsr"   r    r"   r"   r#   rJ     rk   z'DeterministicAlgorithmsVariable.fn_namern   r   rm   )r'   r(   r)   r*   r   r   r   DETERMINISTIC_ALGORITHMSr   r   r>   r7   rA   r;   rI   rJ   rq   r"   r"   r9   r#   r    s    


r  c                       sT   e Zd ZdZedddZdd fdd	Zd
d ZdddZdd Z	dd Z
  ZS )!DisabledSavedTensorsHooksVariablez;represents torch.autograd.graph.disable_saved_tensors_hook.r@   r   c                 K   s<   t d|gtjj gd|}|| |g ||  |S r   )r  r,   r   	_autograd/_saved_tensors_hooks_get_disabled_error_messager;   r<   r   r"   r"   r#   r>     s   

z(DisabledSavedTensorsHooksVariable.createNr4   c                       t  jd||d| d S r   r6   r7   r   r9   r"   r#   r7     s
   
z*DisabledSavedTensorsHooksVariable.__init__c                 C   r  r   r  r?   r"   r"   r#   rA     rv   z'DisabledSavedTensorsHooksVariable.enterc                 C   st   t |dksJ |d }|d ur&|jdtjjj|fi  tjj| d S |jdtjjjdi  tjj  d S )Nr   r   rg   r"   )r[   rC   r   r,   r   r  _saved_tensors_hooks_disable_saved_tensors_hooks_enabler   r"   r"   r#   r;     s   z,DisabledSavedTensorsHooksVariable._call_funcc                 C   rh   )Nztorch.autograd.graphr"   r    r"   r"   r#   rI     rk   z-DisabledSavedTensorsHooksVariable.module_namec                 C   rh   )Ndisable_saved_tensors_hooksr"   r    r"   r"   r#   rJ     rk   z)DisabledSavedTensorsHooksVariable.fn_namern   r   rm   )r'   r(   r)   r*   r   r>   r7   rA   r;   rI   rJ   rq   r"   r"   r9   r#   r    s    
r  c                       sN   e Zd Zedd Zdd fddZdd	d
Zdd Zdd Zdd Z	  Z
S )AutocastModeVariablec                 C   s   | t jjjt jjjt jjjfv sJ t| j|i |}|	  g }|
  dD ]5}|dkrG| t jjjt jjjfv rG| t jjju rDdnd}n|j| }t|trY||  q)|| q)t|fdd i|}|S )N)device_typedtyper   cache_enabledr#  r   cpur2   )r,   ampautocast_modeautocastr   r&  inspect	signaturebindapply_defaultsclear	argumentsrd   r   appendr   r"  )funcrF   r8   
bound_argsr1   keyargr   r"   r"   r#   r>     s*   
	


zAutocastModeVariable.createNr4   c                    r  r   r  r   r9   r"   r#   r7   ?  s   
zAutocastModeVariable.__init__r@   r   c                 G   s*   | j   |jdtjj| j jfi  d S r   )r3   r%   rC   r   r,   r'  _exit_autocastr   rE   r"   r"   r#   rG   E  s   
zAutocastModeVariable.exitc                    r  )Nc                      r  r   )r,   r'  r5  r"   r   r"   r#   rO   M  r   z,AutocastModeVariable.enter.<locals>.<lambda>rg   )	r,   r'  _enter_autocastr1   r<   rC   r   r3   r   r?   r"   r   r#   rA   K  s
   zAutocastModeVariable.enterc                 C   rh   )Nztorch.amp.autocast_moder"   r    r"   r"   r#   rI   R  rk   z AutocastModeVariable.module_namec                 C   rh   )Nr)  r"   r    r"   r"   r#   rJ   U  rk   zAutocastModeVariable.fn_namer   rm   rn   r   r"   r"   r9   r#   r"    s    

r"  c                       sF   e Zd ZdZdd fddZdd Zdd
dZdd Zdd Z  Z	S )NullContextVariablez>
    This class represents Python contextlib.nullcontext.
    Nr4   c                    s   t  jdd|i| d S Nr1   r"   r  )r!   r1   r8   r9   r"   r#   r7   ^     zNullContextVariable.__init__c                 C   r  r   r  r?   r"   r"   r#   rA   a  rv   zNullContextVariable.enterr@   r   c                 G   r  r   r  rE   r"   r"   r#   rG   d  rv   zNullContextVariable.exitc                 C   rh   N
contextlibr"   r    r"   r"   r#   rI   g  rk   zNullContextVariable.module_namec                 C   rh   Nnullcontextr"   r    r"   r"   r#   rJ   j  rk   zNullContextVariable.fn_namer   rm   rn   )
r'   r(   r)   r*   r7   rA   rG   rI   rJ   rq   r"   r"   r9   r#   r7  Y  s    
r7  c                       sL   e Zd ZdZd fddZdd Zdd
dZdd Zdd Zdd Z	  Z
S )ProfilerContextVariablea~  
    This class represents a set of torch profiler context objects, where Dynamo
    ignores all the side-effects in the __init__, __enter__ and __exit__ methods
    by treating the object mostly as a `contextlib.nullcontext`, except for edge
    cases like the `__enter__` method which returns the object itself rather
    than `None`, per implementation of the torch objects.
    r4   Nc                    s   t  jddd i| d S r8  r  )r!   r8   r9   r"   r#   r7   w  r9  z ProfilerContextVariable.__init__c                 C   s   | S r   r"   r?   r"   r"   r#   rA   z  rk   zProfilerContextVariable.enterr@   r   c                 G   r  r   r  rE   r"   r"   r#   rG   }  rv   zProfilerContextVariable.exitc                 C   rh   r:  r"   r    r"   r"   r#   rI     rk   z#ProfilerContextVariable.module_namec                 C   rh   r<  r"   r    r"   r"   r#   rJ     rk   zProfilerContextVariable.fn_namec                 C      t dt| dg tjd d S )Nz2torch.profiler object escaped from compiled regionzXDynamo doesn't support compiling a region that returns a torch.profiler context manager.gb_typecontextexplanationhintsr   strr   SUPPORTABLE)r!   r   r"   r"   r#   r\     s   
z#ProfilerContextVariable.reconstructrm   rn   )r'   r(   r)   r*   r7   rA   rG   rI   rJ   r\   rq   r"   r"   r9   r#   r>  n  s    
r>  c                       s@   e Zd ZedddZdd fddZd	d
 ZdddZ  ZS )StreamContextVariabler@   r   c              	   K   sN   ddl m} t|jj}|t| | jd|di }td|g|g|jd|S )Nr   wrap_fx_proxy_clsrg   r   )r1   r2   r  r"   )	builderrJ  r   r  current_streamStreamVariablerC   create_proxyrH  )r@   r   r8   rJ  current_stream_methodrL  r"   r"   r#   r>     s.   
zStreamContextVariable.createNr4   c                    s>   t  jd||d| || _t| jj| _t| jj| _d S r   )r6   r7   r  r   
set_stream_set_stream_by_idset_stream_id)r!   r1   r  r2   r8   r9   r"   r#   r7     s   zStreamContextVariable.__init__c                    s    j d  d ur|jd j j d  fi  n j d j}|jd j|j|j|j	fi    j d j  
| fdd d S )Nr   rg   c                      s      jd jS )Nr   )rP  r2   rs   r"   r    r"   r#   rO     r   z-StreamContextVariable.enter.<locals>.<lambda>)r1   r   rC   rN  rP  rs   rR  	stream_iddevice_indexr#  r<   )r!   r@   streamr"   r    r#   rA     s    zStreamContextVariable.enterc                 G   s.   |j d| j| jd  fi  | j  d S )Nrg   r   )rC   rN  rP  r2   r   r3   r%   rE   r"   r"   r#   rG     s   zStreamContextVariable.exitrn   r   rm   )	r'   r(   r)   r   r>   r7   rA   rG   rq   r"   r"   r9   r#   rH    s    rH  c                       sT   e Zd ZdZedd Zedd Zd fdd	Zd
d ZdddZ	dd Z
  ZS )PreserveVersionContextVariablez?
    Wraps torch.autograd._unsafe_preserve_version_counter
    c                    sX   t |tjrt fdd|fD }t|g}nt fdd|jD }t||S )Nc                       g | ]}|  d qS _versionvar_getattrrT   r~   r@   r"   r#   rW         zNPreserveVersionContextVariable._create_lambda_from_tensors.<locals>.<listcomp>c                    rW  rX  rZ  r\  r]  r"   r#   rW     r^  )rd   r	   TensorVariableTupleVariabler   rV  )r@   tensorsversionsr"   r]  r#   _create_lambda_from_tensors  s   
z:PreserveVersionContextVariable._create_lambda_from_tensorsc                    s   t  fddS )Nc                    s   t  | S r   )rV  rc  )ra  r]  r"   r#   rO     s    z<PreserveVersionContextVariable.constructor.<locals>.<lambda>)r	   LambdaVariabler]  r"   r]  r#   constructor  s   
z*PreserveVersionContextVariable.constructorr4   Nc                    st   | dd  t jdi | || _|| _t| jtjr$t| jg| _t| jtj	tj
fr8t| jg| _d S d S r8  )
setdefaultr6   r7   ra  prev_versionsrd   r	   r_  r`  r=   r   )r!   ra  rg  r8   r9   r"   r#   r7     s   z'PreserveVersionContextVariable.__init__c                 C   s   d S r   r"   r?   r"   r"   r#   rA     rk   z$PreserveVersionContextVariable.enterr@   r   c                 G   s(   ddl m} t||| j| jgi S )Nr   )_unsafe_set_version_counter)tensor_version_oprh  r	   TorchInGraphFunctionVariablerg   ra  rg  )r!   r@   rF   rh  r"   r"   r#   rG     s   z#PreserveVersionContextVariable.exitc                 C   r?  )NzLtorch.autograd._unsafe_preserve_version_counter escaped from compiled regionzyDynamo doesn't support compiling a region that returns a torch.autograd._unsafe_preserve_version_counter context manager.r@  rE  rK   r"   r"   r#   r\     s   
z*PreserveVersionContextVariable.reconstructrm   rn   )r'   r(   r)   r*   r   rc  re  r7   rA   rG   r\   rq   r"   r"   r9   r#   rV    s    


rV  c                       s   e Zd Zee ejZedddZ		d	d fddZ
d	d
 ZdddZ						d fddZdddZdd Zdd Z  ZS )&FSDPParamGroupUseTrainingStateVariabler@   r   c                 K   s"   t d||g|jjgd|}|S )N)param_group_varr1   r2   r"   )rk  rs   _training_state)r@   rl  r   r8   r   r"   r"   r#   r>     s   z-FSDPParamGroupUseTrainingStateVariable.createNr4   c                    s,   t  jd||d| || _t| j d S r   )r6   r7   rl  r   r   )r!   rl  r1   r2   r8   r9   r"   r#   r7     s   z/FSDPParamGroupUseTrainingStateVariable.__init__c                 C   r   r   r   r?   r"   r"   r#   rA   &  r&   z,FSDPParamGroupUseTrainingStateVariable.enterc                 G   r   r   r   rE   r"   r"   r#   rG   *  r&   z+FSDPParamGroupUseTrainingStateVariable.exitrF   ra   r8   rb   c                    r   r   r   rf   r9   r"   r#   rg   .  r   z4FSDPParamGroupUseTrainingStateVariable.call_functionc                 C   s\   t |dksJ |d }| jjj|kr,| j|dtjdt|fi  || jj_d S d S )Nr   r   __setattr__rm  )	r[   rl  rs   rm  call_methodr	   r=   r>   EnumVariabler   r"   r"   r#   r;   7  s   
	z1FSDPParamGroupUseTrainingStateVariable._call_funcc                 C   rh   )NzDtorch.distributed.fsdp._fully_shard._fsdp_param_group.FSDPParamGroupr"   r    r"   r"   r#   rI   F  rk   z2FSDPParamGroupUseTrainingStateVariable.module_namec                 C   rh   )Nuse_training_stater"   r    r"   r"   r#   rJ   I  rk   z.FSDPParamGroupUseTrainingStateVariable.fn_namern   r   rm   r   )r'   r(   r)   r   r   r   FSDP_TRAINING_STATEr   r   r>   r7   rA   rG   rg   r;   rI   rJ   rq   r"   r"   r9   r#   rk    s(    
	

	rk  c                       st   e Zd ZdZedddZ	ddeejj	j
 ddf fd	d
Zedd Zdd ZdddZdd Zdd Z  ZS )SDPAKernelVariablez)represents torch.nn.attention.sdpa_kernelr@   r   c                 K   s.   t |tjjjr|g}td|d d|}|S r   )rd   r,   nn	attention
SDPBackendrs  )r@   backendsr8   r   r"   r"   r#   r>   P  s   zSDPAKernelVariable.createNr1   r4   c                    r  r   r  r   r9   r"   r#   r7   [  s
   
zSDPAKernelVariable.__init__c                    s    fdd|D }|S )Nc                    s(   g | ]} j d tjjj|jfi qS )rg   )rC   r   r,   rt  ru  _backend_from_stringname)rT   backendr]  r"   r#   rW   h  s    z9SDPAKernelVariable._backends_to_nodes.<locals>.<listcomp>r"   )r@   rw  nodesr"   r]  r#   _backends_to_nodese  s   
	z%SDPAKernelVariable._backends_to_nodesc                    sf   t jj  _ | fdd t jj j  | j}|j	
dt jjj|fi  tjd S )Nc                      r   r   )r,   rt  ru  _sdpa_kernelprev_backendsr"   r    r"   r#   rO   v  r   z*SDPAKernelVariable.enter.<locals>.<lambda>rg   )r,   rt  ru  _cur_sdpa_kernel_backendsr~  r<   r}  r1   r|  rC   r   r	   r=   r>   )r!   r@   r4  r"   r    r#   rA   s  s   zSDPAKernelVariable.enterc                 G   s>   | j   | || j}|jdtjjj	|fi  t
jd S r   )r3   r%   r|  r~  rC   r   r,   rt  ru  r}  r	   r=   r>   )r!   r@   rF   r4  r"   r"   r#   rG     s   
zSDPAKernelVariable.exitc                 C   rh   )Nztorch.nn.attentionr"   r    r"   r"   r#   rI     rk   zSDPAKernelVariable.module_namec                 C   rh   )N_sdpa_kernel_variadicr"   r    r"   r"   r#   rJ     rk   zSDPAKernelVariable.fn_namern   r   )r'   r(   r)   r*   r   r>   listr,   rt  ru  rv  r7   r|  rA   rG   rI   rJ   rq   r"   r"   r9   r#   rs  M  s     


rs  c                       sP   e Zd Zd fddZdd Z					
		d fddZdd Zdd Z  ZS )rM  r4   Nc                    sf   |d urd|j jv r|j jd |ksJ |jj|jksJ dt jdi | || _|| _|| _d S )Nexample_valuez.stream value is not equal to the passed devicer"   )r   metar  ru   r6   r7   r   rs   )r!   r   rs   r  r8   r9   r"   r#   r7     s   
zStreamVariable.__init__c                 C   s   t jS r   )r,   Streamr    r"   r"   r#   python_type     zStreamVariable.python_typerF   ra   r8   rb   r   c           	         s6  t | j|sJ d| ddlm}m} ddlm} |dv r5|jjd|g|| g| |R   t	
d S |dkrP|t	j
||jjd|g|| g| |R  d	S |d
krj|t||jjd|g|| g| |R  d	S ||v rt|dkr|s|d }t|tst	j
tS t	j
|| | j|jS t ||||S )Nzno stream method found named r   )cmp_name_to_op_mappingproxy_args_kwargsr   rI  )wait_streamsynchronize
wait_eventro  query
target_clsr@   r   record_eventr   )hasattrrs   utilsr  r  rK  rJ  rC   rN  r	   r=   EventVariabler[   rd   rM  r>   NotImplementedr6   ro  )	r!   r@   ry  rF   r8   r  r  rJ  otherr9   r"   r#   ro    sL   

zStreamVariable.call_methodc                 C      | j S r   r   r    r"   r"   r#   r     r  zStreamVariable.as_proxyc                 C   s@   | j rJ d| j }|jj|| j}||j|dd d S )N_stream_Tadd)ry   r  r@   rC   install_global_by_idrs   append_outputcreate_load_globalr!   rL   prefixry  r"   r"   r#   r\     s   
zStreamVariable.reconstructrm   rF   ra   r8   rb   r4   r   )	r'   r(   r)   r7   r  ro  r   r\   rq   r"   r"   r9   r#   rM    s    -rM  c                       sD   e Zd Zd fddZ							dd
dZdd Zdd Z  ZS )r  r4   Nc                    sJ   |d urd|j jv r|j jd |ksJ t jdi | || _|| _d S )Nr  r"   )r   r  r6   r7   r   rs   )r!   r   rs   r8   r9   r"   r#   r7     s
   
zEventVariable.__init__rF   ra   r8   rb   r   c                 C   s   ddl m} ddlm} |dv r&|jjd|g|| g| |R   td S |dkrA|tj||jjd|g|| g| |R  dS td	t	|d
| dfg t
jd d S )Nr   )r  r   rI  )waitrecordr  ro  r  r  z#Unsupported torch.cuda.Event methodz4Dynamo doesn't support tracing the torch.cuda.Event.zC method. We currently support wait, record, synchronize, and query.r@  )r  r  rK  rJ  rC   rN  r	   r=   r   rF  r   rG  )r!   r@   ry  rF   r8   r  rJ  r"   r"   r#   ro    s8   


zEventVariable.call_methodc                 C   r  r   r  r    r"   r"   r#   r     r  zEventVariable.as_proxyc                 C   s8   | j rJ d}|jj|| j}||j|dd d S )N_eventTr  )ry   r@   rC   r  rs   r  r  r  r"   r"   r#   r\     s   
zEventVariable.reconstructrm   r  )r'   r(   r)   r7   ro  r   r\   rq   r"   r"   r9   r#   r    s    
$r  c                       s\   e Zd ZdhejZdeeef ddf fddZ					
				dddZ	dd Z
  ZS )WithExitFunctionVariabletargetr   r4   Nc                    s4   t  jdi | t|ttfsJ || _|| _d S r5   )r6   r7   rd   r/   rr   r   r  )r!   r   r  r8   r9   r"   r#   r7   #  s   
z!WithExitFunctionVariable.__init__r@   r   rF   ra   r8   rb   r   c                 C   s   |rJ | j j|g|R  S r   )r   rG   rf   r"   r"   r#   rg   0  s   z&WithExitFunctionVariable.call_functionc                    s   | j    jjjrPtjdkr% td tjdk r% tddd  	 fdd| j j
D   	tt| j j
d	  t| j  td
 d S d S )N)      	PUSH_NULL)r     SWAPr   )r4  c                    rP   r"   rQ   rS   rV   r"   r#   rW   D  rX   z8WithExitFunctionVariable.reconstruct.<locals>.<listcomp>FPOP_TOP)r   rM   r@   rC   partial_convertsysversion_infor  r   rZ   r1   r
   r[   r   r  rK   r"   rV   r#   r\   9  s   


z$WithExitFunctionVariable.reconstructro   )r'   r(   r)   r   rp   r   r/   rr   r7   rg   r\   rq   r"   r"   r9   r#   r    s(    

	r  )Gr*   dataclassesr*  r  r   typingr   r   r   r   torch._Cr,   torch._guardsr    r   r	   bytecode_transformationr
   r   r   device_interfacer   excr   guardsr   r   ry   r   r   baser   	functionsr   r   r   r   r   user_definedr   torch._dynamo.symbolic_convertr   	dataclassr   r/   rr   r   r   r   r   r   r   r   r   r   r   r   r
  r  r  r"  r7  r>  rH  rV  rk  rs  rM  r  r  r"   r"   r"   r#   <module>   s^   N+($)&&'1:22!,4;#@?=IN9