o
    IhWA                     @   s  d Z ddlZddlZddlmZ ddlZddlmZ ddlm	Z	 ddl
mZmZ ddlmZmZmZmZmZmZ dd	lmZ d
dlmZ d
dlmZ d
dlmZ d
dlmZ d
dlmZ d
dl m!Z! erjddl"m#Z# G dd de$Z%G dd de$Z&ee'dZ(dd Z)G dd de!Z*dS )a
  
This module implements variable tracking for PyTorch optimizers during Dynamo tracing.

The OptimizerVariable class provides specialized handling for optimizer instances by:
- Optimizing the tracing of expensive optimizer initialization
- Managing optimizer state and parameter group tracking
- Handling tensor sources and guards for optimizer state tensors
- Supporting CUDA graph execution through static tensor address management
- Providing special handling for parameter gradients and optimizer state tensors

Key features include:
- Efficient initialization tracing via _init_group optimization
- Automatic marking of optimizer state tensors as static for CUDA graphs
- Proper source tracking for parameter groups, gradients, and state tensors
- Guard installation for optimizer state structure
- Support for both CPU and GPU tensor handling
- Cleanup of static tensor references via finalizers

The module integrates with Dynamo's broader tracing system while providing
optimizer-specific optimizations and safety guarantees.
    N)TYPE_CHECKING)getArtifactLogger)tree_map_only   )GuardBuilderinstall_guard)
AttrSourceConstDictKeySourceDictGetItemSourceGetItemSourceGlobalWeakRefSource
GradSource)GLOBAL_KEY_PREFIX   )VariableTracker)ConstantVariable)ConstDictVariable)ListVariable)GetAttrVariable)UserDefinedObjectVariable)InstructionTranslatorc                   @      e Zd ZdS )ArgMappingExceptionN__name__
__module____qualname__ r   r   U/var/www/vscode/kcb/lib/python3.10/site-packages/torch/_dynamo/variables/optimizer.pyr   7       r   c                   @   r   )GuardInstallExceptionNr   r   r   r   r   r    ;   r   r    
perf_hintsc                 C   sN   ddl m} | jr%|| jjd}tjj| d u}|r#|p"|j	
| S |S dS )Nr   )get_managerFT)torch._inductor.cudagraph_treesr"   is_cudadeviceindextorch_dynamoutilsget_static_address_typecurrent_node_is_cuda_graph_recorded_tensor)xr"   manageris_static_addressr   r   r   _is_static_for_cudagraphsB   s   
r0   c                       s   e Zd ZdddhejZ			d#	d$ fddZ				
			d% fddZd& fddZdd Zdd Z	dd Z
dd Zdd Zd&ddZ		d&dd Zd!d" Z  ZS )'OptimizerVariablegrad_to_sourcetensor_to_sourcestatic_tensor_namesNreturnc                    s8   t  j|fi | |pi | _|pi | _|pt | _d S N)super__init__r2   r3   setr4   )selfvaluer2   r4   r3   kwargs	__class__r   r   r8   \   s   

zOptimizerVariable.__init__argslist[VariableTracker]r<   dict[str, VariableTracker]r   c           
   
      s   |dkr]zD|  | |   | j|i |\}}| jj|i |}| | | ||||| dt| j }||| j | 	| t
|W S  ttfy\ }	 zW Y d}	~	nd}	~	ww t ||||S )zVThis is an optimization to avoid tracing the very slow initialization of the optimizer_init_group__optimizer_N)graph_break_if_pending_mutationmove_step_if_cpuget_python_argsr;   rB   map_sources_and_install_guardsupdate_list_argsidstore_global_weakref_by_idcreate_finalizerr   creater   r    r7   call_method)
r:   txnamer?   r<   py_args	py_kwargsret_valmangled_name_r=   r   r   rM   i   s"   


zOptimizerVariable.call_methodrN   r   c                    sn   |dv rt | |t| j|dS |dkr0ddlm} | jjD ]}|d D ]}|| q#q| | t 	||S )N)rB   step)sourceparam_groupsr   mark_static_addressparams)
r   r   rV   
decoratorsrY   r;   rW   _set_capturabler7   var_getattr)r:   rN   rO   rY   grouppr=   r   r   r]      s   

zOptimizerVariable.var_getattrc                 C   s\   | j jD ]'}|d D ] }|jj}|jt|d }|r*||r*ddlm	} |dq
qd S )NrZ   r   )UnsupportedzPending mutation on parameter)
r;   rW   outputside_effectsid_to_variablegetrI   has_pending_mutationexcr`   )r:   rN   gr_   rb   variabler`   r   r   r   rD      s   z1OptimizerVariable.graph_break_if_pending_mutationc           	         s   ddl m}  fdd} jjD ]
}||rd|d< q jo#t jd}|t| jj|}|j	D ]}t
td}td|j	|< q3d S )Nr   LazyVariableTrackerc                    sJ   d}d}|  dg D ]}||jp|jM }|| jjvM }q
d| v o$|o$|S )NTrZ   
capturable)rd   r$   is_xpur;   state)r^   all_uninitializedall_gpur_   r:   r   r   safe_to_set_capturable   s   zAOptimizerVariable._set_capturable.<locals>.safe_to_set_capturableTrk   rW   ) rj   r;   rW   rV   r   realize_allr   builditemsr   _HashableTrackerr   rL   )	r:   rN   rj   rq   r^   rV   param_groups_vtparam_group_vtkeyr   rp   r   r\      s    
z!OptimizerVariable._set_capturablec                    s<   fdd  fdd|D } fdd|  D }||fS )z9Get python values equivalent to the variable tracker argsc                    sl   t | tr	|  S t | tr| jsg S t | tr4t | jtr4t | jjt	r4| jjj
dkr4 jj| jj S t)NrW   )
isinstancer   as_python_constantr   ru   r   rV   r   baser   memberr;   rW   r&   r   )argrp   r   r   map_arg   s   

z2OptimizerVariable.get_python_args.<locals>.map_argc                    s   g | ]} |qS r   r   ).0r~   r   r   r   
<listcomp>       z5OptimizerVariable.get_python_args.<locals>.<listcomp>c                    s   i | ]	\}}| |qS r   r   )r   kvr   r   r   
<dictcomp>   s    z5OptimizerVariable.get_python_args.<locals>.<dictcomp>)ru   )r:   r?   r<   new_args
new_kwargsr   )r   r:   r   rF      s   z!OptimizerVariable.get_python_argsc                 C   s@   | j j D ]\}}d|v r|d jr|d |j|d< qd S )NrU   )r;   rm   ru   is_cputor%   )r:   r_   rm   r   r   r   rE      s
   z"OptimizerVariable.move_step_if_cpuc                    s  ddl m  ddlm} i | _i | _ fdd}ttj|| j	j
 | jo)t| jd}|t|| j	j|}| jo>t| jd}t|| j	j
|}|  |jj|  t| j	j|jD ]\}}	t|d	 d
kr|d	 D ]6}
|
jd urd }t| j	j
 D ]\}}||
u r|} nq~|r|t|| j	j
|
 t|t||  nqm|	|t d	}d}g }tt|d	 |!|D ]3\}\}}|j}|| j|< t"|d}|jd ur|| j|j< t#|jsd}|$| qt%|&t'j( q|st)*t+j,rdd |D }t)-d| q]t| j	j
 D ]D\}\}}t|t||}|jj|  t| D ]%\}\}}t.|tjrR|| jvrR|| jvrRt|t||| j|< q.qd S )Nr   rX   r   ri   c                    s    |  d S r6   r   )r-   rX   r   r   mark_static   s   zEOptimizerVariable.map_sources_and_install_guards.<locals>.mark_staticrW   rm   rZ   r   TgradFc                 S   s   g | ]}|  qS r   rO   )r   srcr   r   r   r   A  r   zDOptimizerVariable.map_sources_and_install_guards.<locals>.<listcomp>)zGrad tensors %s will be copied during cudagraphs execution.If using cudagraphs and the grad tensor addresses will be the same across runs, use torch._dynamo.decorators.mark_static_address to elide this copy.)/r[   rY   lazyrj   r2   r3   r   r'   Tensorr;   rm   rV   r   rs   r   rt   rW   realizera   guard_on_key_orderaddrO   zipru   lenr   	enumeratekeysr
   r	   getitem_constr   rL   unpack_var_sequencer   r0   appendr   
make_guardr   CONSTANT_MATCHperf_hint_logisEnabledForloggingDEBUGwarningrz   )r:   rN   rj   r   params_groups_sourcerw   state_sourcestate_vtr^   group_vtparam	key_indexir   	params_vt
all_staticnon_static_gradsp_indr_   p_vtparam_sourcegrad_sourceidxr;   p_state_source	inner_idxr   r   rX   r   rG      s   











z0OptimizerVariable.map_sources_and_install_guardsc                 C   s   ddl m} || jv r!|| | j| }| j|j|  n%|| jv r,| j| }n|| |	t
|}t|}| j|j|  t|||S )z%Wrap state tensor in a TensorVariabler   rX   )r[   rY   r3   r4   r   ra   module_key_namerO   r2   rJ   r   r   r   rt   )r:   rN   tensor_valuerY   rV   global_namer   r   r   wrap_tensor\  s   


zOptimizerVariable.wrap_tensorc              	   C   s   t ||D ]G\}}t|trLt|tsJ dt|D ]0\}}	|jj| t|	tj	r7|j
| ||	 q|jo?t|j|}
|j
t||	|
 qqdS )z7Update the args and kwargs to the traced optimizer callz-py_arg should be a list in optimizer variableN)r   rz   r   listr   ra   rb   mutationr'   r   ru   r   r   rV   r   r   rt   )r:   rN   r?   r<   rP   rQ   r~   py_argr   valrV   r   r   r   rH   v  s   
z"OptimizerVariable.update_list_argsc                    s4   | j  | j|jj fdd}|j| d S )Nc                    s     fdd}t | d S )Nc                     sJ   D ] }  j | d   j| d  jrj  jr"j  qd S r6   )_bufferspop_parametersparams_flatclearparams_flat_unwrap_subclassesr   )gmnames_to_deletetcr   r   clear_static_tensor_refs  s   

z\OptimizerVariable.create_finalizer.<locals>.init_finalizer.<locals>.clear_static_tensor_refs)weakreffinalize)r   r   r   r   r;   )r   r   init_finalizer  s   	z:OptimizerVariable.create_finalizer.<locals>.init_finalizer)r4   r;   ra   tracing_contextadd_graph_finalizer)r:   rN   r   r   r   r   rK     s
   z"OptimizerVariable.create_finalizer)NNN)r5   N)r?   r@   r<   rA   r5   r   )rN   r   )r   r   r   r   _nonvar_fieldsr8   rM   r]   rD   r\   rF   rE   rG   r   rH   rK   __classcell__r   r   r=   r   r1   T   s>    
! 
q
r1   )+__doc__r   r   typingr   r'   torch._loggingr   torch.utils._pytreer   guardsr   r   rV   r   r	   r
   r   r   r   r)   r   r|   r   constantr   dictsr   listsr   miscr   user_definedr   torch._dynamo.symbolic_convertr   	Exceptionr   r    r   r   r0   r1   r   r   r   r   <module>   s.    
