o
    Ih-                     @   sn  d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZ d dlm	Z	 d dl
mZ d dlmZmZmZmZ d dlmZ edZe ZedZee ee eej d	e_ed
ZedZ G dd de!Z"ddee# de$fddZ%ddee# de#fddZ&dee$ de$fddZ'G dd dZ(	ddee( dedeeee f geeee  f f fddZ)dS )    N)Sequence)Lock)default_timer)AnyCallableOptionalTypeVar)	ParamSpecstrobelight_function_profilerzB%(name)s, line %(lineno)d, %(asctime)s, %(levelname)s: %(message)sF_P_Rc                   @   s   e Zd ZdZdS )StrobelightCLIProfilerErrorzC
    Raised when an error happens during strobelight profiling
    N)__name__
__module____qualname____doc__ r   r   \/var/www/vscode/kcb/lib/python3.10/site-packages/torch/_strobelight/cli_function_profiler.pyr       s    r   pidreturnc                 C   s    d}| pt  } t || S )zFReturns the link to the process's namespace, example: pid:[4026531836]z/proc/{}/ns/pid)osgetpidreadlinkformat)r   PID_NAMESPACE_PATHr   r   r   _pid_namespace_link&   s   r   c                 C   s.   | pt  } t| }t||dd d S )z"Returns the process's namespace id[   )r   r   r   intfind)r   linkr   r   r   _pid_namespace-   s   r"   commandc                 C   s
   d | S )N )join)r#   r   r   r   _command_to_string4   s   
r&   c                   @   s   e Zd ZdZe Zddddddddddd	
d
ededede	dedede
ee	  de
ee	  dedefddZd+ddZd,deddfddZd+ddZd+dd Zd!eddfd"d#Zdefd$d%Zd&eeef d'ejd(ejde
e fd)d*ZdS )-StrobelightCLIFunctionProfilera  
    Note: this is a Meta only tool.

    StrobelightCLIFunctionProfiler can be used to profile a python function and
    generate a strobelight link with the results. It works on meta servers but
    does not requries an fbcode target.
    When stop_at_error is false(default), error during profiling does not prevent
    the work function from running.

    Check function_profiler_example.py for an example.
    FiX  g    cAzpytorch-strobelight-ondemand<   N   )
stop_at_errormax_profile_duration_secsample_eachrun_user_nametimeout_wait_for_running_sectimeout_wait_for_finished_secrecorded_env_variablessample_tagsstack_max_lenasync_stack_max_lenr*   r+   r,   r-   r.   r/   r0   r1   r2   r3   c       
         C   s:   || _ || _|| _|| _|| _|| _d | _d | _|| _d S N)	r*   r+   r,   r-   r.   r/   current_run_idprofile_resultr1   )selfr*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r   r   r   __init__H   s   
z'StrobelightCLIFunctionProfiler.__init__r   c                 C   s   t  }t|}ddddddddt| j d	t| jd
  d| d| g}| jr7|d |d| j t	
dt| tj|dd}|jd}t	
d| |jdkr^td| td| }rpt|d| _d S td| )Nstrobeclientrunz
--profilerpyperfz--eventcyclesz--asyncz--sample-intervalz--duration-msi  z--pid:z--sample-tags,running command: %sTcapture_outpututf-8output:
{%s}r   z:failed to start strobelight profiling, error in run_async:zINFO Run Id: (-?\d+)r   z9failed to start strobelight profiling, unexpected result )r   r   r"   r   r,   r+   r1   appendr%   loggerdebugr&   
subprocessr:   stderrdecode
returncoder   researchgroupr5   )r7   	processId	namespacer#   resultoutputmatchr   r   r   
_run_asyncb   sB   


z)StrobelightCLIFunctionProfiler._run_asyncr   counterc                 C   s   |dkrt dddd| j g}tdt| tj|dd}|jd	}td
| |j	dkr7t d| t
d| }rd|d}|dkrJd S |dkr\td | |d  d S t d| dt d| d)N   z*wait_for_running called more than 20 timesr9   getRunStatus--run-idr?   Tr@   rB   rC   r   zAfailed to start strobelight profiling, error in wait_for_running:zProfile run status: (.*)r   RUNNING	PREPARING
   zunexpected z phaseunexpected output
: r$   )r   r5   rE   rF   r&   rG   r:   rH   rI   rJ   rK   rL   rM   timesleep_wait_for_running)r7   rT   r#   rP   rQ   rR   current_statusr   r   r   r^      s.   


z0StrobelightCLIFunctionProfiler._wait_for_runningc                 C   s   dddt | jg}tdt| tj|dd}|jd}td| |j	d	kr0t
d
| td| }rL|d}|drDd S t
d| dt
d| d)Nr9   stopRunrW   r?   Tr@   rB   rC   r   z<failed to stop strobelight profiling, return code is not 0 :INFO ::1:(.*)r   zSuccess!z*failed to stop strobelight profiling, got z resultr[   r$   )strr5   rE   rF   r&   rG   r:   rH   rI   rJ   r   rK   rL   rM   __contains__)r7   r#   rP   rQ   rR   r_   r   r   r   	_stop_run   s"   



z(StrobelightCLIFunctionProfiler._stop_runc                 C   s   dddt | jg}tdt| tj|dd}|jd}td| |j	d	kr0t
d
| td| }rY|d}|drMtd |   d S |dsYt
d| g | _td|D ]}|  j|d	 7  _t|d	  qbd S )Nr9   rV   rW   r?   Tr@   rB   rC   r   z<failed to extract profiling results, return code is not 0 : ra   r   zProfile run status: PROCESSINGrZ   z!Profile run finished with SUCCESSz9failed to extract profiling results, unexpected response zF(Total samples(.*)|GraphProfiler(.*)|Icicle view \(python stack\)(.*)))rb   r5   rE   rF   r&   rG   r:   rH   rI   rJ   r   rK   rL   rM   rc   r\   r]   _get_resultsr6   findallinfo)r7   r#   rP   rQ   rR   r_   itemr   r   r   re      s6   




z+StrobelightCLIFunctionProfiler._get_resultscollect_resultsc                 C   sX   z|    td td |sW d S |   W d S  ty+   tjddd Y d S w )Nzstrobelight profiling stoppedzcollection stoppedzerror during stop_strobelightTexc_info)rd   rE   rg   rF   re   	Exceptionwarning)r7   ri   r   r   r   _stop_strobelight_no_throw   s   

z9StrobelightCLIFunctionProfiler._stop_strobelight_no_throwc                 C   sj   d}z|    d}td| j |   td W dS  ty4   tjddd |r1| jdd Y dS w )NFTzstrobelight run id is: %szstrobelight profiling runningzerror during start_strobelight:rj   ri   )rS   rE   rg   r5   r^   rl   rm   rn   )r7   strobelight_startedr   r   r   _start_strobelight   s   
z1StrobelightCLIFunctionProfiler._start_strobelightwork_functionargskwargsc              
   O   s   d | _ d | _tjd }r|s#| jrtdtd ||i |S | 	 }|sC| jr5tj
  td||i |}tj
  |S z*td t }||i |}t }|| }	td|	 | jdd tj
  |W S  ty }
 ztjddd	 | jdd tj
  |
d }
~
ww d S )
NFzconcurrent runs not supportedz%failed to start strobelight profilingzcollection startedzwork function took %s secondsTro   zwork function throw exceptionrj   )r5   r6   r'   _lockacquirer*   r   rE   rm   rq   releaserF   timerrg   rn   rl   )r7   rr   rs   rt   lockedstartedrP   startend
total_timeerrorr   r   r   profile  sH   





z&StrobelightCLIFunctionProfiler.profile)r   N)r   )r   r   r   r   r   ru   boolr   floatrb   r   listr8   rS   r^   rd   re   rn   rq   r   r   r   rs   rt   r   r   r   r   r   r'   8   sj    
	



)



r'   profilerrt   c                    s@    s	t di | dtttf dtttt f f fdd}|S )Nrr   r   c                    s2   t  dtjdtjdtt f fdd}|S )Nrs   rt   r   c                     s    j g| R i |S r4   )r   )rs   rt   )r   rr   r   r   wrapper_function;  s   z@strobelight.<locals>.strobelight_inner.<locals>.wrapper_function)	functoolswrapsr   rs   rt   r   r   )rr   r   r   )rr   r   strobelight_inner8  s   &z&strobelight.<locals>.strobelight_innerr   )r'   r   r   r   r   )r   rt   r   r   r   r   strobelight2  s   
	r   r4   )*r   loggingr   rK   rG   r\   collections.abcr   	threadingr   timeitr   rx   typingr   r   r   r   typing_extensionsr	   	getLoggerrE   StreamHandlerconsole_handler	Formatter	formattersetFormatter
addHandlersetLevelINFO	propagater   r   rl   r   r   rb   r   r"   r&   r'   r   r   r   r   r   <module>   sH   


 | 