o I h6@s>ddlZddlZddlZddlZddlZddlmZddlmZddl m Z m Z ddl Z ddl Z ddlmZddlmZddlmmZddl mZddlmZddlmZdd lmZmZmZdd lm Z dd l!m"Z"m#Z#m$Z$e%e&Z'd d Z(eddZ)edej*de fddZ+d1ddZ,ddZ-edej*de fddZ.Gdddej/Z0edej*de fddZ1edd Z2d!d"Z3e j4j5Z5e5j6e5j7e5j8e5j9e5j:e5j;e5je5j?e5j@e5jAe5jBe5jCjDe5jCjEe5jFe5jGe5jHe5jIe5jJe5jKe5jLhZMeeMZMed#d$ZNd%e e ejOffd&d'ZPd(d)ZQdaRd*d+ZSd,d-ZTd2d/d0ZUdS)3N)contextmanager)partial)CallableUnion)SymInt)get_decompositions) bind_symbols) aot_function aot_modulemake_boxed_compiler)strip_overloads)default_partition draw_graph#min_cut_rematerialization_partitioncCs4|jjdtjjjdD]}tjjj|_q ||S)N call_functionoptarget) graph find_nodestorchopsaten_to_copytor recompile)fx_gnoderN/var/www/vscode/kcb/lib/python3.10/site-packages/torch/_functorch/compilers.py _canonicalize$s   r!c cs6tjd}z dVWtj|dStj|w)NF)r_C_jit_set_autocast_mode)old_jit_autocast_flagrrr _disable_jit_autocast-s  r%rreturncCs:tt||jjdtjjjdD]}t|j dkr.t|j dkr.d|j vr.tjjj |_ q|jj D]}i}|j D]\}}t|tjrI|j}|||<q<||_ q3|j|tj|}tj|jtj|}tj|}tdd|Ds||Wd|SWd|S1swY|S)a Compiles the :attr:`fx_g` with Torchscript compiler. .. warning:: This API is experimental and likely to change. Args: fx_g(fx.GraphModule): The input Fx graph module to be compiled. Returns: Torch scripted model. rrr dtypecss|] }t|tjjVqdSN) isinstancer _subclasses FakeTensor).0trrr `szts_compile..N)r%r rrrrrrlenargskwargsrrnodesitemsr)devicetypelintrjitscriptr"_jit_pass_remove_mutationfreezeevaloptimize_for_inferenceany)rinpsr new_kwargskvfrrr ts_compile6s<  &          rCTcCst|jt|||d|S)N) clear_meta)printcoder)r_namerDrrr _draw_graph_compilees rIcCsttt|dS)NrH)r rrIrJrrr draw_graph_compileksrKcCs|S)z Returns the :attr:`fx_g` Fx graph module as it is. This is a no-op compiler and can be used to check accuracy. .. warning:: This API is experimental and likely to change. rrrGrrr nopos rMcs(eZdZfddZfddZZS)DebugInterpretercs$t|jg|R|_tj|dSr()rmodulesymbol_mappingsuperrun)selfr0 __class__rr rR}szDebugInterpreter.runc sfddfddfddfdd}t|}d |jvrlt|jd \}}t|\}}t|t|ksJJt|d t|ttt|||D]\}} t| t j s`qT||| fd d qT|S) Ncs:t|ts|St|jjj}|jsJ|t |Sr() r)rsympyexpandrexprxreplacerP is_numberint)nir)rSrr subst_symints z/DebugInterpreter.run_node..subst_symintcstfdd|DS)Nc3s|]}|VqdSr(r)r,r\r^rr r.szHDebugInterpreter.run_node..subst_symint_tuple..)tuple)nisr_rr subst_symint_tuplesz5DebugInterpreter.run_node..subst_symint_tuplecsT|dkr(t|jD]}||||kr'||dkr'dSq dS)Nrr FT)numelrangendimstridesize)abidxr_rr check_significant_stridessz.check_significant_stridesc st|sJ|j|jksJ|d|jd|j||ks=J|d|d|d|||}|s\J|d|d|d|dS)Nz:  != z aka )callabler'rgrf)nvrvdesc same_strides)rkrbrr checks ** *z(DebugInterpreter.run_node..checkvalrlcsddjS)Nzoutput z where )rPr)irSrr sz+DebugInterpreter.run_node..) rQrun_nodemetapytree tree_flattenr/ziprdr)rTensor) rSnrrr]n_vals_n_specr_vals_r_specrnrorT)rkrtrSr^rbr rvs    * zDebugInterpreter.run_node)__name__ __module__ __qualname__rRrv __classcell__rrrTr rN|s rNcCs t|jS)z Returns a (slow) interpreter over the FX graph module that also checks various debugging properties (e.g., that tracing strides matched real strides.) )rNrRrLrrr debug_nops rcCs(t|tj|}tj|}|Sr()r rr7r8r:r;)rrGrBrrr simple_ts_compiles rcCs t|tSr()r r)rBrrr nnc_jits rcCst|j|Sr()rErFrLrrr print_compiles rfncKsFttttd}||t|tjjrt|fi|St |fi|S)a Wrapper function over :func:`aot_function` and :func:`aot_module` to perform memory efficient fusion. It uses the :func:`min_cut_rematerialization_partition` partitioner to perform efficient recomputation. It uses NVFuser to compile the generated forward and backward graphs. .. warning:: This API is experimental and likely to change. Args: fn (Union[Callable, nn.Module]): A Python function or a ``nn.Module`` that takes one ore more arguments. Must return one or more Tensors. **kwargs: Any other overrides you want to make to the settings Returns: Returns a ``Callable`` or ``nn.Module`` that retains the eager behavior of the original :attr:`fn`, but whose forward and backward graphs have gone through recomputation optimizations, and the graphs have been compiled with nvfuser.  fw_compiler bw_compiler partition_fndecompositions) rCrdefault_decompositionsupdater)rnnModuler r )rr1configrrr memory_efficient_fusions rcCsH|dtddd|Ddddlm}||t||S)NfooaQ ############################################################## # To minimize FX graph, copy and paste the below and run it # ############################################################## import torch import torch.fx as fx from functorch.compile import minifier, check_nvfuser_subprocess, check_nvfuser_correctness_subprocess inps = cSsg|]}|j|jfqSr)shaper')r,rtrrr sz!debug_compile..a? inps = [torch.ones(shape, dtype=dtype, device='cuda') for (shape, dtype) in inps] from foo import FxModule mod = FxModule().cuda() with torch.jit.fuser("fuser2"): # check_nvfuser_subprocess can be replaced with check_nvfuser_correctness_subprocess minifier(fx.symbolic_trace(mod), inps, check_nvfuser_subprocess) r)FxModule) to_folderrErrcudarC)rr>rrrr debug_compiles    rc Csg}t|dU}t|}g}|D]B}t|dkr"|}|t}n,|\}}}} } | tjtjtj tj tjtj tt hvrFtj dd|| | d}ntj|| | d}||qWd|S1s_wY|S)zZ Return a random input for the given inputs meta generated from _save_fx_default. rbr r)r'r4N)openpickleloadr/randomrandrr[int32int64booluint8floatrandintappend) input_data_pathinputsrB inputs_metarwr5inputr_strider'r4rrr get_inputs3s6      rc sbddlm}fddfddfdd}fd d }fd d }||||||td S)aO The forward, backward, and joint computation graph will be stored in {folder_name}/{current_name}/{current_name}_forward_{graph_index}, {folder_name}/{current_name}/{current_name}_backward_{graph_index}, and {folder_name}/{current_name}/{current_name}_joint_{graph_index} respectively. The input shape of the graphs will be stored in the .input files. These files can be loaded with pickle, and is a list of format (type, shape, stride, dtype, device). In the case of type = int or float, it is just (type,). For joint graph input, it is a nested list [[],[]] where the two inner lists have the same format. If dump_example_input is True, example_inputs will be stored in .pt file. Since each function might produce multiple graphs, the graph_index is used to distinguish difference graphs r)aot_module_simplifiedcsg}t|dkr!t|dtr!||d7}||d7}|S|D](}t|tks1t|tkr:|t|fq#|t||j||j |j fq#|S)Nrr ) r/r)r`r5r[rrrrfr'r4)r0 input_metaarg)get_input_metarr rdsz(_save_fx_default..get_input_metacst|jjdkrttjd|tdSt|}|j t j j | |}tjddd|ddd|dt t|tddd|dtdd|dtddrt |ddd|dtdd|dtd dSdS) Nrz!No nodes in graph {%s}_{%s}_{%s}./T)exist_okrGz.inputwbz.pt)r/rr2logloggingWARNING graph_indexcopydeepcopy set_codegenrfxCodeGenrosmakedirsrrdumprsave) gm_to_saver0 type_namegmr) current_namedump_example_input folder_namerrr graph_saver_helperss< 22z,_save_fx_default..graph_saver_helpercs||d|S)Nforwardr)rfw_argsrrr graph_saver_forwards z-_save_fx_default..graph_saver_forwardcs||dtd7a|S)Nbackwardr )r)rbw_argsrrr graph_saver_backwards z._save_fx_default..graph_saver_backwardcs||dt||S)Njoint)r)r joint_argsrrr graph_saver_joints  z+_save_fx_default..graph_saver_jointr)functorch.compilerr) rrrrexample_inputsrrrrr)rrrrrr _save_fx_defaultRs   #  rFcCsdatt|||S)as Dump the forward, backward, and joint computation graph. Example Usage: save_fx_func = graph_dumper_aot(current_name, folder_name, dump_example_input = False) optimize_ctx = torchdynamo.optimize( save_fx_func ) with torch.enable_grad(): with optimize_ctx: result = forward_and_backward_pass(model, example_inputs) r)rrr)rrrrrr graph_dumper_aots r)T)F)Vrrrrr contextlibr functoolsrtypingrrrVrtorch.fxrtorch.nnrtorch.utils._pytreeutils_pytreerxr torch._decompr%torch.fx.experimental.symbolic_shapesr aot_autogradr r r compile_utilsr partitionersrrr getLoggerrrr!r% GraphModulerCrIrKrM InterpreterrNrrrrrdetach gelu_backwardleaky_relu_backwardsigmoid_backwardthreshold_backwardhardtanh_backwardhardsigmoid_backwardhardswish_backward tanh_backward silu_backward elu_backwardcudnn_batch_normcudnn_batch_norm_backward masked_fillScalarr{elu leaky_reluhardtanh hardswish hardsigmoid conj_physical is_same_sizerrrrrrrrrrrrr s           . 7    ']