o I hE@sddlmZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddlm Z ddl m Z mZmZmZmZmZmZddlmZddlZddlZddlZddlmZddlmZddlmZmZm Z dd l!m"Z"m#Z#m$Z$m%Z%d d l&m'Z'd d l(m)Z)m*Z*m+Z+d dl,m-Z-d dl.m/Z/d dl0m1Z1m2Z2m3Z3d dl*m4Z4m5Z5d dl6m7Z7d dl8m9Z9m:Z:d dl+m;Z;mm?Z?m@Z@mAZAmBZBmCZCmDZDmEZEmFZFmGZGmHZHmIZId dlJmKZKmLZLmMZMddlNmOZOddlPmQZQmRZRmSZSmTZTddlUmVZVddlWmXZXmYZYmZZZm[Z[erddl\m]Z]m^Z^m_Z_e`eaZbejcdeadZeejcdeadZfejcdeadZgeTjhZiegd ZjejkGd!d"d"ZlGd#d$d$elZmGd%d&d&elZnd9d+d,Zoed-eQeQd.ZpGd/d0d0eSepeepZqGd1d2d2e<Zrejkd3d4Gd5d6d6ZsGd7d8d8etZudS):) annotationsN)Counter)AnyCallableGeneric no_type_checkOptional TYPE_CHECKINGUnion)TypeVar)immutable_dict) OrderedSet)FloorDivIdentityModularIndexing)free_symbol_is_type prefix_strsymbol_is_typeSymT)counters)configir scheduler)prologue_preserves_zero_mask) code_hash) MemoryDepStarDepWeakDep)IRNodeTritonTemplateBuffer)!indexing_dtype_strength_reduction) green_text yellow_text)BaseSchedulerNodeBaseScheduling WhyNoFuse) cache_on_selfexpr_fits_within_32bitget_dtype_sizeIndentedBuffer Placeholderprefix_is_reduction'set_kernel_post_grad_provenance_tracingsympy_index_symbol sympy_product sympy_subsunique)ops OpsWrapperV)BlockPatternMatcher) CSEVariableindex_prevent_reorderingKernel PythonPrinter) MultiKernel)DisableReductionEnableReductionNodeScheduleMarkerSIMDKernelFeatures)IterableIteratorSequence perf_hintsschedulefusion)zyxr0_r1_csheZdZdZejjejjddfddZee e dddZ dddZ ee e d ddZ ZS)!IterationRangesa Each range tree represents multiple sets of iteration indexing in a single tiled dimension in the output kernel. If you have two loops ranges one (4, 3, 2) and another (4, 6), then the range tree will be: 4 (i0) 3 (i1) 6 (i3) 2 (i2) Where i0 is shared between both loops, but then the split into different indexing vars. All loop ranges must iterate over the same number of elements. )divisorlengthnamestrvar_listlist[sympy.Symbol] var_rangesdict[sympy.Symbol, sympy.Expr]numel sympy.Exprprefixkernel SIMDKernelrootIterationRangesRootreturnNonec sDt||_||_||_||_||_||_||_||_ | |_ dSN) super__init__rOrQrSrUrWrMrNrXrZ) selfrOrQrSrUrWrXrMrNrZ __class__P/var/www/vscode/kcb/lib/python3.10/site-packages/torch/_inductor/codegen/simd.pyr``s  zIterationRanges.__init__boolcC t|jSr^)r-rWrardrdre is_reductionxs zIterationRanges.is_reduction sympy.SymbolcCrgr^)r/rOrhrdrdresymbol~ zIterationRanges.symbolrcCsddtD}||jS)NcSi|]\}}||qSrdrd).0symtrWrdrdre z(IterationRanges.symt..)ritemsrW)raprefix_to_symtrdrdreros zIterationRanges.symt)rOrPrQrRrSrTrUrVrWrPrXrYrZr[r\r]r\rfr\rj)r\r)__name__ __module__ __qualname____doc__sympySOner`propertyr(rrirkro __classcell__rdrdrbrerLPs rLcsheZdZ d.d/fdd Zd0ddZd1ddZd2ddZd3d!d"Zd4d&d'Zd5d)d*Z d6d,d-Z Z S)7r[NrOrPrUrVrWindexintrXrY pid_cacheOptional[dict[str, str]]is_looprf tensor_dim Optional[int]grid_dimhas_zdimr\r]c sj|duri}tj|gi||||d||_i|_||_|r'|jr%| dus'J||_||_| |_| |_ dS)N)rOrQrSrUrWrXrZ) r_r`rnodesrrirrrr) rarOrUrWrrXrrrrrrbrdrer`s&  zIterationRangesRoot.__init__cCsd|jd|jdS)NzIterationRangesRoot(, z, ...))rOrUrhrdrdre__repr__szIterationRangesRoot.__repr__cCs|jD]}|qdSr^)rvalues cache_clear)ranoderdrdrers zIterationRangesRoot.cache_clearrjcCst|jdS)Nr)r/rWrhrdrdre index_symszIterationRangesRoot.index_symrMrNIterationRangesEntrycCstjj|||jrt||}nt|||}||jvrMt |j t tj j ||||}|tj j|<|j|||j|<||j|<|j|S)zF Lookup a given RangeTreeEntry, creating it if needed )r5graphsizevarsstatically_known_equalsrUrrrrrrWnextrXiter_vars_countrange_tree_nodesrkrQappendrS)rarMrNexprrrdrdrelookups    zIterationRangesRoot.lookuplengthslist[sympy.Expr]list[IterationRangesEntry]cCs@tjj}g}t|D]}||||||}q gt|Sr^)rzr{r|reversedrr)rarrMitervarsrNrdrdreconstruct_entriess    z%IterationRangesRoot.construct_entriesrRcCsdd||DS)NcSg|]}|qSrd)rk)rnerdrdre z1IterationRangesRoot.construct..)rrarrdrdre constructzIterationRangesRoot.construct+tuple[list[sympy.Symbol], list[sympy.Expr]]csdd|jD}fdd|D}|jdddtjjggfdd}|D]}tjj|j sE| t |j |j ||q+tjjj s_| t j gt gt fS) z,Figure out vars from this tree used in indexcSsg|] }tjj|qSrd)r5rXrgetrnsrdrdrersz6IterationRangesRoot.vars_and_sizes..cs g|] }|r|jjkr|qSrdrWrnnrhrdrers cSstjjj|jtjdS)N)fallback)r5rr size_hintrMrunbacked_symint_fallbackrIrdrdresz4IterationRangesRoot.vars_and_sizes..keycs(||j|jdSr^)rrkrN)r)rM index_varssizesrdreadds z/IterationRangesRoot.vars_and_sizes..add) free_symbolssortrzr{r|r5rrrrMrrrUr)rarrrrrd)rMrrarrevars_and_sizess" z"IterationRangesRoot.vars_and_sizesr^)rOrPrUrVrWrPrrrXrYrrrrfrrrrrrfr\r]r\rPr\r]ru)rMrVrNrVr\r)rrr\r)rrr\rR)rrVr\r) rvrwrxr`rrrrrrrr~rdrdrbrer[s *     r[csdeZdZd fd d Zd!d dZd"ddZd#ddZd!ddZd$ddZd%ddZ d&ddZ Z S)'rrOrPrMrVrNrparentrLr\r]c sPtj||j||j|j|j|||j|jd ||_t d|j |_ ||_ dS)N) rOrUrQrSrWrMrNrXrZ)r_r`rUrQrSrWrXrZr functools lru_cache_codegencodegenr)rarOrMrNrrrbrdrer`s  zIterationRangesEntry.__init__c Cs.d|jd|jd|jd|jd|jd S)NzIterationRangesEntry(r))rOrMrNrrSrhrdrdrers.zIterationRangesEntry.__repr__cs$fdd|_dd|j_|_dS)NcsSr^rdrdrOrdrer z/IterationRangesEntry.set_name..cSdSr^rdrdrdrdrer!r)rrrO)rarOrdrreset_names  zIterationRangesEntry.set_namecCs|jdSr^)rrrhrdrdrer$sz IterationRangesEntry.cache_clearcCstj||jSr^)r5rXcodegen_iteration_ranges_entryrOrhrdrdrer's zIterationRangesEntry._codegenrcCsg}t|jtjr |St|jttfsJt|j|jjddD]"}t|tjtjfsD|j }t |dkrDt dd|DrD| |q"|S)Nr6rcss|] }t|tjVqdSr^)rrSIZErrdrdre 4s  z8IterationRangesEntry.precomputed_args..) isinstancerrzSymbolrrtypeargsIntegerrlenallr)raprecomputed_argsargsymbolsrdrdrer+s z%IterationRangesEntry.precomputed_argsrcCrgr^)hashrOrhrdrdre__hash__:rlzIterationRangesEntry.__hash__otherobjectrfcCst|tsJ|j|jkSr^)rrrO)rarrdrdre__eq__=s zIterationRangesEntry.__eq__) rOrPrMrVrNrVrrVrrLr\r]r)rOrPr\r]r)r\rr\r)rrr\rf) rvrwrxr`rrrrrrrr~rdrdrbrers      rvalueUnion[int, float]r\rPcCs6|tdkrdS|tdkrdSt|rdSt|S)Ninfz float("inf")z-infz float("-inf")z float("nan"))floatmathisnanrepr)rrdrdre constant_reprBs   rCSEVariableType)bounddefaultcseZdZUdZeZded<ded<dZded<ded < ddfdd Ze e e dddZ dddZ e ddd Zdd!d"Zdd(d)Zdd+d,Zdd/d0Zdd6d7Zdd8d9Zdd:d;Zdd=d>Zdd?d@ZddBdCZddEdFZddGdHZddIdJZddMdNZddOdPZddRdSZddVdWZe dd\d]Z!e"e#j$j%fdd_d`Z&ddbdcZ'e"ddddeZ(ddfdgZ)ddhdiZ*ddjdkZ+ddldmZ,dddodpZ-ddrdsZ.ddtduZ/dddxdyZ0e1j2dd~dZ3dddZ4e ddZ5ddZ6ddZ7ddZ8ddZ9ddZ:ddZ;dddZ<Z=S)rYzo Common base class for Triton/Halide codegen which both use flattened indexing rather than loop nests. zCallable[[sympy.Expr], str]sexprkexprFrfallow_block_ptrrP kernel_nameNtilingdict[str, sympy.Expr]featuresr@rroverride_persistent_reductionOptional[bool]override_cooperative_reductionr\r]cs|duri}t|_|_t_t_dd|D_ g_ i_ t _|_|dur;|n_|durF|n__d_tddfdd }|_|dS)NcSs i|] \}}|tjj|qSrd)r5rrsimplify)rnrWvalrdrdrerphsz'SIMDKernel.__init__..rrVcs6tjj|}jD]}||}q |Sr^)r5rrsimplify_with_rangesrS range_treescombine_contiguous_dimscombine_modular_indexing_pairs)rtreerhrdresimplify_indexing}s  z.SIMDKernel.__init__..simplify_indexing)rrV)r_r`r get_mutations mutationsr+body indexing_coderrnumelsrr itertoolscountrriinside_reduction should_use_cooperative_reductioncooperative_reductionshould_use_persistent_reductionpersistent_reduction want_no_x_dimno_x_dimrrrrinitialize_range_tree)rarrrrrrrbrhrer`Ys8     zSIMDKernel.__init__rcCtdd|jDS)Ncs|]}t|VqdSr^)r-rnrWrdrdrerz0SIMDKernel.num_reduction_dims..)sumrrhrdrdrenum_reduction_dimsszSIMDKernel.num_reduction_dimsdtype torch.dtypecCtr^NotImplementedError)rar rdrdre dtype_to_strzSIMDKernel.dtype_to_strcCs||jSr^)rrselect_index_dtyperhrdrdre index_dtypeszSIMDKernel.index_dtypecCdSNFrdrhrdrdrerrzSIMDKernel.want_no_x_dimrrirrlist[IterationRangesRoot]cstfddtD}| p| }d dd}gd} dd g} |r#| } n |r(| } n| | } || |} || t} g}t|D]6\}}t|}| |}| |}|durT|n|}|t|d ||||||oi|j ||d vd q<|S)Nc3|] }|vr|VqdSr^rdrrrdrersz3SIMDKernel.construct_range_trees..r\dict[Any, int]cs ddtfdd|DDS)NcSrmrdrd)rnidxrrdrdrerps zPSIMDKernel.construct_range_trees..filtered_index_map..c3rr^rd)rnrmaskrdrerzOSIMDKernel.construct_range_trees..filtered_index_map..) enumerate)seqrrdrrefiltered_index_mapsz.filtered_index_map)rIrHrGrJrKrrG)rrrrr)r\r)r all_prefixesrr-rrr[r)rarrrirractive_prefixesno_r_dimr  grid_dimsreduction_dims tensor_dimstensor_dim_map grid_dim_maprirWrrrrdrreconstruct_range_treessF        z SIMDKernel.construct_range_treesdict[str, str]cCs.|||j|j|j|j}|j|dSr^)r*rrrirrrextend)rarrrdrdrersz SIMDKernel.initialize_range_treeindicesSequence[sympy.Expr]cCr)zr Hook called right before codegen with every index that will be used in the fused kernel. Nrd)rar-rdrdrefinalize_indexingrzSIMDKernel.finalize_indexingrOrrVrr8cCs,|j}d|_z ||||W||_S||_wr)rstore)rarOrrpriorrdrdrestore_reductions zSIMDKernel.store_reductioncCrrrdrhrdrdrerrz+SIMDKernel.should_use_cooperative_reductioncCrrrdrhrdrdrerrz*SIMDKernel.should_use_persistent_reductionrTcCsttjdd|jDS)Ncss|]}|jVqdSr^)rSrrrnrrdrdrers  z(SIMDKernel.var_ranges..)dictrchain from_iterablerrhrdrdrerSs  zSIMDKernel.var_rangescCr)Ncss|] }t|jduVqdSr^)rrr3rdrdrersz0SIMDKernel.triton_tensor_ndim..)r rrhrdrdretriton_tensor_ndimrzSIMDKernel.triton_tensor_ndimr)cCs(dg|}d||<dd|dS)Nr]:[r])r7join)rar)rrdrdreindexing_size_strszSIMDKernel.indexing_size_str list[str]cCsLdg|}|jD]}|jdurq |jr|jr#|jd||j<q |S)N1BLOCK)r7rrrirrWupper)rarrrdrdredense_size_lists   zSIMDKernel.dense_size_listcCs|}dd|dS)Nr9rr:)rAr;rarrdrdredense_size_strszSIMDKernel.dense_size_strcCsxt|ts|S|jd}|j|}dur|St|||ji}tjj |}t||j |j tjj|j jiS)Nr)rrrrrr1rr5rrrrZrrrzr{r|rUrk)rarrI tree_node new_indexrdrdrer s   z)SIMDKernel.combine_modular_indexing_pairsrr[cCs8tjj|}r|\}}t||||S|||Sr^)r5rrexpand_floor_divr_combine_contiguous_dims)rarr expand_resrE denominatorrdrdrers z"SIMDKernel.combine_contiguous_dimsc Cst|tjtjfr |S||\}}t|dkr|Stjj ||t |g||\}}}||kr1|S| |}t |t t|||} | S)zI More aggressive simplification to merge contiguous dims r6)rrzrrrrr5rr_simplify_loopsr9rr1r4zip) rarrrr new_sizesreindex_prunenew_index_varsrErdrdrerG$s   z#SIMDKernel._combine_contiguous_dims'contextlib.AbstractContextManager[None]cs,jdjpjtjfdd}|S)Nc3sfjsjr JdVdSrd_zdVr)Wd_dSWd_dSd_w)NFT)rrir codegen_bodyrdra should_flushrdrectx;s    z)SIMDKernel.disable_reduction..ctx)rrr contextlibcontextmanager)rarUrdrSredisable_reduction8szSIMDKernel.disable_reductionrrRcGs,t|t|jks Jddt||jDS)NcSsg|] \}}||qSrd)r)rnrNrangesrdrdrerRsz)SIMDKernel.set_ranges..)rrrKrrdrdre set_rangesPs zSIMDKernel.set_rangesgroupsIterable[sympy.Expr]Sequence[Sequence[sympy.Expr]]Stuple[list[list[sympy.Expr]], list[list[Callable[[list[sympy.Expr]], sympy.Expr]]]]c stdd|Drdd|DgfStjjdd|Dfdd|Dtdfd d }ddd}g}d}|D]~}g}|D]r}|drU|ddqE|tkru|dru|d7}|tkru|dsc|dtkr ||r ||st |} t ||} ||| ||| ||d| qE|t |||qE||q?tddDsJdd||fS)Ncss|] }t|dkVqdSrNr)rnrNrdrdrer^rz5SIMDKernel._split_iteration_ranges..cSg|]}gqSrdrd)rngrouprdrdrer_z6SIMDKernel._split_iteration_ranges..cSrardrd)rn_rdrdrerbrccsg|]}|qSrdr)rng)svrdrercrqr)rrrVr\csF|}||stt|||<||tSr^)rstatically_known_multiple_of CantSplitrrr)r)r new_ranges remainingrg var_countrdre add_rangefs z5SIMDKernel._split_iteration_ranges..add_rangesizeidx1idx2(Callable[[list[sympy.Expr]], sympy.Expr]csdfdd }|S)N flat_varsrr\rVcs||Sr^rd)rsrprqrordregetterrrzISIMDKernel._split_iteration_ranges..make_combined..getter)rsrr\rVrd)rorprqrurdrtre make_combinedosz9SIMDKernel._split_iteration_ranges..make_combinedrr6cSstjjSr^)rzr{Zero)rdrdrdrer}sz4SIMDKernel._split_iteration_ranges..css"|] }tjj|dkVqdS)r6Nr5rrrrrdrdrer zfailed to set ranges  )r)rrrVr\r)rorVrprrqrr\rr)rr5rrrrrrrstatically_known_gtrhriroperator itemgetter) r[rrnrvreturn_getters_groups current_group length_groupreturn_gettersrosize1size2rdrjre_split_iteration_rangesWsb    z"SIMDKernel._split_iteration_rangesreduction_numelcCsjtjj}t|ddkr!|t|t|d|r!|d|gf}z |||WdSty4YdSw)Nr6rTF)r5rrrrr0rri)clsr[rrrrdrdre is_compatibles  zSIMDKernel.is_compatiblelist[list[sympy.Expr]]cCsPdd|jD}|js|D] }t|rtjj||<q g|}||||jS)NcSsi|]}|j|jqSrd)rWrU)rnrtrdrdrerprqz3SIMDKernel.split_and_set_ranges..) rrr-rzr{r|rmap_kernel_groups_to_node_sizesrZ)rarrrWr[rdrdresplit_and_set_rangess  zSIMDKernel.split_and_set_rangescsft|t|krtddt||Dr||S|||\}}gtj||fdd|DS)a We may want to fuse `for i0 in s0*s1` into a tiled kernel with groups (s0, s1). To do this we need to split up the iteration space of i0 into something like: for i1 in s0: for i2 in s1: i0 = i1*s1 + i2 .... This function matches and resplits lengths to the groups of this kernel to enable tiled + non-tiled fusions. css.|]\}}tjjt||dkVqdSr_r5rrrr0)rnrIrfrdrdrers  z=SIMDKernel.map_kernel_groups_to_node_sizes..csg|] }fdd|DqS)csg|]}|qSrdrd)rnfnrrdrerrzISIMDKernel.map_kernel_groups_to_node_sizes...rd)rnfnsrrdrersz>SIMDKernel.map_kernel_groups_to_node_sizes..)rrrKrrr5r6)rr[rrZrkr~rdrrersz*SIMDKernel.map_kernel_groups_to_node_sizescCs t|tjSr^)rrTMPrarrdrdreis_indirect_indexings zSIMDKernel.is_indirect_indexingcs||rdSdgt|j}|jD] }||jvrq|j|}t|jts'J||jj|j 9<qt j j j tfddt||jDS)NFr6c3s$|] \}}||kVqdSr^rd)rn idx_range iter_rangererdrers  z,SIMDKernel.is_broadcasted..)rrrrrrrr[rrNr5rrranyrKr)rar index_numelsrkentryrdrereis_broadcasteds      zSIMDKernel.is_broadcastedcCs4t|trddt|j|dS|||S)a Convert an index expr to a string that can be used in output code. e.g. a sympy expression "s2" may actually appear as "ks1" in the generated kernel. Index expressions often need to be passed in as arguments to the triton kernel. Rename_indexing and codegen_indexing keep track of the needed indices and add new parameters to the function signature. r9rr:)rlistr;map index_to_strrrename_indexingrrdrdrers zSIMDKernel.index_to_strcCs||}t|tjjj}t|tj st|tj r%| tjjj}t|tj rV|tj D]"}|j }t|dkrUt dd|DrU|tjj|i}t||}q3||}t|tsb|n|jd}||S)Nrcss"|] }t|tjtjfVqdSr^)rrrPRECOMPUTED_SIZErrdrdrers  z.SIMDKernel.prepare_indexing..)rr1r5rrprecomputed_replacementsratomsrzfloorceilingsubsrrlookup_precomputed_sizerrrcodegen_indexing)rarar replacements simp_indexrdrdreprepare_indexing s"     zSIMDKernel.prepare_indexingreordercsfddjD}|rIt|dkrItdd|D}ddd|d|Dd| dks=Jd d|d|Dt|d||d|<|S) Ncsg|] }|jr jr|qSrd)rirrntrhrdrer2sz1SIMDKernel.active_range_trees..r6css|]}|jdvVqdS)xyzNrrrdrdrer6sz0SIMDKernel.active_range_trees..css|]}|jVqdSr^rrrdrdrer7szyxcSg|]}|jqSrdrrrdrdrer7s)rrr r;r)rartreesrrdrhreactive_range_trees1s 2 zSIMDKernel.active_range_treesrcCstjj||}t|jtdD]6}||jvrGi}|j| D] }tjj |||<q!t |dkr@t |j|j ||j|_ |j|q|S)Nrr)r5rrrrSsortedrrPrrrrr1rr)rarsymrpsrdrdrer=s    zSIMDKernel.codegen_indexingcCtd)NzNYI: codegen_nan_checkrrhrdrdrecodegen_nan_checkNzSIMDKernel.codegen_nan_checkrOptional[IRNode]cCr)NzNYI: call_kernelr)rarOrrdrdre call_kernelQrzSIMDKernel.call_kernelrUnion[str, OpsWrapper]r Iterator[str]ccs\|j}|j}|rt||}t|}||_||_z |VW||_||_dS||_||_w)z:Context manager to add an additional mask to tl.load/storeN) _load_mask _load_otherr3 logical_andr4_unwrap)rarrr1 prior_valrdrdre mask_loadsTs   zSIMDKernel.mask_loadscCs\dd|jD}t||}i}|jD]}t|j}t||dit||di||<q|S)a\ This gets the stride of the index for each of the tiling variables (technically, it does it at index 0) For example, if xindex = x0 + 512*x1 + 1024*r0 x0 = (xindex//512) x1 = (xindex % 512) r0 = rindex // 1024 this function would return {xindex: 512, rindex: 1024} cSsi|]\}}||jqSrd)r)rnkvrdrdrerpvz2SIMDKernel.get_strides_of_load..r6r)rrrr1rr/rO)rarindex_to_tile_indexesindex_in_tile_varsstrides range_treerrdrdreget_strides_of_loadhs    zSIMDKernel.get_strides_of_loadcCs t|tr tt||S||Sr^)rtupler)rrrdrdre_map_tuple_or_scalars zSIMDKernel._map_tuple_or_scalarcCs0g}tt|jj}|j\}}}}|j}tj j t |j }t|D]i\}}||vr8|dq*tj |} tj j | } | |krxtt} d} ||D]} t| ttfrj| d| | d7} qT| | jqTt| |}n| }tj |}t|}|||dt||kq*t|S)a+ Try the best to estimate the total size (in bytes) of the kernel's inputs and outputs, which is used for estimating the memory throughput of this kernel. This information is used for checking how far we are from the peak memory bandwidth. It's important that we want to avoid overestimating the sizes of the inputs and outputs, because it can wrongfully give us a very large memory traffic value, which may be even larger than the theoretical bandwidth and thus become very misleading. This is particularly problematic for cases where we slice some inputs. In those cases, we should only count the size of the "slices" instead of the original inputs, because only the slices contribute to the real memory traffic. r no_index_dep_r6)rr2rinplace_buffersrpython_argdefsr buf_accessesr5rrrr0rrr get_numelr rrrrrr get_dtyper*rr )ranbytes ninplace_argsrd call_argsr out_numelr)r arg_numelbuf_sizer-no_index_dep_countdeprUr  dtype_sizerdrdreestimate_kernel_num_bytess2        z$SIMDKernel.estimate_kernel_num_bytescCstt|jjdkrt|jjdkrt|jjdkrdS|j\}}}}d}|D]}tj|}|s2q'| } t| j dkrtdd| j DdkrJq't | j } |durW| }q'|| krtd|d d | d |} t| d d|D} d d|D} dd|D}dd|D}td|d|d| d| d|d} t| dSq'td|d} t| dS)zr Print message if the kernel have mixed layout inputs. Only care about 4D tensor for now. r6rNcSsg|]}|dkr|qS)r6rdrnrIrdrdrerrz.SIMDKernel.warn_mix_layout..rzExpected stride order z, but found stride orderrzz for kernel cSs4g|]}tj|rttj|jndqSr^)r5rtry_get_bufferrget_stride_order get_buffer get_layoutstridernrOrdrdrers cSs.g|]}tj|rtj|jndqSr^)r5rrrrrorrdrdrers  cSs0g|]}|tjjvr dn |tjjvrdndqS) GraphInputIntermediateBufferN)r5r graph_inputsname_to_bufferrrdrdrers  cSrrdrrrdrdrerz param names z buf names z strides z sizes z sources  z%All the inputs for the triton kernel z have uniform layout)rr input_buffersoutput_buffersrrr5rrrrorrrr$logwarningr#)rarargdefsr _signaturerduniform_stride_orderarg_namebuflayout stride_ordermsgstride_order_list size_list source_list argdef_namesrdrdrewarn_mix_layouts\       zSIMDKernel.warn_mix_layoutc Cspt||d|}d|_t|jj|}t||}d|_t||}t||}t||d|}t |||fS)Nr FT) r3 reductionr index_exprrrtruedivsubmulr4r) rar rsum_rnumelmeandxdx2m2rdrdrewelford_reduce_fallback s   z"SIMDKernel.welford_reduce_fallbackcCsDt||d|}t||}t|}t||d|}t||fS)Nmaxr )r3rrexpr4r)rar rvmaxrrvsumrdrdre prepare_softmax_twopass_fallbacks   z+SIMDKernel.prepare_softmax_twopass_fallbackcCr r^rrhrdrdrecodegen_kernelrzSIMDKernel.codegen_kernelcCrr^rdrhrdrdrerR"rzSIMDKernel.codegen_bodyrrcCrr^rd)rarrdrdrer%rz)SIMDKernel.codegen_iteration_ranges_entry)NNN) rrrr@rrrrrrr\r]r)r r r\rPrrt) rrrrfrirfrrrrfr\r)rr+r\r])r-r.r\r])rOrPrrVrr8r\r])r\rT)r)rr\rP)r\r=)rrVr\rV)rrVrr[r\rV)r\rP)rrVr\rR)r[r\rr]r\r^)r[r\rr]rrVr\rf)rr]r\r)r[r.rr]r\r)rrVr\rf)rrVr\rPF)rrfr\r)rrVr\rVrr^)rOrPrrr\r])rrrrr\r)rrVr\rT)rr)>rvrwrxrypexprr__annotations__rr`r}r(rr rrrr*rr/r2rrrSr7r<rArCrrrGrXrZ staticmethodr classmethodrzr{r|rrrrrrrrrrrrVrWrrrrrrr r rRrr~rdrdrbrerYOs|   /     6            L      &     ?G rYc@s.eZdZUeZded<ddZddZeZeZ ddZ dQd d Z e dRddZ dSddZdTddZddZdddUd!d"Zd#d$Z dVdWd,d-Zd.d/Zeed0dXd2d3ZedYd8d9ZedZd.)rrBrdrdregroup_fn,zSIMDScheduling.group_fncsdt|tjs t|tjrtj||S|j\}\}}|j\}\t||}|r7|s7|r6|dn|rG|sG|rG|d|rc|rc|koV|k}|sa|d|||S|s7|s7|kru|ks|s|d||dS| D]+}|rn$| | @sq|j\}\} } || kr|| ks|d|| || dSqt ||fdD]\} } | rt| t} | s|| d| Sq|| ||}|| ||}|| | ||}tjjr5d}t|d krt|d kr||ko|kn}n||k}n t|d kr)||k}|s5|d |||dSdS|s|r|d krKd ksMJ|krtfd d | Dsi|ddStjjr|st|| ||d fd ffv}|s|d|SdS|kr|d|kS|r|rJ|||S)z Hook called by Scheduler to determine if the Triton backend can fuse node1 and node2. These nodes might already be FusedSchedulerNodes. z&Split scan cannot fuse with reductionsz1numel/rnumel mismatch (reduce) (%s, %s), (%s, %s)z5numel/rnumel mismatch (non-reduce) (%s, %s), (%s, %s)Fz:numel/rnumel mismatch prologue mismatch (%s, %s), (%s, %s))node1node2z is not TritonTemplateBufferTrztiling mismatch (%s, %s, %s)r6c3s$|] }tf|VqdSr^)rYr get_rangesrnumel2rnumel2rdrers  z*SIMDScheduling.can_fuse..z"nodes numel/rnumel incompatibilityzinvalid tiling for reductionznodes numel incompatibility)rrForeachKernelSchedulerNodecan_fuserbr' is_split_scanri is_template get_nodesused_buffer_namesget_buffer_namesrKget_template_noder! select_tilingrtriton tiling_prevents_pointwise_fusionrr tiling_prevents_reduction_fusionrrcan_fuse_horizontal)rarrrdnumel1rnumel1whyreduction_can_fuser pro_numel pro_rnumelr node_nameis_triton_templatetiling1tiling2tiling3condis_reduction_tiling_validrdrrer/s         zSIMDScheduling.can_fusec sbgttjttttdfdd}fdd}fdd}fdd}tjfd d }fd d } |D]h} | vrMqF| || r| | rn|Wdn1siwYr{|| s{pytnd|| qF|| r| | Wdn1swYqFt d dd| j dS)Ncs2|j\}\}}|kr|kp|ko|dkSNr6rbrrd node_numel node_rnumelrUrrdrefits_in_main_bodysz@SIMDScheduling.generate_node_schedule..fits_in_main_bodycs&|j\}\}}|ko|dkodkSr6r7r8r;rdrefits_outside_reductionszESIMDScheduling.generate_node_schedule..fits_outside_reductioncs"|jjD] }|jvrdSqdS)NTF) read_writesreadsrO)rread)current_loop_buffer_usagerdreexpect_improved_memory_usages  zKSIMDScheduling.generate_node_schedule..expect_improved_memory_usagecs||dd|jjD|r8t|tjr8t|j t j r8t|j j t j s8|dSdd|jjDdS)NcSrrdrrrdrdrerrzXSIMDScheduling.generate_node_schedule..schedule_node_in_loop..cSrrdrrrdrdrerr)rrupdater>r?rirr SchedulerNoderrComputedBufferdataScanget_namewrites)r)rAdone node_schedulenot_ready_yet_nodesrdreschedule_node_in_loops    zDSIMDScheduling.generate_node_schedule..schedule_node_in_loopc3snrdturntr%tdtddVtdS)NrQr6)r>poprr=insertclearrd)rAmaybe_split_indexrKrLrdreend_current_reduction_loops     zISIMDScheduling.generate_node_schedule..end_current_reduction_loopcs<dkrdS|j@s dS|rt|dttfrJtS)Nr6FrQ) ancestorsrr>r=rf)rrK)rLrrdre#requires_closing_previous_reductions  zRSIMDScheduling.generate_node_schedule..requires_closing_previous_reductionzunexpected group: (rz) != r6) r rr%rPrVrWrrrrrb) rarrUrr<r=rBrMrRrTrrd)rArJrQrKrLrUrregenerate_node_schedulesD          z%SIMDScheduling.generate_node_scheduler.rz Schedule: %s)r rrbrU schedule_logdebugcodegen_node_scheduler@)rarrrdrUrrKrdrdre codegen_nodes  zSIMDScheduling.codegen_noderUrVbuffers.csrr^)r))rnrordrdrer>rz8SIMDScheduling.can_use_32bit_indexing..T) torchiinfoint32rr)rr5rr guard_leq)rUr^int_max buf_sizesrordrdrecan_use_32bit_indexing,sz%SIMDScheduling.can_use_32bit_indexingkernel_featuresr@c Cs"|j}|||j|j}|||gd|i}|D]}|||qt||D]9}t | | }Wdn1s>wY| |||}t j jrSt||td|||_t||_q(~t|dkrnt|}n|\}t ||D]} | q{Wdn1swY||||jt jr|t jr||djtjj|jO_tjj|jO_tjjj r t j!r |dj"#} |D]5} | $} | | vrq| j%dusJ| j%&} | dur t'ddd7<tjj(d| j)d| d q|*dS) Nrz+Generating kernel code with kernel_name: %sr6rinductorintermediate_hookszrun_intermediate_hooks(rr)+rKr$rUrcreate_kernel_choices!codegen_node_schedule_with_kernelr<merge_workspaces_inplacer5set_kernel_handlerr  define_kernelrtraceenabledr.rr[rrrscheduler_nodesmark_runcodegen_commentr nan_assertsrrrremoved_buffersinplaced_to_remove wrapper_codesupports_intermediate_hooksgenerate_intermediate_hooksrlive_output_buffersrHrget_origin_noder writelinerOfree_buffers_in_scheduler) rarirKrkernelsrXsrc_coder final_kernelr live_outsrO origin_noderdrdrer\Hst                  z$SIMDScheduling.codegen_node_schedulelist[SIMDKernel]cCs|j|i|gSr^)r)rari kernel_args kernel_kwargsrdrdrerls z$SIMDScheduling.create_kernel_choicesc Cs|tt}i}|D]0}|tur||q |tur"|q ||| }| t |j |q |||D](}|turS||qE|tur\|qEt|j || }||qEWddS1sywYdSr^)rV ExitStackr= enter_contextrXr>closedecide_inplace_updaterrrCr4fromkeys_bodyindexing_from_argsrr/keysr"r)rarKrXstack all_indexingrrrdrdrerms4    "z0SIMDScheduling.codegen_node_schedule_with_kernelFonly_gen_src_code Optional[str]c Cs|j\}\}}|dks J|j|j\}} i} |} g} |D]+} | }| | || @rKt|dks7J| | tt|<|j tt|g} q t| dksTJ||se|g|D]}| q^| }| d|D] }| ||qp|jtWdn1swY|jD]|\}}d|d}| |g} rtdd| D}td | N| |9| D]'}t|dkrt| dkrt|r|j|O_| ||q|jtWdn1swYWdn 1swYqWdn 1s wYt|ts7|d |jd d d t ||j!D]}d|d}|j|d d qB| dt|trb|}n|d|j"}Wdn 1suwYg|||}tj#r|$d}|%d|d|&|'}|r|WdS|(|||}tj)j*rt+||Wdn 1swY|,||-||jtj.j/|j/O_/tj.j0|j0O_0|1dS)z Codegen a triton template If `only_gen_src_code` the src code will be returned instead of codegen'd into the wrapper r6rzNz cs|]}|VqdSr^)can_codegen_without_upcasts)rnp_nrdrdrers z2SIMDScheduling.codegen_template..ztriton.codegen_upcast_to_fp32z z F)strictgeAr)2rbrmake_kernel_renderr!r"rrriterprologue_fused_inputsrrtset_subgraph_bodyrrrcse invalidater named_input_nodesrrrrHrrpatchr#prologue_fused_inputs_preserve_zerorrP finalize_hookr5rorcodebenchmark_kernelrimports_for_benchmark_kernelcodegen_kernel_benchmarkgetvaluerprqrrr.rurrrwrxr)ra template_nodeepilogue_nodesprologue_nodesrrd_numelrrXrenderbuf_name_to_prologue_grouptemplate_readsprologue_groupprologuenamesr partial_code input_namebuffer subgraph_namecan_codegen_without_upcast prologue_noderrKnum_gbrrdrdrecodegen_templates         .            zSIMDScheduling.codegen_templatecCstjjtjjdSr^)r5rryr~ device_ops synchronizerhrdrdre codegen_sync-szSIMDScheduling.codegen_syncsubkernel_nodeslist[BaseSchedulerNode]custom_part_algorithmenable_autotune mixed_sizesrlist[tuple[str, Any, Any]]c Csddlm}dd|D}ii}} t||D]6\} } t| dddj\} \} }|| | |}||| |}||| |f| | <|j|t|| || d|| <q|j ||||| d }t d t |d d|Dg}|D]s}d d|D}|||d }t||D]R\} } | | | d||| || }| | d}|st|t|D]}|qWdn1swYtjj|jO_tjj|jO_q~|}||||fqj|S)Nr6) ComboKernelcSrrdr rnrrdrdrer:rz=SIMDScheduling.generate_combo_kernel_code..cSrWr^rXrrdrdrer=rYz;SIMDScheduling.generate_combo_kernel_code..r)r optimize_mask)rtriton_schedulingcustom_algorithm kernel_map node_info_mapz1ComboKernels: %d nodes partitioned into %s groupscSsg|]}t|qSrdr`)rnprdrdrerQrcSrrdrrrdrdrerUr)rrr)triton_combo_kernelrrKrrbrUr$create_triton_kernelr@horizontal_partitionrr[rrmcreate_sub_kernelr5ror? only_nodesrtrrwrxr r)rarrrrrrfused_node_lists subkernel_mapnode_schedule_mappnrrdrUrrKr partitionskernel_code_list node_grouprX subkernelrrrdrdregenerate_combo_kernel_code0sd          z)SIMDScheduling.generate_combo_kernel_codec Cs|}|j}|j}tjdkptjdko|}|||||}|D]!\}}} |||g|} ||gt d| | t j j | q |dS)Nr6z"ComboKernels: generated kernel %s.)get_subkernel_nodesuse_custom_partition_algorrcombo_kernel_allow_mixed_sizesrrprurr[rr5rryr) racombo_kernel_noderrrrrrrXrdrrdrdrecodegen_combo_kernelms     z#SIMDScheduling.codegen_combo_kernel list[CandidateTiling]c sdk}d fdd }|\}t|dkr!tdkr!gS|\}|||r-|n||}fdd |D}|S) Nr6 is_pointwiserfr\rc s t|jt|ksJd|jd||j|jg}tddtj|Ds)Jddtj|D}tdd|jD}dd d }t ||g|dddg}|D]}t j j |j|j} t| t|ksjJz | dd} | t|krzWqTtdd| | dDrWqTWn tyYqTw||d| ||| df} t j j tddt|| D} |j|vr| d9} t | dr| d9} t | dr| d9} t j j | tt|dkr|t ||d| ||| dg| |jdqT|S)zX Compute tiling candidates by dividing up the iteration ranges. zrw.range_vars=z ranges=css|] }t|ttfVqdSr^)rrrrnrrdrdrers  zHSIMDScheduling.candidate_tilings..tile_ranges..cSs(g|]}|jtjjvrt|tr|qSrd)rOr5rrwrrrrdrdrerszISIMDScheduling.candidate_tilings..tile_ranges..cSrrdrrrdrdrerrrYr.r\rVcSstjjt|Sr^r)rYrdrdrecollapse_rangesrzNSIMDScheduling.candidate_tilings..tile_ranges..collapse_rangesnoner)rrOscorer6css|]}|dkVqdSr_rdrrdrdrerrNcss |] \}}|dkr|VqdSr_rd)rnrorrdrdrersrrrrO)rYr.r\rV)r range_varsr?rIrrr5r6r CandidateTilingcreate_partial_tilingr5rr stride_hintsr ValueErrorrr0rKrO is_good_sizer) rrYrw dep_sourcesdeps write_namesrtilingsrrsplit tiled_groupsr)rrreduction_rangesrdre tile_rangess(          z5SIMDScheduling.candidate_tilings..tile_rangescs*g|]}t|j|j|jdqS)r)rcomplete_partial_tilingrrrO)rnr)rrUrrdrersz4SIMDScheduling.candidate_tilings..)rrfr\r)rr"pointwise_or_reduction_read_writes) rrrUrrrpointwise_rangespartial_tilings full_tilingsrd)rrUrrrecandidate_tilingss ^   z SIMDScheduling.candidate_tilings pw_tilingr.reduction_tilingrcCsFgdt| d}ddgdt|}tgt||t||S)zK Create a tiling dict from pointwise and reduction splits. )rGrHrINrJrK)rr rK)rrr pw_prefixesreduction_prefixesrdrdre create_tilings zSIMDScheduling.create_tilingrrcCs||r|ng|s |SgSr^)r)rrrrdrdrer s  z$SIMDScheduling.create_partial_tilingrc CsHt|}d|v}||}|t|g}|r||fn||f}|j|S)zb Given a tiling for only pointwise or reduction dimensions, adds the missing one. rI)rrr0r) rrrUrsplitsr total_numelmissing_tiling tiling_argsrdrdrers  z&SIMDScheduling.complete_partial_tiling"list[dict[str, tuple[sympy.Expr]]]c Cs|dk}ttttjf}t|D]}t|tj sq| }|s+t |ddkr+q||r0dnd}|g} dd|j D} | D]~} g| j} tjj} tjj}t| D]\}\}}| |9} || |rinqW|| |sqqB|d}|r}| d|n| |d}g}|D]3\}}t| j|}td|t|tt |}t||||}|dur|dn|g}| |q| !|qB| D]2}tdt |t"j#j$}|d}t%|d|}|ft&||d}|'|(|)||||qqt*|t dd}|S) z Creates N-dimensional tiling candidiates, attempting to simplify loads/stores by tiling the kernel into higher dimensions. Returns a list of tilings ranked by dimensionality. r6rcSs(g|]}t|trt|jdkr|qS)r)rrrrYrrdrdrerKsz1SIMDScheduling.get_nd_tilings..NrT)rreverse)+r r4rPrzExprr>filterrrrDrrr>reads_and_writesrYrrr{r|r5rrrstatically_known_geqrr7get_subexpr_involving_symbolrrrrrmatch_mod_div_block_exprr,rrr% max_tilesr0rrrrr)rrKpointwise_numelrrrr node_rangesranges_to_tile node_tilings memory_depsrall_var_rangespointwise_vars_numelrpointwise_end_idxvarrUreduction_start_idxrS index_tilingrnum_dims match_resultdims node_tilingnum_leading_dimsfirst_trailing_dimcollapsed_leading_dimcollapsed_splitsranked_tilingsrdrdreget_nd_tilings+s        zSIMDScheduling.get_nd_tilingscsdk}||gg}|stjjrtjjdkrBtjtjkr@t |D]}tjjs?t | ||dkr?t td|Sq#|Stt}t}t |D]&}| ||D]} | j|vraqY| jdurl|| j|| | j7<qYqPdd|D} tjjdkr|rdd d} tdt | D]} | | d| | } | dur| g| } nqt | dkrt d| tjjr|||| } | D]ttsJtfdd|Dr߈Sq|S)z Heuristics to decide how to tile kernels. Currently, we tile based on stride-1 dimensions. Returns: `(tile1, tile2, reduction_numel)` s.t. `tile1 * tile2 == numel` r6rz Reduction over non-contiguous dims. Consider setting config.triton.tile_reductions to True. NcSsg|]\}}|jqSrd)r)rncandidate_tilingrrdrdrersz0SIMDScheduling.select_tiling..rtiling0rr1r\Optional[dict[str, sympy.Expr]]cSs|d|dd}}|d|dd}}tjj||dkr#dStjj||dkr;||f||f\}}\}}tjj||dksHJtjj||sRdS|t||||dd}|S)NrIrHr6rrJ)rGrHrIrJ)rr5rrrrhr)r%r1a0a1b0b1 new_tilingrdrdreconvert_tiling_to_3dsz:SIMDScheduling.select_tiling..convert_tiling_to_3dzpossibly bad tiling: %sc3s4|]}t|tjrtj|dVqdS))rN)rrrDrYrrrrrrrdrers  z/SIMDScheduling.select_tiling..)r%rr1rr\r&)rrr%tile_reductionsr perf_hint_loglevelloggingWARNINGr>r rrinfotextwrapdedentr rP collectionsrrOrr most_commonrangeprefer_nd_tilingr#rr4r)rrKrUrrdefault_tilingr seen_namescandidate_tilesr$r"r,r) new_3d_tilingrdr-rer$sv              zSIMDScheduling.select_tilingcCrr^rdrhrdrdreflushrzSIMDScheduling.flushcCrrrdrhrdrdreready_to_flushrzSIMDScheduling.ready_to_flushc Cs2tdd|Dset|dddj\}\}}||||}||||}|j|t|||d}|||t d|!t | | } Wdn1sPwYWdn1s_wYn)|d |\} } } t d||j| | | d d } Wdn1swY| ttjd } | S) Ncsrr^)rrrdrdrerrzASIMDScheduling.generate_kernel_code_from_nodes..cSrWr^rXrrdrdrerrYz@SIMDScheduling.generate_kernel_code_from_nodes..r)rrrTrtriton_)rrrbrUr$rr@rmrrr5ror get_prologue_template_epiloguerreplacerPr, KERNEL_NAME) rarrrdrUrrKrrXrrtemplateepiloguerdrdregenerate_kernel_code_from_nodess>     z.SIMDScheduling.generate_kernel_code_from_nodescCrr^rd)rarKrdrdreru4rzSIMDScheduling.codegen_commentcCr r^r)rarrKrXrdrdrerp7rzSIMDScheduling.define_kernelN)rrV)rUrVr^r_r\rf)rir@)rir@r\r)r\rr ) rrrrfrrfrrfrrfr\r)r\r)rr.rr.r\r)rr.rrfr\r)rrrUrVrrVr\r)r\r)r\rrt)&rvrwrxrYrrrrcan_fuse_verticalr(rUr]rrhr\rlrmrrrrrrrrrrrr#rzr{r|r$r>r?rFrurprdrdrdrer)sN   `   C #v ={    qr   rT)frozenc@s6eZdZUded<ded<dZded<edd ZdS) rrrrrNrrOcCs"tjj|}|dko|ddkS)z@Somewhat arbitrary heuristic used to boost scores for some sizesrrrx)rrdrdrerAszCandidateTiling.is_good_size)rvrwrxrrOrrrdrdrdrer;s  rc@s eZdZdS)riN)rvrwrxrdrdrdreriHsri)rrr\rP)v __future__rr6rV dataclassesrrr1rr|r4rtypingrrrrrr r typing_extensionsr rzrbtorch._loggingtorch.fx.immutable_collectionsr torch.utils._ordered_setr torch.utils._sympy.functionsrrrtorch.utils._sympy.symbolrrrr _dynamo.utilsrrrrranalyze_preserves_zero_maskr codecacher dependenciesrrrr r!optimize_indexingr"runtime.runtime_utilsr#r$r%r&r'utilsr(r)r*r+r,r-r.r/r0r1r2 virtualizedr3r4r5block_analysisr7commonr8r9r:r; multi_kernelr<simd_kernel_featuresr=r>r?r@collections.abcrArBrC getLoggerrvr_logginggetArtifactLoggerr/rZ fusion_logdoprintrr! dataclassrLr[rrrrYrr Exceptionrirdrdrdres  $       4    8{ > _