o 0 h*@sfUddlZddlZddlmZddlmZmZdeede defddZ d ed ed ed ee eee ffd dZ deede ede efddZdedefddZdedefddZdede eeefeeefe eeeeefffddZde ede eeefeeefe eeeeefffddZd>dededeefd d!Zdedefd"d#Z $d?deeeefde ed%e ed&ed'eeefd(edeefd)d*Zd+Zeed,<d-d.hZe eed/<e Ze eed0<d1d1d2Zeeefed3<d4Zeed5<d-d.hZe eed6<e Ze eed7<d8d9iZeeefed:<d@d;d<Z e!d=kr1e dSdS)AN) defaultdict)AnyUnionlines indentationreturncCsFd}dd|}t|D]\}}|dkr||7}||d|7}q |S)N  r) enumeratereplace)rroutputnew_line_with_indentilinerV/var/www/vscode/kcb/lib/python3.10/site-packages/torch/utils/data/datapipes/gen_pyi.pymaterialize_liness rdir template_name output_name replacementsc Cstj||}tj||}t| }|}Wdn1s!wY|D](\}} } t|d}||t| | }||Wdn1sKwYq(dS)Nw)ospathjoinopenreadr rwrite) rrrr template_path output_pathfcontent placeholderrrrrrgen_from_templates     r$ dir_pathsfiles_to_excludecsVt}|D]#t}dd|D}fdd|D}|fdd|Dq|S)z When given a path to a directory, returns the paths to the relevant files within it. This function does NOT recursive traverse to subdirectories. cSs h|] }d|ddkr|qS)z.pyNr.0fnamerrr .s z"find_file_paths..csh|]}|vr|qSrrr(r&rrr+/scsh|] }tj|qSrrrrr()dir_pathrrr+2)setrlistdirupdate)r%r&paths all_files python_files filter_filesr)r.r&rfind_file_paths%s  r7rcCs\d|vr d\}}nd|vrd\}}ntd|||t|||}}|||S)zXExtract method name from decorator in the form of "@functional_datapipe({method_name})".(")r8z")(')r9z')z4Unable to find appropriate method name within line: ) RuntimeErrorfindlenr start_token end_tokenstartendrrrextract_method_name6s   rBcCs2d}d}||t|||}}|||S)zVExtract class name from class definition in the form of "class {CLASS_NAME}({Type}):".class ()r;r<r=rrrextract_class_nameDs rE file_pathcCsiit}}}tt}t|}d}d\}}} d} |D]} | dddkr+| } | s1d| vr9||| qd| vrFt| }g||<q|rQd| vrQt| }q|rwd | vs[d | vrwd | vrd|||d7}| d t d } | | d } |dkr|| d 7}|| d 8}|dkr| d } | | d | 7} t | ||<|||<d\}}} q|dkrt d| | dd7} qWd n1swY||||fS)zfGiven a path to file, parses the file and returns a dictionary of method names to function signatures.r)rrrFz"""z@functional_datapiperCz def __init__(z def __new__(rDN)z7open parenthesis count < 0. This shouldn't be possible.r r )r0rlistrcountappendrBrEaddr;r<rfindprocess_signaturer:strip)rFmethod_to_signaturemethod_to_class_namespecial_output_typedoc_string_dictr!open_paren_count method_name class_name signatureskiprr@rArrrparse_datapipe_fileLs^         'rZ file_pathsc Csdiit}}}i}|D]}t|\}}}} |||||||| q ||||fS)N)r0rZr2) r[methods_and_signaturesmethods_and_class_names!methods_with_special_output_typesmethods_and_doc_stringsrrQrR$methods_needing_special_output_typesrTrrrparse_datapipe_filess,     ra, delimitercCspd}d}g}|D](}|dkr|d7}n|dkr|d8}n||kr,|dkr,||d}q||7}q|||S)zRGiven a line of text, split it on comma unless the comma is within a bracket '[]'.rr[rH])rL)rrc bracket_count curr_tokenrescharrrrsplit_outside_brackets     rjcCst|}t|D]B\}}|d||<|dkrd||<q|dkr5d||dkr5||ddkr5d||<qd|vrJ|d d \}}|dd ||<qd d |D}d|}|S)z Clean up a given raw function signature. This includes removing the self-referential datapipe argument, default arguments of input functions, newlines, and spaces. r clsselfrrH*rz Callable ==rGz= ...cSsg|]}|dkr|qSrr)r)trrr sz%process_signature..z, )rjr rPrsplitr)rtokensrtokenhead _default_argrrrrOs (  rOrdeprecated_filesdefault_output_typemethod_to_special_output_typerootc s dkrtttjt|tr|gn|}fdd|D}t|||d}t |\}}} } |D] } | | vr?| | q4g} | D]4\} }|| }| | vrW|| }n|}d | | }|dkrfd}| d|d| d|d |d | qF| jd d d | S)a #.pyi generation for functional DataPipes Process. # 1. Find files that we want to process (exclude the ones who don't) # 2. Parse method name and signature # 3. Remove first argument after self (unless it is "*datapipes"), default args, and spaces rcsg|] }tj|qSrr-)r)rrzrrrqr/z*get_method_definitions..r,z ... z# Functional form of 'z' def rDz) -> z: cSs|ddS)Nr rH)split)srrrsz(get_method_definitions..)key)strpathlibPath__file__parentresolve isinstancer7unionrarMitemsrrLsort)rFr&rwrxryrzr[r\r]methods_w_special_output_typesr_fn_namemethod_definitionsrV argumentsrW output_type doc_stringrr{rget_method_definitionssR   riteriterDP_file_pathz __init__.pyzutils.pyiterDP_files_to_excludeiterDP_deprecated_fileszList[IterDataPipe])demuxfork$iterDP_method_to_special_output_typemapmapDP_file_pathmapDP_files_to_excludemapDP_deprecated_filesshuffle IterDataPipe#mapDP_method_to_special_output_typecCs\ttttdt}ttttdt}t t j }d|dfd|dfg}tt|dd|dd S) a # Inject file into template datapipe.pyi.in. TODO: The current implementation of this script only generates interfaces for built-in methods. To generate interface for user-defined DataPipes, consider changing `IterDataPipe.register_datapipe_as_function`. r MapDataPipez${IterDataPipeMethods}z${MapDataPipeMethods}zdatapipe.pyi.inz datapipe.pyi)rrrrN)rrrrrrrrrrrrrrr$r)iter_method_definitionsmap_method_definitionsrrrrrmain s0 r__main__)rbro)rN)"rr collectionsrtypingrrrJrintrtupler$r0r7rBrEdictrZrarjrOrr__annotations__rrrrrrrr__name__rrrrsv    ". 4.   8   $